如何在discovery选项卡中获取日志中提供的字段

x6h2sr28  于 2021-06-10  发布在  ElasticSearch
关注(0)|答案(1)|浏览(350)

首先，我对kibana并不陌生，只不过是devops方面和索引Map。有人能帮助我如何为我的索引字段，如日志级别和价格？
提供给kibana的日志消息示例：
{“@timestamp”：“2020-09-28t19:09:05.438z”，“message”：“接收到的消息：{\n\”timestamp\“：\”2020-09-28t20:00:00\“，\n\”type\“：\”usd\“，\n\”price\“：10877.07，\n\”volume\“：39.065\n}”，“logger\u name”：“project.me.consumers.priceconsumer”，“thread\u name”：“defaultmessagelistenercontainer-2”，“level”：“info”}\n
我的当前字段：

它在日志下显示为json

{
"_index": "logstash-2020.09.28",
"_type": "_doc",
"_id": "------------",
"_version": 1,
"_score": null,
"_source": {
"log": "{\"@timestamp\":\"2020-09-28T19:09:05.438Z\",\"message\":\"Received Message: {\\n    \\\"timestamp\\\": \\\"2020-09-28T20:00:00\\\",\\n    \\\"type\\\": \\\"usd\\\",\\n    \\\"price\\\": 10877.07,\\n \\\"volume\\\":39.065\\n}\",\"logger_name\":\"project.me.consumers.PriceConsumer\",\"thread_name\":\"DefaultMessageListenerContainer-2\",\"level\":\"INFO\"}\n",
"stream": "stdout",
"docker": {
  "container_id": "------------------------------"
},
"kubernetes": {
  "container_name": "db-gateway",
  "namespace_name": "------------------------------",
  "pod_name": ------------------------------",
  "container_image": "------------------------------:1.0.0-b40",
  "container_image_id": "docker-pullable://------------------------------",
  "pod_id": "------------------------------",
  "labels": {
    "app": "db-gateway",
    "pod-template-hash": "------------------------------"
  },
  "host": "------------------------------",
  "master_url": "https://------------------------------:443/api",
  "namespace_id": "------------------------------"
},
"@timestamp": "2020-09-28T19:09:05.438928347+00:00",
"tag": "------------------------------"
},
"fields": {
"@timestamp": [
  "2020-09-28T19:09:05.438Z"
]
},
"highlight": {
"kubernetes.container_name": [
  "@kibana-highlighted-field@db@/kibana-highlighted-field@-gateway"
]
},
"sort": [
1601320145438
]
}

使用fluentd和logback，一切都在我的kubernetes集群上。
我在日志字段中尝试的配置：

<filter log.**>
  @type parser
  format json # apache2, nginx, etc...
  key_name log
  reserve_data true
</filter>

<filter log.**>
  @type concat
  key log
  stream_identity_key container_id
  multiline_start_regexp /^-e:2:in `\/'/
  multiline_end_regexp /^-e:4:in/
</filter>

<filter _source.log.**>
  @type parser
  format json # apache2, nginx, etc...
  key_name log
  reserve_data true
</filter>

我的fluentd配置：

<match fluent.**>
  @type null
</match>
<source>
  @type tail
  @id in_tail_container_logs
  path /var/log/containers/*.log
  pos_file /var/log/fluentd-containers.log.pos
  tag kubernetes.*
  read_from_head true
  <parse>
    @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
    time_format %Y-%m-%dT%H:%M:%S.%NZ
  </parse>
</source>
<source>
  @type tail
  @id in_tail_minion
  path /var/log/salt/minion
  pos_file /var/log/fluentd-salt.pos
  tag salt
  <parse>
    @type regexp
    expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
    time_format %Y-%m-%d %H:%M:%S
  </parse>
</source>
<source>
  @type tail
  @id in_tail_startupscript
  path /var/log/startupscript.log
  pos_file /var/log/fluentd-startupscript.log.pos
  tag startupscript
  <parse>
    @type syslog
  </parse>
</source>
<source>
  @type tail
  @id in_tail_docker
  path /var/log/docker.log
  pos_file /var/log/fluentd-docker.log.pos
  tag docker
  <parse>
    @type regexp
    expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
  </parse>
</source>
<source>
  @type tail
  @id in_tail_etcd
  path /var/log/etcd.log
  pos_file /var/log/fluentd-etcd.log.pos
  tag etcd
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id in_tail_kubelet
  multiline_flush_interval 5s
  path /var/log/kubelet.log
  pos_file /var/log/fluentd-kubelet.log.pos
  tag kubelet
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_kube_proxy
  multiline_flush_interval 5s
  path /var/log/kube-proxy.log
  pos_file /var/log/fluentd-kube-proxy.log.pos
  tag kube-proxy
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_kube_apiserver
  multiline_flush_interval 5s
  path /var/log/kube-apiserver.log
  pos_file /var/log/fluentd-kube-apiserver.log.pos
  tag kube-apiserver
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_kube_controller_manager
  multiline_flush_interval 5s
  path /var/log/kube-controller-manager.log
  pos_file /var/log/fluentd-kube-controller-manager.log.pos
  tag kube-controller-manager
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_kube_scheduler
  multiline_flush_interval 5s
  path /var/log/kube-scheduler.log
  pos_file /var/log/fluentd-kube-scheduler.log.pos
  tag kube-scheduler
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_rescheduler
  multiline_flush_interval 5s
  path /var/log/rescheduler.log
  pos_file /var/log/fluentd-rescheduler.log.pos
  tag rescheduler
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_glbc
  multiline_flush_interval 5s
  path /var/log/glbc.log
  pos_file /var/log/fluentd-glbc.log.pos
  tag glbc
  <parse>
    @type kubernetes
  </parse>
</source>
<source>
  @type tail
  @id in_tail_cluster_autoscaler
  multiline_flush_interval 5s
  path /var/log/cluster-autoscaler.log
  pos_file /var/log/fluentd-cluster-autoscaler.log.pos
  tag cluster-autoscaler
  <parse>
    @type kubernetes
  </parse>
</source>

# Example:

# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"

# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"

<source>
  @type tail
  @id in_tail_kube_apiserver_audit
  multiline_flush_interval 5s
  path /var/log/kubernetes/kube-apiserver-audit.log
  pos_file /var/log/kube-apiserver-audit.log.pos
  tag kube-apiserver-audit
  <parse>
    @type multiline
    format_firstline /^\S+\s+AUDIT:/
    # Fields must be explicitly captured by name to be parsed into the record.
    # Fields may not always be present, and order may change, so this just looks
    # for a list of key="\"quoted\" value" pairs separated by spaces.
    # Unknown fields are ignored.
    # Note: We can't separate query/response lines as format1/format2 because
    #       they don't always come one after the other for a given query.
format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
    time_format %Y-%m-%dT%T.%L%Z
  </parse>
</source>
<filter kubernetes.**>
  @type kubernetes_metadata
  @id filter_kube_metadata
</filter>
elasticsearchspring-bootlogbackkibana

1条答案

按热度按时间
xdnvmnnf

xdnvmnnf1#

为该类型设置的Map的问题是 log 字段被视为文本（甚至可能是关键字）。elasticsearch不会尝试将提交的文本字段解析为json对象结构。这就是为什么在将有效负载放到elasticsearch之前需要在管道中更早地进行转换。
你可以使用fluentdjson解析插件。请看这里的文件。还有一些关于docker日志记录和解析 log 现场。

赞(0）分享  回复(0）举报  2021-06-10
我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com