logstash:Map中字段的错误序列

6qftjkof  于 2021-06-15  发布在  ElasticSearch
关注(0)|答案(1)|浏览(408)

我有这样一个logstash配置：

input {
    beats {
        port => '5044'
    }
}
filter {
    grok {
        patterns_dir => ['./patterns']
        match => { 'message' => '%{TIME:timestamp}(","Event\s)(?<Event>([^"]+))(","Form\s)?(?<Form>([^"]+))?(","ParentType\s)?(?<parent_type>([^"]+))?(","ParentName\s)?(?<parent_name>([^"]+))?(","Type\s)?(?<type>([^"]+))?(","Name\s)?(?<Name_of_form>([^"]+))?(","Detail\s)?(?<Detail>([^"]+))?(","t=)?(?<t>([\d]+))?(",")?(?<Status>(end|beg))?' }
        add_tag => [ '%{Status}' ]
    }
    dissect {
        mapping => {
            '[log][file][path]' => 'C:\Logs\%{login}\%{server}\%{user}\%{filename}.txt'
        }
    }
    date {
        match => [ 'timestamp', 'dd.MM.yyyy HH:mm:ss' ]
    }
    mutate {
        add_field => { 'uniqueEvent' => '%{Event}_%{filename}' }
    }
    elapsed {
        unique_id_field => 'uniqueEvent'
        start_tag => 'beg' 
        end_tag => 'end'
        new_event_on_match => false
    }

    if 'elapsed' in [tags] {
        aggregate {
            task_id => '%{uniqueEvent}'
            code => 'map["duration"] = [(event.get("elapsed_time")*1000).to_i]'
            map_action => 'create'
        }
    }
    mutate {
        remove_field => ['timestamp', 'ecs', 'log', 'tags', 'message', '@version', 'filename', 'input', 'host', 'agent', 't', 'parent_type', 'parent_name', 'type']
        rename => {'elapsed_time' => 'Event_duration'}
    }
}
output {
    elasticsearch {
        hosts => ['localhost:9200']
        index => 'logs-%{+dd.MM.YYYY}'
    }
}

在elasticsearch中，我需要的字段总是以不同的顺序显示，这有点恼火。我只有几个文本字段，两个日期字段和一个数字字段。elasticsearch本身完美地定义了类型，但是我希望所有的数据在每个索引中都有严格的顺序。这能做到吗？
Map应严格按照以下顺序进行：

"mappings": {
            "properties": {
                "@timestamp": {
                    "type": "date"
                },
                "Event": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "Form": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "Name_of_form": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "Detail": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "Event_duration": {
                    "type": "float"
                },
                "elapsed_timestamp_start": {
                    "type": "date"
                },
                "user": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "login": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "server": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "Status": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "uniqueEvent": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                }
            }
        }

我试着根据第一个也是唯一一个关于这个主题的指南来实现这个，但是没有任何效果。根据其他人的文章，我在logstash目录中创建了template.json，但是启动时这个模板不起作用，错误类型是：这个模板不在elasticsearch中。
如何正确创建此文件？如何将其正确写入输入块？这些都行吗？
谢谢你的帮助！
错误仍然是这样的：

[2020-08-25T19:33:13,365][INFO ][logstash.outputs.elasticsearch][main] Using mapping template from {:path=>"C:\\Users\\moral\\elk_stack\\logstash-7.8.1\\template\\template.json"}
[2020-08-25T19:33:13,406][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"template"=>"logs-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"properties"=>{"@timestamp"=>{"type"=>"date"}, "Event"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Form"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Name_of_form"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Detail"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Event_duration"=>{"type"=>"float"}, "elapsed_timestamp_start"=>{"type"=>"date"}, "user"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "login"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "server"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Status"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "uniqueEvent"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}}}}
[2020-08-25T19:33:13,421][INFO ][logstash.outputs.elasticsearch][main] Installing elasticsearch template to _template/log-template
[2020-08-25T19:33:13,468][ERROR][logstash.outputs.elasticsearch][main] Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://localhost:9200/_template/log-template'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:332:in `perform_request_to_url'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:319:in `block in perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:414:in `with_connection'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in `perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:326:in `block in Pool'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:352:in `template_put'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:86:in `template_install'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:28:in `install'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:16:in `install_template'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/common.rb:205:in `install_template'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/common.rb:49:in `block in setup_after_successful_connection'"]}

template.json文件：

{
  "template" : "logs-*",
  "version" : 50001,
  "settings" : {
    "index.refresh_interval" : "5s"
  },
  "mappings" : {
    "_default_" : {
        "properties": {
            "@timestamp": {
                "type": "date"
            },
            "Event": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "Form": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "Name_of_form": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "Detail": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "Event_duration": {
                "type": "float"
            },
            "elapsed_timestamp_start": {
                "type": "date"
            },
            "user": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "login": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "server": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "Status": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "uniqueEvent": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            }
        }
    }
  }
}
elasticsearchlogstashelastic-stacklogstash-configurationlogstash-grok

1条答案

按热度按时间
tjvv9vkg

tjvv9vkg1#

您需要做的只是告诉logstash在启动时安装模板。只要确保 index_patterns 在模板中匹配索引名（即。 logs-* )

output {
    elasticsearch {
        hosts => ['localhost:9200']
        index => 'logs-%{+dd.MM.YYYY}'

        manage_template => true
        template_overwrite => true
        template_name => "log-template"
        template => "/path/to/template.json"
    }
}
赞(0）分享  回复(0）举报  2021-06-15
我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com