上传JSON文件到elasticsearch/kibana

syqv5f0l  于 2022-12-09  发布在  Kibana
关注(0)|答案(1)|浏览(198)

这段代码正在将access.log日志转换为JSON格式。我正在尝试将其上传到elasticsearch,这样索引也会创建。我该如何做?

import json 
import re
import logging
import ecs_logging
import time
import sys, requests, os
import time
from datetime import datetime
from elasticsearch import Elasticsearch
from pprint import pprint

res = requests.get('http://localhost:9200')
print (res.content)

es = Elasticsearch([{'host': 'localhost', 'port': '9200'}])

i = 0
result = {}

with open('access.log') as f:
    lines = f.readlines()

regex = '(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>.*)\] \"(?P<httpstatus>(GET|POST) .+ HTTP\/1\.1)\" (?P<returnstatus>\d{3} \d+) (\".*\")(?P<browserinfo>.*)\"'

for line in lines:

    r = re.match(regex,line)
    
    if r != None:
        result[i] = {'IP address': r.group('ipaddress'), 'Time Stamp': r.group('dateandtime'), 
                     'HTTP status': r.group('httpstatus'), 'Return status': 
                     r.group('returnstatus'), 'Browser Info': r.group('browserinfo')}
        i += 1
    
print(result)

with open('data.json', 'w') as fp:
    json.dump(result, fp) ```
zlhcx6iw

zlhcx6iw1#

修改了您的代码,希望这对您有所帮助,如果您启用了身份验证,那么我们应该提到相同的内容,并且您还要求创建一个名为“my-index”的索引

import json 
import re
import logging
import ecs_logging
import time
import sys, requests, os
import time
from datetime import datetime
from elasticsearch import Elasticsearch, helpers
from pprint import pprint
import json
from datetime import datetime
from time import sleep
from requests.auth import HTTPBasicAuth

client = Elasticsearch(["localhost:9200"], http_auth=('elastic', '<your-es-pwd>'))
#no authentication
#client = Elasticsearch(["localhost:9200"], http_auth=('elastic', '<your-es-pwd>'))

i = 0
result = {}

with open('access.log') as f:
    lines = f.readlines()

regex = '(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>.*)\] \"(?P<httpstatus>(GET|POST) .+ HTTP\/1\.1)\" (?P<returnstatus>\d{3} \d+) (\".*\")(?P<browserinfo>.*)\"'

for line in lines:

    r = re.match(regex,line)
    
    if r != None:
        result[i] = {'IP address': r.group('ipaddress'), 'Time Stamp': r.group('dateandtime'), 
                     'HTTP status': r.group('httpstatus'), 'Return status': 
                     r.group('returnstatus'), 'Browser Info': r.group('browserinfo')}
        i += 1

with open('data.json', 'w') as fp:
    json.dump(result, fp)
    
directory = '.'

def load_json(directory):
    data = []
    for filename in os.listdir(directory):
        if filename.endswith('data.json'):
            with open(filename, 'r') as open_file:
                json_data = json.load(open_file)
                for i,k in enumerate(json_data.keys()):
                    data.append({
                        "_index": "my-index",
                         "_type": "my-type",
                        "_id": i,
                        "_source": json_data[k]
                    })
    return data

helpers.bulk(client, load_json('.'))

res = requests.get('http://localhost:9200', auth = HTTPBasicAuth('elastic', '<your-password'))
#no authentication enabled
#res = requests.get('http://localhost:9200')
print(res.content)

相关问题