使用nodejs和elasticsearch过滤数据

sg3maiej  于 2023-02-08  发布在  Node.js
关注(0)|答案(1)|浏览(121)
    • bounty已结束**。此问题的答案可获得+50的信誉奖励。奖励宽限期将在20小时后结束。Richard Branson正在寻找来自信誉良好来源的答案

我在ReactJS中实现的datatable遇到了一个问题。我正在从elasticsearch中检索数据并填充datatable。在没有应用过滤器的情况下,数据检索过程工作正常,但是,当我对数据应用过滤器时,datatable仍然为空,即使data_source有匹配的记录。

    • 我要发送的参数结构如下:**
{
  pageIndex: 1,
  pageSize: 10,
  sort: { order: '', key: '' },
  query: '',
  filterData: {
    analysis: [ '0', '1', '2', '3' ],
    threat_level_id: [ '1', '2', '3', '4' ],
  }
}
    • 终点:**

POST/api/v1/事件/公共/列表

    • 控制器:**
exports.getPublicEvents = async (req, res) => {
  try {
    client.ping()
    const { pageIndex, pageSize, sort, query, filterData } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          bool: {
            must: [
              {
                match_all: {},
              },
            ],
            filter: [],
          },
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query.bool.must = [
        {
          match: {
            'Event.info': {
              query: query,
              fuzziness: 'AUTO',
            },
          },
        },
      ]
    }
    if (filterData.analysis.length > 0) {
      esQuery.body.query.bool.filter.push({
        terms: {
          'Event.analysis': filterData.analysis,
        },
      })
    }
    if (filterData.threat_level_id.length > 0) {
      esQuery.body.query.bool.filter.push({
        terms: {
          'Event.threat_level_id': filterData.threat_level_id,
        },
      })
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}

下面的控制器是没有过滤器,它的工作刚刚好.

exports.getPublicEvents = async (req, res) => {
  try {
    client.ping()
    const { pageIndex, pageSize, sort, query } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          match_all: {},
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query = {
        match: {
          'Event.info': {
            query: query,
            fuzziness: 'AUTO',
          },
        },
      }
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}
    • 弹性密封版本:七、十七、八**
    • 结果:控制台日志(JSON字符串化(esQuery))**
{
  "index": "INDEX_NAME",
  "body": {
    "query": {
      "bool": {
        "must": [{ "match_all": {} }],
        "filter": [
          { "terms": { "Event.analysis": ["0", "1", "2"] } },
          { "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
        ]
      }
    },
    "from": 0,
    "size": 10
  }
}
    • ElasticSearch架构中的数据**
{
    "@version": "1",
    "@timestamp": "2023-02-01T14:43:09.997Z",
    "Event": {
        "info": ".......................",
        
        "description": ".......................",
        "analysis": 0,
        "threat_level_id": "4",
        "created_at": 1516566351,
        "uuid": "5a64f74f0e543738c12bc973322",
        "updated_at": 1675262417
    }
}
    • 索引Map**
{
    "index_patterns": ["INDEX_NAME"],
    "template": "TEMPLATE_NAME",
    "settings": {
      "number_of_replicas": 0,
      "index.mapping.nested_objects.limit": 10000000
      },
    "mappings": {
      "dynamic": false,
      "properties": {
          "@timestamp": {
          "type": "date"
        },
        "Event": {
          "type": "nested",
          "properties": {
            "date_occured": {
              "type": "date"
            },
            "threat_level_id": {
              "type": "integer"
            },
            "description": {
              "type": "text"
            },
            "is_shared": {
              "type": "boolean"
            },
            "analysis": {
              "type": "integer"
            },
            "uuid": {
              "type": "text"
            },
            "created_at": {
              "type": "date"
            },
            "info": {
              "type": "text"
            },
            "shared_with": {
                "type": "nested",
                 "properties": {
                  "_id": {
                    "type": "text"
                }
              }
            },
            "updated_at": {
              "type": "date"
            },
            "author": {
              "type": "text"
            },
            "Attributes": {
              "type": "nested",
              "properties": {
                "data": {
                  "type": "text"
                },
                "type": {
                  "type": "text"
                },
                "uuid": {
                  "type": "text"
                },
                "comment": {
                  "type": "text"
                },
                "category": {
                  "type": "text"
                },
                "value": {
                  "type": "text"
                },
                "timestamp": {
                  "type": "date"
                }
              }
            }, 
            "organisation": {
              "type": "nested",
              "properties": {
                "name": {
                  "type": "text"
                },
                "uuid": {
                  "type": "text"
                }
              }
            },
            "Tags": {
              "type": "nested",
              "properties": {
                "color": {
                  "type": "text"
                },
                "name": {
                  "type": "text"
                }
              }
            },
            "TLP": {
              "type": "nested",
              "properties": {
                "color": {
                  "type": "text"
                },
                "name": {
                  "type": "text"
                }
              }
            }
          }
        }  
      }
    }
  }
x8diyxa7

x8diyxa71#

Event是一个嵌套字段,因此需要使用嵌套查询,如下所示:

{
  "index": "INDEX_NAME",
  "body": {
    "query": {
      "bool": {
        "must": [{ "match_all": {} }],
        "filter": [
          {
            "nested": {
              "path": "Event",
              "query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
            } 
          },
          {
            "nested": {
              "path": "Event",
              "query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
            } 
          }
        ]
      }
    },
    "from": 0,
    "size": 10
  }
}

相关问题