OpenSearch删除的文档没有释放磁盘空间，搜索性能下降

llmtgqce  于 2022-09-20  发布在  ElasticSearch
关注(0)|答案(1)|浏览(258)

我们正在使用具有ElasticSearch v7.10.2的OpenSearch，我们有一个大型索引，其中我们定期索引数据并定期删除数据(一个月前的数据)，但随着时间的推移，我们在主要使用HAS_CHILD查询的查询中遇到了搜索性能下降，我想知道被删除的文档是否仍在我的索引中引用，它们是否仍在消耗资源，如果是，那么我如何删除它们，下面我附上了几个统计数据。

获取_cat/segments?index=segment_index_570de84a4f0d925f98343571&s=docs.deleted

我的一些数据段也达到了5 GB，为了更好地理解，这里仅附上几个数据段示例

segment_index_570de84a4f0d925f98343571 83 r x.x.x.x _36u6 148830  4566709 5083016    4.2gb 292024 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 83 p x.x.x.x _36u6 148830  4566709 5083016    4.2gb 292024 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 84 p x.x.x.x  _37hm 149674  4569844 5127644    4.4gb 290112 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 84 r x.x.x.x _37hm 149674  4569838 5127650    4.4gb 290112 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 40 p x.x.x.x _30oh 140849  3765550 5460181    4.7gb 292616 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 40 r x.x.x.x  _30oh 140849  3765544 5460187    4.7gb 292616 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 29 r x.x.x.x _2ygs 137980  3359481 5519471    4.9gb 287504 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 29 p x.x.x.x  _2ygs 137980  3359481 5519471    4.9gb 287504 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 13 p x.x.x.x  _2u1h 132245  2731374 5565823      4gb 289480 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 13 r x.x.x.x _2u1h 132245  2731372 5565825      4gb 289480 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 57 p x.x.x.x  _3bx3 155415  4144634 5615761    4.5gb 304728 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 57 r x.x.x.x _3bx3 155415  4144620 5615775    4.5gb 304728 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 10 r x.x.x.x  _2yau 137766  3535782 5821903    4.6gb 290072 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 10 p x.x.x.x _2yau 137766  3535778 5821907    4.6gb 290072 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 5  r x.x.x.x _2vsz 134531  3052529 5940943      5gb 287288 true  true  8.7.0 false
segment_index_570de84a4f0d925f98343571 5  p x.x.x.x  _2vsz 134531  3052529 5940943      5gb 287288 true  true  8.7.0 false

GET SECTION_INDEX_570de84a4f0d925f98343571/_STATS

{
  "_shards" : {
    "total" : 200,
    "successful" : 200,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 2764210965,
        "deleted" : 768121801
      },
      "store" : {
        "size_in_bytes" : 1882036902899,
        "reserved_in_bytes" : 0
      },
      "indexing" : {
        "index_total" : 35049143,
        "index_time_in_millis" : 93342006,
        "index_current" : 1,
        "index_failed" : 7,
        "delete_total" : 28158400,
        "delete_time_in_millis" : 2241164,
        "delete_current" : 0,
        "noop_update_total" : 108,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 8272612,
        "time_in_millis" : 9407739,
        "exists_total" : 7714730,
        "exists_time_in_millis" : 6864869,
        "missing_total" : 557882,
        "missing_time_in_millis" : 2542870,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 99,
        "query_total" : 781661,
        "query_time_in_millis" : 46180985,
        "query_current" : 0,
        "fetch_total" : 25828,
        "fetch_time_in_millis" : 31922549,
        "fetch_current" : 0,
        "scroll_total" : 150005,
        "scroll_time_in_millis" : 3934488045,
        "scroll_current" : 99,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size_in_bytes" : 0,
        "total" : 5069,
        "total_time_in_millis" : 15670663,
        "total_docs" : 232287891,
        "total_size_in_bytes" : 144734357228,
        "total_stopped_time_in_millis" : 0,
        "total_throttled_time_in_millis" : 3012035,
        "total_auto_throttle_in_bytes" : 1635281344
      },
      "refresh" : {
        "total" : 26258,
        "total_time_in_millis" : 24294337,
        "external_total" : 23116,
        "external_total_time_in_millis" : 51934585,
        "listeners" : 0
      },
      "flush" : {
        "total" : 129,
        "periodic" : 29,
        "total_time_in_millis" : 83553
      },
      "warmer" : {
        "current" : 0,
        "total" : 23099,
        "total_time_in_millis" : 28150896
      },
      "query_cache" : {
        "memory_size_in_bytes" : 1971367836,
        "total_count" : 5941141,
        "hit_count" : 1281540,
        "miss_count" : 4659601,
        "cache_size" : 34136,
        "cache_count" : 34655,
        "evictions" : 519
      },
      "fielddata" : {
        "memory_size_in_bytes" : 2270860360,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 3062,
        "memory_in_bytes" : 686053834,
        "terms_memory_in_bytes" : 583830952,
        "stored_fields_memory_in_bytes" : 2159936,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 84022720,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 16040226,
        "index_writer_memory_in_bytes" : 3083169874,
        "version_map_memory_in_bytes" : 14212574,
        "fixed_bit_set_memory_in_bytes" : 441678080,
        "max_unsafe_auto_id_timestamp" : -1,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 1145991,
        "size_in_bytes" : 1862599220,
        "uncommitted_operations" : 1145991,
        "uncommitted_size_in_bytes" : 1862599220,
        "earliest_last_modified_age" : 10
      },
      "request_cache" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0,
        "hit_count" : 692,
        "miss_count" : 7824
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 146589584
      }
    },
    "total" : {
      "docs" : {
        "count" : 5528419715,
        "deleted" : 1568758887
      },
      "store" : {
        "size_in_bytes" : 3779599075512,
        "reserved_in_bytes" : 0
      },
      "indexing" : {
        "index_total" : 65246167,
        "index_time_in_millis" : 116379853,
        "index_current" : 2,
        "index_failed" : 7,
        "delete_total" : 56316800,
        "delete_time_in_millis" : 4569453,
        "delete_current" : 0,
        "noop_update_total" : 108,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 8279717,
        "time_in_millis" : 9461541,
        "exists_total" : 7721681,
        "exists_time_in_millis" : 6917878,
        "missing_total" : 558036,
        "missing_time_in_millis" : 2543663,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 200,
        "query_total" : 1421264,
        "query_time_in_millis" : 84711977,
        "query_current" : 0,
        "fetch_total" : 47121,
        "fetch_time_in_millis" : 55494456,
        "fetch_current" : 2,
        "scroll_total" : 282690,
        "scroll_time_in_millis" : 6909135621,
        "scroll_current" : 200,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size_in_bytes" : 0,
        "total" : 8563,
        "total_time_in_millis" : 30676821,
        "total_docs" : 452795172,
        "total_size_in_bytes" : 273814327525,
        "total_stopped_time_in_millis" : 0,
        "total_throttled_time_in_millis" : 6337362,
        "total_auto_throttle_in_bytes" : 3305840977
      },
      "refresh" : {
        "total" : 47329,
        "total_time_in_millis" : 46367778,
        "external_total" : 43783,
        "external_total_time_in_millis" : 98641382,
        "listeners" : 0
      },
      "flush" : {
        "total" : 298,
        "periodic" : 98,
        "total_time_in_millis" : 210368
      },
      "warmer" : {
        "current" : 0,
        "total" : 43760,
        "total_time_in_millis" : 52941301
      },
      "query_cache" : {
        "memory_size_in_bytes" : 3882183058,
        "total_count" : 10826442,
        "hit_count" : 2195511,
        "miss_count" : 8630931,
        "cache_size" : 66063,
        "cache_count" : 66884,
        "evictions" : 821
      },
      "fielddata" : {
        "memory_size_in_bytes" : 4524309840,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 6121,
        "memory_in_bytes" : 1359222728,
        "terms_memory_in_bytes" : 1155693088,
        "stored_fields_memory_in_bytes" : 4324024,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 166294144,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 32911472,
        "index_writer_memory_in_bytes" : 5666776518,
        "version_map_memory_in_bytes" : 26231773,
        "fixed_bit_set_memory_in_bytes" : 887417576,
        "max_unsafe_auto_id_timestamp" : -1,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 31206542,
        "size_in_bytes" : 28262050766,
        "uncommitted_operations" : 31206542,
        "uncommitted_size_in_bytes" : 28262050766,
        "earliest_last_modified_age" : 10
      },
      "request_cache" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0,
        "hit_count" : 1296,
        "miss_count" : 13655
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 229545608
      }
    }
  },
  "indices" : {
    "segment_index_570de84a4f0d925f98343571" : {
      "uuid" : "fhZUqTwfSeum3hHlyFaILw",
      "primaries" : {
        "docs" : {
          "count" : 2764210965,
          "deleted" : 768121801
        },
        "store" : {
          "size_in_bytes" : 1882036902899,
          "reserved_in_bytes" : 0
        },
        "indexing" : {
          "index_total" : 35049143,
          "index_time_in_millis" : 93342006,
          "index_current" : 1,
          "index_failed" : 7,
          "delete_total" : 28158400,
          "delete_time_in_millis" : 2241164,
          "delete_current" : 0,
          "noop_update_total" : 108,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 8272612,
          "time_in_millis" : 9407739,
          "exists_total" : 7714730,
          "exists_time_in_millis" : 6864869,
          "missing_total" : 557882,
          "missing_time_in_millis" : 2542870,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 99,
          "query_total" : 781661,
          "query_time_in_millis" : 46180985,
          "query_current" : 0,
          "fetch_total" : 25828,
          "fetch_time_in_millis" : 31922549,
          "fetch_current" : 0,
          "scroll_total" : 150005,
          "scroll_time_in_millis" : 3934488045,
          "scroll_current" : 99,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 0,
          "current_docs" : 0,
          "current_size_in_bytes" : 0,
          "total" : 5069,
          "total_time_in_millis" : 15670663,
          "total_docs" : 232287891,
          "total_size_in_bytes" : 144734357228,
          "total_stopped_time_in_millis" : 0,
          "total_throttled_time_in_millis" : 3012035,
          "total_auto_throttle_in_bytes" : 1635281344
        },
        "refresh" : {
          "total" : 26258,
          "total_time_in_millis" : 24294337,
          "external_total" : 23116,
          "external_total_time_in_millis" : 51934585,
          "listeners" : 0
        },
        "flush" : {
          "total" : 129,
          "periodic" : 29,
          "total_time_in_millis" : 83553
        },
        "warmer" : {
          "current" : 0,
          "total" : 23099,
          "total_time_in_millis" : 28150896
        },
        "query_cache" : {
          "memory_size_in_bytes" : 1971367836,
          "total_count" : 5941141,
          "hit_count" : 1281540,
          "miss_count" : 4659601,
          "cache_size" : 34136,
          "cache_count" : 34655,
          "evictions" : 519
        },
        "fielddata" : {
          "memory_size_in_bytes" : 2270860360,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 3062,
          "memory_in_bytes" : 686053834,
          "terms_memory_in_bytes" : 583830952,
          "stored_fields_memory_in_bytes" : 2159936,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 84022720,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 16040226,
          "index_writer_memory_in_bytes" : 3083169874,
          "version_map_memory_in_bytes" : 14212574,
          "fixed_bit_set_memory_in_bytes" : 441678080,
          "max_unsafe_auto_id_timestamp" : -1,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 1145991,
          "size_in_bytes" : 1862599220,
          "uncommitted_operations" : 1145991,
          "uncommitted_size_in_bytes" : 1862599220,
          "earliest_last_modified_age" : 10
        },
        "request_cache" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0,
          "hit_count" : 692,
          "miss_count" : 7824
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 146589584
        }
      },
      "total" : {
        "docs" : {
          "count" : 5528419715,
          "deleted" : 1568758887
        },
        "store" : {
          "size_in_bytes" : 3779599075512,
          "reserved_in_bytes" : 0
        },
        "indexing" : {
          "index_total" : 65246167,
          "index_time_in_millis" : 116379853,
          "index_current" : 2,
          "index_failed" : 7,
          "delete_total" : 56316800,
          "delete_time_in_millis" : 4569453,
          "delete_current" : 0,
          "noop_update_total" : 108,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 8279717,
          "time_in_millis" : 9461541,
          "exists_total" : 7721681,
          "exists_time_in_millis" : 6917878,
          "missing_total" : 558036,
          "missing_time_in_millis" : 2543663,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 200,
          "query_total" : 1421264,
          "query_time_in_millis" : 84711977,
          "query_current" : 0,
          "fetch_total" : 47121,
          "fetch_time_in_millis" : 55494456,
          "fetch_current" : 2,
          "scroll_total" : 282690,
          "scroll_time_in_millis" : 6909135621,
          "scroll_current" : 200,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 0,
          "current_docs" : 0,
          "current_size_in_bytes" : 0,
          "total" : 8563,
          "total_time_in_millis" : 30676821,
          "total_docs" : 452795172,
          "total_size_in_bytes" : 273814327525,
          "total_stopped_time_in_millis" : 0,
          "total_throttled_time_in_millis" : 6337362,
          "total_auto_throttle_in_bytes" : 3305840977
        },
        "refresh" : {
          "total" : 47329,
          "total_time_in_millis" : 46367778,
          "external_total" : 43783,
          "external_total_time_in_millis" : 98641382,
          "listeners" : 0
        },
        "flush" : {
          "total" : 298,
          "periodic" : 98,
          "total_time_in_millis" : 210368
        },
        "warmer" : {
          "current" : 0,
          "total" : 43760,
          "total_time_in_millis" : 52941301
        },
        "query_cache" : {
          "memory_size_in_bytes" : 3882183058,
          "total_count" : 10826442,
          "hit_count" : 2195511,
          "miss_count" : 8630931,
          "cache_size" : 66063,
          "cache_count" : 66884,
          "evictions" : 821
        },
        "fielddata" : {
          "memory_size_in_bytes" : 4524309840,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 6121,
          "memory_in_bytes" : 1359222728,
          "terms_memory_in_bytes" : 1155693088,
          "stored_fields_memory_in_bytes" : 4324024,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 166294144,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 32911472,
          "index_writer_memory_in_bytes" : 5666776518,
          "version_map_memory_in_bytes" : 26231773,
          "fixed_bit_set_memory_in_bytes" : 887417576,
          "max_unsafe_auto_id_timestamp" : -1,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 31206542,
          "size_in_bytes" : 28262050766,
          "uncommitted_operations" : 31206542,
          "uncommitted_size_in_bytes" : 28262050766,
          "earliest_last_modified_age" : 10
        },
        "request_cache" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0,
          "hit_count" : 1296,
          "miss_count" : 13655
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 229545608
        }
      }
    }
  }
}

我会要求以适当的方法指导我处理这一问题并优化搜索性能的最佳方法

elasticsearch

1条答案

按热度按时间
iyzzxitl

iyzzxitl1#

在这种情况下，我通常会运行forcEmerge，只删除已删除的文档

POST _forcemerge?only_expunge_deletes=true

由于删除的文档占总文档的比例约为30%，这应该会让您重新获得一些空间...

首先在单个索引上尝试它。记录之前和之后的大小，您应该会看到一些空间增加。

赞(0）分享  回复(0）举报  2022-09-20
我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com