我使用ES reindex将文档从src索引复制到目标索引。在这样做的同时，我正在运行一些摄取管道。重新索引非常慢。1小时内甚至没有上传一个文档。我在MacBook Pro 2022上运行ES。
下面是重新索引任务的详细信息。
{
"completed": false,
"task": {
"node": "zn7bFe98Qly-eEd_VrXAMQ",
"id": 688,
"type": "transport",
"action": "indices:data/write/reindex",
"status": {
"total": 2461,
"updated": 0,
"created": 0,
"deleted": 0,
"batches": 1,
"version_conflicts": 0,
"noops": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"requests_per_second": 1,
"throttled_until_millis": 0
},
"description": "reindex from [source_index] to [destination_index]",
"start_time_in_millis": 1681734787348,
"running_time_in_nanos": 1800085540069,
"cancellable": true,
"cancelled": false,
"headers": {}
}
}你可以从2461年看到;什么都没有创造出来。
我用requests_per_second=1
而摄入管道是
{
"document-pipeline": {
"description": "abc",
"on_failure": [
{
"set": {
"description": "Index document to 'failed-<index>'",
"field": "_index",
"value": "failed-{{{_index}}}"
}
},
{
"set": {
"description": "Sazzad pycharm ingestion failed error message",
"field": "ingest.failure",
"value": "{{_ingest.on_failure_message}}"
}
}
],
"processors": [
{
"inference": {
"model_id": "sentence-transformers__paraphrase-mpnet-base-v2",
"target_field": "name_embedding",
"field_map": {
"name": "text_field"
}
}
},
{
"inference": {
"model_id": "sentence-transformers__paraphrase-mpnet-base-v2",
"target_field": "description_embedding",
"field_map": {
"description": "text_field"
}
}
},
{
"inference": {
"model_id": "sentence-transformers__paraphrase-mpnet-base-v2",
"target_field": "custom_field1_embedding",
"field_map": {
"custom_field1": "text_field"
}
}
}
}
}
]
}
}有线索吗？引擎盖下面有什么问题？
编辑2 -GET _cat/tasks/?v&actions=*reindex&detailed的输出
`action task_id parent_task_id type start_time timestamp running_time ip node description
indices:data/write/reindex zn7bFe98Qly-eEd_VrXAMQ:1250 - transport 1681738821297 13:40:21 3.2m 172.19.0.2 4cdda1de1ed0 reindex from [source] to [destination]编辑3 -GET _nodes/hot_threads的输出
::: {4cdda1de1ed0}{zn7bFe98Qly-eEd_VrXAMQ}{nlHIUMF4SnGX8h_vrEYVYA}{4cdda1de1ed0}{172.19.0.2}{172.19.0.2:9300}{cdfhilmrstw}{8.7.0}{ml.max_jvm_size=4118806528, ml.allocated_processors=4, ml.machine_memory=8233017344, xpack.installed=true, ml.allocated_processors_double=4.0}
Hot threads at 2023-04-17T14:08:51.922Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
0.6% [cpu=0.6%, idle=99.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[4cdda1de1ed0][transport_worker][T#4]'
3/10 snapshots sharing following 3 elements
io.netty.common@4.1.86.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
io.netty.common@4.1.86.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@19.0.2/java.lang.Thread.run(Thread.java:1589)编辑4：GET /_nodes/stats?pretty&filter_path=**.ingest.pipelines的输出
{
"nodes": {
"zn7bFe98Qly-eEd_VrXAMQ": {
"ingest": {
"pipelines": {
"recipe-name_description_custom_steps_custom_ingredients_custom_tags-field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"recipe-custom_steps-field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"recipe-description-field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"recipe--field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 1000,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 1000,
"time_in_millis": 288844826,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 471,
"time_in_millis": 645430266,
"current": 529,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 471,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"recipe-name-field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"recipe-custom_ingredients-field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"synthetics-browser.screenshot-0.11.8": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"pipeline:synthetics-browser.screenshot@custom": {
"type": "pipeline",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
".fleet_final_pipeline-1": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"date:truncate-subseconds-event-ingested": {
"type": "date",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"remove": {
"type": "remove",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"set_security_user": {
"type": "set_security_user",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"script:agent-id-status": {
"type": "script",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"remove": {
"type": "remove",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"synthetics-icmp-0.11.8": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"pipeline:synthetics-icmp@custom": {
"type": "pipeline",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"synthetics-http-0.11.8": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"pipeline:synthetics-http@custom": {
"type": "pipeline",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"recipe-name_description_custom_steps_custom_ingredients-field-embeddings-pipeline": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
},
{
"inference": {
"type": "inference",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"synthetics-browser.network-0.11.8": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"pipeline:synthetics-browser.network@custom": {
"type": "pipeline",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"synthetics-browser-0.11.8": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"pipeline:synthetics-browser@custom": {
"type": "pipeline",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
},
"synthetics-tcp-0.11.8": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"pipeline:synthetics-tcp@custom": {
"type": "pipeline",
"stats": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
]
}
}
}
}
}
}
1条答案
按热度按时间ajsxfq5m1#
如果查看
recipe--field-embeddings-pipeline，您可以看到管道中有1000个文档：这意味着它正在工作，但这需要时间，可能是因为
requests_per_second=1使它真的很慢。所以没什么错，只是慢。你可以通过
GET /_nodes/stats?pretty&filter_path=**.ingest.pipelines调优和跟踪进度，看看是否能让它走得更快。