描述bug

在使用标准的RAG管道时，我遇到了上述错误。

错误信息

File "/home/felix/PycharmProjects/anychat/src/anychat/analysis/rag.py", line 124, in query_rag_in_document_store
    result = self.llm_pipeline.run(
             ^^^^^^^^^^^^^^^^^^^^^^
  File "/home/felix/anaconda3/envs/anychat/lib/python3.11/site-packages/haystack/core/pipeline/pipeline.py", line 197, in run
    res = comp.run(**last_inputs[name])
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/felix/anaconda3/envs/anychat/lib/python3.11/site-packages/haystack/components/generators/hugging_face_api.py", line 187, in run
    return self._run_non_streaming(prompt, generation_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/felix/anaconda3/envs/anychat/lib/python3.11/site-packages/haystack/components/generators/hugging_face_api.py", line 211, in _run_non_streaming
    tgr: TextGenerationOutput = self._client.text_generation(prompt, details=True, **generation_kwargs)
                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/felix/anaconda3/envs/anychat/lib/python3.11/site-packages/huggingface_hub/inference/_client.py", line 2061, in text_generation
    raise_text_generation_error(e)
  File "/home/felix/anaconda3/envs/anychat/lib/python3.11/site-packages/huggingface_hub/inference/_common.py", line 457, in raise_text_generation_error
    raise exception from http_error
huggingface_hub.errors.ValidationError: Input validation error: `inputs` must have less than 4095 tokens. Given: 4701

预期行为

我的预期是，内置有一个截断功能，可以截断输入，以便不会将太多的令牌传递给模型。理想情况下，输入应该在特定部分进行截断(例如，而不是在提示的末尾截断，那样问题也会被截断)。

重现

query_template = """Beantworte die Frage basierend auf dem nachfolgenden Kontext und Chatverlauf. Antworte so detailliert wie möglich, aber nur mit Informationen aus dem Kontext. Wenn du eine Antwort nicht weißt, sag, dass du sie nicht kennst. Wenn sich eine Frage nicht auf den Kontext bezieht, sag, dass du die Frage nicht beantworten kannst und höre auf. Stelle nie selbst eine Frage.

Kontext:
{% for document in documents %}
    {{ document.content }}
{% endfor %}

Vorheriger Chatverlauf:
{{ history }}

Frage: {{ question }}
Antwort: """

    def _create_generator(self):
        if AnyChatConfig.hf_use_local_generator:
            return HuggingFaceLocalGenerator(
                model=self.model_id,
                task="text2text-generation",
                device=ComponentDevice.from_str("cuda:0"),
                huggingface_pipeline_kwargs={
                    "device_map": "auto",
                    "model_kwargs": {
                        "load_in_4bit": True,
                        "bnb_4bit_use_double_quant": True,
                        "bnb_4bit_quant_type": "nf4",
                        "bnb_4bit_compute_dtype": torch.bfloat16,
                    },
                },
                generation_kwargs={"max_new_tokens": 350},
            )
        else:
            return HuggingFaceAPIGenerator(
                api_type="text_generation_inference",
                api_params={"url": AnyChatConfig.hf_api_generator_url},
            )

    def create_llm_pipeline(self, document_store):
        """
        Creates an LLM pipeline that employs RAG on a document store that must have been set up before.
        :return:
        """
        # create the pipeline with the individual components
        self.llm_pipeline = Pipeline()
        self.llm_pipeline.add_component(
            "embedder",
            SentenceTransformersTextEmbedder(
                model=DocumentManager.embedding_model_id,
                device=ComponentDevice.from_str(
                    AnyChatConfig.hf_device_rag_text_embedder
                ),
            ),
        )
        self.llm_pipeline.add_component(
            "retriever",
            InMemoryEmbeddingRetriever(document_store=document_store, top_k=8),
        )
        self.llm_pipeline.add_component(
            "prompt_builder", PromptBuilder(template=query_template)
        )

        self.llm_pipeline.add_component("llm", self._create_generator())

        # connect the individual nodes to create the final pipeline
        self.llm_pipeline.connect("embedder.embedding", "retriever.query_embedding")
        self.llm_pipeline.connect("retriever", "prompt_builder.documents")
        self.llm_pipeline.connect("prompt_builder", "llm")

    def _get_formatted_history(self):
        history = ""
        for message in self.conversation_history:
            history += f"{message[0]}: {message[1]}\n"
        history = history.strip()

        return history

    def query_rag_in_document_store(self, query):
        """
        Uses the LLM and RAG to provide an answer to the given query based on the documents in the document store.
        :param query:
        :return:
        """
        logger.debug("querying using rag with: {}", query)

        # run the query through the pipeline
        result = self.llm_pipeline.run(
            {
                "embedder": {"text": query},
                "prompt_builder": {
                    "question": query,
                    "history": self._get_formatted_history(),
                },
                "llm": {"generation_kwargs": {"max_new_tokens": 350}},
            }
        )
        response = result["llm"]["replies"][0]
        post_processed_response = self._post_process_response(response, query)
        logger.debug(post_processed_response)

        self.conversation_history.append(("Frage", query))
        self.conversation_history.append(("Antwort", post_processed_response))

        return post_processed_response

常见问题解答检查

• 你查看过our new FAQ page吗？
  系统：
• OS: debian
• GPU/CPU: NVIDIA RTX A6000 (CUDA 12.4)
• Haystack版本(提交或版本号): 2.2.1
• 文档存储：InMemoryDocumentStore
• 阅读器：PyPDFToDocument
• 检索器：InMemoryEmbeddingRetriever