sql_lora_path = "/home/zyn/models/slot_lora_gd"
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
llm = LLM(model="/home/models/dem_14b/base",
enable_lora=True,
trust_remote_code=True)
sampling_params = SamplingParams(temperature=0,
max_tokens=256,
stop=["[/assistant]"])
prompts = [
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
]
outputs = llm.generate(prompts,
sampling_params,
lora_request=LoRARequest("sql_adapter", 1,
sql_lora_path))
llm = LLM(model="/home/models/dem_14b/base",
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/entrypoints/llm.py", line 109, in init
self.llm_engine = LLMEngine.from_engine_args(engine_args)
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 391, in from_engine_args
engine = cls(*engine_configs,
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 128, in init
self._init_workers()
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 181, in _init_workers
self._run_workers("load_model")
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 1041, in _run_workers
driver_worker_output = getattr(self.driver_worker,
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/worker/worker.py", line 100, in load_model
self.model_runner.load_model()
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 88, in load_model
self.model = get_model(self.model_config,
File "/root/miniconda3/envs/qwen/lib/python3.10/site-packages/vllm