A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix][Neuron] Fix soft prompt method error in NeuronExecutor (#6313)
Woosuk Kwon committed
Jul 10, 2024 at 16:39 UTC
997df46a32f3b2c2debe3e17730895cef0d94d2a
1 file changed
+16
vllm/executor/neuron_executor.py
+16
@@ -70,6 +70,22 @@ class NeuronExecutor(ExecutorBase):
70
def list_loras(self) -> Set[int]:
71
return self.driver_worker.list_loras()
72
73
+ def add_prompt_adapter(self, prompt_adapter_request) -> bool:
74
+ raise NotImplementedError(
75
+ "Soft prompt is currently not supported by the Neuron backend.")
76
+
77
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
78
+ raise NotImplementedError(
79
+ "Soft prompt is currently not supported by the Neuron backend.")
80
+
81
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
82
+ raise NotImplementedError(
83
+ "Soft prompt is currently not supported by the Neuron backend.")
84
+
85
+ def list_prompt_adapters(self) -> Set[int]:
86
+ raise NotImplementedError(
87
+ "Soft prompt is currently not supported by the Neuron backend.")
88
+
89
def check_health(self) -> None:
90
# NeuronExecutor will always be healthy as long as
91
# it's running.