@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 997df46a32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix][Neuron] Fix soft prompt method error in NeuronExecutor (#6313)

Woosuk Kwon committed Jul 10, 2024 at 16:39 UTC 997df46a32f3b2c2debe3e17730895cef0d94d2a
1 file changed +16
vllm/executor/neuron_executor.py
+16
@@ -70,6 +70,22 @@ class NeuronExecutor(ExecutorBase):
70 def list_loras(self) -> Set[int]:
71 return self.driver_worker.list_loras()
72
73 + def add_prompt_adapter(self, prompt_adapter_request) -> bool:
74 + raise NotImplementedError(
75 + "Soft prompt is currently not supported by the Neuron backend.")
76 +
77 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
78 + raise NotImplementedError(
79 + "Soft prompt is currently not supported by the Neuron backend.")
80 +
81 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
82 + raise NotImplementedError(
83 + "Soft prompt is currently not supported by the Neuron backend.")
84 +
85 + def list_prompt_adapters(self) -> Set[int]:
86 + raise NotImplementedError(
87 + "Soft prompt is currently not supported by the Neuron backend.")
88 +
89 def check_health(self) -> None:
90 # NeuronExecutor will always be healthy as long as
91 # it's running.