@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5ed3505d82
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix][TPU] Add prompt adapter methods to TPUExecutor (#6279)

Woosuk Kwon committed Jul 9, 2024 at 19:30 UTC 5ed3505d827658fe4f71f30fecf93a66baabfe26
1 file changed +25 -6
vllm/executor/tpu_executor.py
+25 -6
@@ -81,8 +81,7 @@ class TPUExecutor(ExecutorBase):
81
82 def determine_num_available_blocks(self) -> Tuple[int, int]:
83 """Determine the number of available KV blocks by invoking the
84 - underlying worker.
85 - """
84 + underlying worker."""
85 return self.driver_worker.determine_num_available_blocks()
86
87 def execute_model(
@@ -93,16 +92,36 @@ class TPUExecutor(ExecutorBase):
92 return output
93
94 def add_lora(self, lora_request: LoRARequest) -> bool:
96 - raise NotImplementedError("LoRA is not implemented for TPU backend.")
95 + raise NotImplementedError(
96 + "LoRA is currently not supported by the TPU backend.")
97
98 def remove_lora(self, lora_id: int) -> bool:
99 - raise NotImplementedError("LoRA is not implemented for TPU backend.")
99 + raise NotImplementedError(
100 + "LoRA is currently not supported by the TPU backend.")
101
102 def pin_lora(self, lora_id: int) -> bool:
102 - raise NotImplementedError("LoRA is not implemented for TPU backend.")
103 + raise NotImplementedError(
104 + "LoRA is currently not supported by the TPU backend.")
105
106 def list_loras(self) -> Set[int]:
105 - raise NotImplementedError("LoRA is not implemented for TPU backend.")
107 + raise NotImplementedError(
108 + "LoRA is currently not supported by the TPU backend.")
109 +
110 + def add_prompt_adapter(self, prompt_adapter_request) -> bool:
111 + raise NotImplementedError(
112 + "Soft prompt is currently not supported by the TPU backend.")
113 +
114 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
115 + raise NotImplementedError(
116 + "Soft prompt is currently not supported by the TPU backend.")
117 +
118 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
119 + raise NotImplementedError(
120 + "Soft prompt is currently not supported by the TPU backend.")
121 +
122 + def list_prompt_adapters(self) -> Set[int]:
123 + raise NotImplementedError(
124 + "Soft prompt is currently not supported by the TPU backend.")
125
126 def check_health(self) -> None:
127 # TPUExecutor will always be healthy as long as it's running.