A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix][TPU] Add prompt adapter methods to TPUExecutor (#6279)
Woosuk Kwon committed
Jul 9, 2024 at 19:30 UTC
5ed3505d827658fe4f71f30fecf93a66baabfe26
1 file changed
+25
-6
vllm/executor/tpu_executor.py
+25
-6
@@ -81,8 +81,7 @@ class TPUExecutor(ExecutorBase):
81
82
def determine_num_available_blocks(self) -> Tuple[int, int]:
83
"""Determine the number of available KV blocks by invoking the
84
- underlying worker.
85
- """
84
+ underlying worker."""
85
return self.driver_worker.determine_num_available_blocks()
86
87
def execute_model(
@@ -93,16 +92,36 @@ class TPUExecutor(ExecutorBase):
92
return output
93
94
def add_lora(self, lora_request: LoRARequest) -> bool:
96
- raise NotImplementedError("LoRA is not implemented for TPU backend.")
95
+ raise NotImplementedError(
96
+ "LoRA is currently not supported by the TPU backend.")
97
98
def remove_lora(self, lora_id: int) -> bool:
99
- raise NotImplementedError("LoRA is not implemented for TPU backend.")
99
+ raise NotImplementedError(
100
+ "LoRA is currently not supported by the TPU backend.")
101
102
def pin_lora(self, lora_id: int) -> bool:
102
- raise NotImplementedError("LoRA is not implemented for TPU backend.")
103
+ raise NotImplementedError(
104
+ "LoRA is currently not supported by the TPU backend.")
105
106
def list_loras(self) -> Set[int]:
105
- raise NotImplementedError("LoRA is not implemented for TPU backend.")
107
+ raise NotImplementedError(
108
+ "LoRA is currently not supported by the TPU backend.")
109
+
110
+ def add_prompt_adapter(self, prompt_adapter_request) -> bool:
111
+ raise NotImplementedError(
112
+ "Soft prompt is currently not supported by the TPU backend.")
113
+
114
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
115
+ raise NotImplementedError(
116
+ "Soft prompt is currently not supported by the TPU backend.")
117
+
118
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
119
+ raise NotImplementedError(
120
+ "Soft prompt is currently not supported by the TPU backend.")
121
+
122
+ def list_prompt_adapters(self) -> Set[int]:
123
+ raise NotImplementedError(
124
+ "Soft prompt is currently not supported by the TPU backend.")
125
126
def check_health(self) -> None:
127
# TPUExecutor will always be healthy as long as it's running.