A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix][TPU] Fix TPU sampler output (#5978)
Woosuk Kwon committed
Jun 28, 2024 at 18:14 UTC
54814fd85b5182fc140febfebbb2560420d2ed2a
1 file changed
+1
-1
vllm/worker/tpu_worker.py
+1
-1
@@ -215,7 +215,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
215
assert len(seq_group_metadata_list) > 0
216
output = self.model_runner.execute_model(seq_group_metadata_list,
217
self.tpu_cache)
218
- return [output]
218
+ return output
219
220
def cache_swap(
221
self,