@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54814fd85b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix][TPU] Fix TPU sampler output (#5978)

Woosuk Kwon committed Jun 28, 2024 at 18:14 UTC 54814fd85b5182fc140febfebbb2560420d2ed2a
1 file changed +1 -1
vllm/worker/tpu_worker.py
+1 -1
@@ -215,7 +215,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
215 assert len(seq_group_metadata_list) > 0
216 output = self.model_runner.execute_model(seq_group_metadata_list,
217 self.tpu_cache)
218 - return [output]
218 + return output
219
220 def cache_swap(
221 self,