@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d5b4c5fe5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix][TPU] Add missing None to model input (#6245)

Woosuk Kwon committed Jul 9, 2024 at 00:21 UTC 5d5b4c5fe524c3b62453bba7ad4434a27c81317a
1 file changed +1 -1
vllm/worker/tpu_model_runner.py
+1 -1
@@ -154,7 +154,7 @@ class TPUModelRunner:
154 # Dummy run.
155 num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
156 self.model(token_ids, position_ids, kv_caches, attn_metadata,
157 - input_lens, t, p, num_samples)
157 + input_lens, None, t, p, num_samples)
158
159 def warmup_model(
160 self,