A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix][TPU] Add missing None to model input (#6245)
Woosuk Kwon committed
Jul 9, 2024 at 00:21 UTC
5d5b4c5fe524c3b62453bba7ad4434a27c81317a
1 file changed
+1
-1
vllm/worker/tpu_model_runner.py
+1
-1
@@ -154,7 +154,7 @@ class TPUModelRunner:
154
# Dummy run.
155
num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
156
self.model(token_ids, position_ids, kv_caches, attn_metadata,
157
- input_lens, t, p, num_samples)
157
+ input_lens, None, t, p, num_samples)
158
159
def warmup_model(
160
self,