A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Sets `is_first_step_output` for TPUModelRunner (#9202)
Allen Wang committed
Oct 11, 2024 at 15:28 UTC
c6cf9295e1dad2aeffbce1d92682971df9f71ddf
1 file changed
+2
-1
vllm/worker/tpu_model_runner.py
+2
-1
@@ -541,7 +541,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
541
seq_group_metadata_list=ctx.seq_group_metadata_list,
542
scheduler_outputs=ctx.scheduler_outputs,
543
is_async=False,
544
- is_last_step=False)
544
+ is_last_step=False,
545
+ is_first_step_output=i == 0)
546
model_input.async_callback()
547
if use_async_out_proc:
548
return [sampler_outputs[-1]]