@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6cf9295e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Sets `is_first_step_output` for TPUModelRunner (#9202)

Allen Wang committed Oct 11, 2024 at 15:28 UTC c6cf9295e1dad2aeffbce1d92682971df9f71ddf
1 file changed +2 -1
vllm/worker/tpu_model_runner.py
+2 -1
@@ -541,7 +541,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
541 seq_group_metadata_list=ctx.seq_group_metadata_list,
542 scheduler_outputs=ctx.scheduler_outputs,
543 is_async=False,
544 - is_last_step=False)
544 + is_last_step=False,
545 + is_first_step_output=i == 0)
546 model_input.async_callback()
547 if use_async_out_proc:
548 return [sampler_outputs[-1]]