@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6dd94dbe94
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[perf] fix perf regression from #12253 (#12380)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 24, 2025 at 11:34 UTC 6dd94dbe94c1820a1e224cba65efcf0befa97995
1 file changed +4 -1
vllm/worker/model_runner.py
+4 -1
@@ -455,7 +455,6 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
455 self.enable_prompt_adapter = (self.runner.prompt_adapter_config
456 is not None)
457 self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
458 - self.decode_only = True
458
459 # Attention metadata inputs.
460 if self.attn_backend is not None:
@@ -477,6 +476,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
476 finished_requests_ids: Optional[List[str]] = None) -> None:
477 self.finished_requests_ids = finished_requests_ids
478
479 + # if the current batch is decode-only.
480 + # will be set to False if there is any non-decode request.
481 + self.decode_only = True
482 +
483 # Intermediate data (data in CPU before going to GPU) for
484 # the current sequence group.
485 self.inter_data_list: List[