A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[perf] fix perf regression from #12253 (#12380)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 24, 2025 at 11:34 UTC
6dd94dbe94c1820a1e224cba65efcf0befa97995
1 file changed
+4
-1
vllm/worker/model_runner.py
+4
-1
@@ -455,7 +455,6 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
455
self.enable_prompt_adapter = (self.runner.prompt_adapter_config
456
is not None)
457
self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
458
- self.decode_only = True
458
459
# Attention metadata inputs.
460
if self.attn_backend is not None:
@@ -477,6 +476,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
476
finished_requests_ids: Optional[List[str]] = None) -> None:
477
self.finished_requests_ids = finished_requests_ids
478
479
+ # if the current batch is decode-only.
480
+ # will be set to False if there is any non-decode request.
481
+ self.decode_only = True
482
+
483
# Intermediate data (data in CPU before going to GPU) for
484
# the current sequence group.
485
self.inter_data_list: List[