A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] [DP/EP] Fix slow execution when BS <= DP (#25407)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Chris Bamford <chrisbam4d@gmail.com>
Matthew Bonanni committed
Sep 22, 2025 at 20:26 UTC
ac0048c0aea977e520aa19f3b9155fc19d696df7
2 files changed
+5
-4
vllm/v1/worker/gpu_model_runner.py
+4
-3
@@ -55,7 +55,7 @@ from vllm.sampling_params import SamplingType
55
from vllm.sequence import IntermediateTensors, PoolerOutput
56
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
57
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
58
- GiB_bytes, check_use_alibi, get_dtype_size,
58
+ GiB_bytes, cdiv, check_use_alibi, get_dtype_size,
59
is_pin_memory_available,
60
length_from_prompt_token_ids_or_embeds, round_up,
61
supports_dynamo)
@@ -2913,12 +2913,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2913
# Note: Overriding max_query_len to be the prefill tokens
2914
max_query_len = num_prefill_tokens
2915
elif uniform_decode:
2916
- num_reqs = num_tokens // max_query_len
2916
+ assert not create_mixed_batch
2917
+ num_reqs = cdiv(num_tokens, max_query_len)
2918
assert num_reqs <= max_num_reqs, \
2919
"Do not capture num_reqs > max_num_reqs for uniform batch"
2920
num_scheduled_tokens_list = [max_query_len] * num_reqs
2921
if num_tokens % max_query_len != 0:
2921
- num_scheduled_tokens_list[-1] += num_tokens % max_query_len
2922
+ num_scheduled_tokens_list[-1] = num_tokens % max_query_len
2923
else:
2924
num_reqs = min(num_tokens, max_num_reqs)
2925
min_tokens_per_req = num_tokens // num_reqs
vllm/v1/worker/gpu_worker.py
+1
-1
@@ -487,7 +487,7 @@ class Worker(WorkerBase):
487
sort_by="self_cuda_time_total"))
488
489
def execute_dummy_batch(self) -> None:
490
- self.model_runner._dummy_run(1)
490
+ self.model_runner._dummy_run(1, uniform_decode=True)
491
492
def add_lora(self, lora_request: LoRARequest) -> bool:
493
return self.model_runner.add_lora(lora_request)