A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 08:30 UTC
[BugFix] Fix DBO failing with TypeError: 'NoneType' object is not iterable (#29698)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Nov 28, 2025 at 23:19 UTC
e23f665d835a40cfe2fbf21df76a4cbdd3d60b4c
3 files changed
+7
-7
tests/v1/distributed/test_dbo.py
-1
@@ -85,5 +85,4 @@ def test_dbo_dp_ep_gsm8k(all2all_backend: str, num_gpus_available):
85
assert accuracy >= MIN_ACCURACY, (
86
f"DBO+DP+EP accuracy too low ({all2all_backend}): "
87
f"{accuracy:.3f} < {MIN_ACCURACY:.3f} "
88
- f"(correct: {results['num_correct']}/{results['num_questions']})"
88
)
vllm/v1/attention/backends/utils.py
+1
-3
@@ -166,9 +166,7 @@ def _make_metadata_with_slice(
166
assert start_locs[first_req] <= first_tok < start_locs[first_req + 1], (
167
"Token slice start outside of first request"
168
)
169
- assert start_locs[last_req] <= last_tok < start_locs[last_req + 1], (
170
- "Token slice end outside of last request"
171
- )
169
+ # NOTE: last token can be outside of the last request if we have CG padding.
170
171
# If the "middle" request has tokens in both ubatches, we have to split it.
172
# If ubatch_slice is the first ubatch then we will be splitting the last
vllm/v1/worker/dp_utils.py
+6
-3
@@ -93,13 +93,16 @@ def _post_process_dp_padding(tensor: torch.Tensor, should_dp_pad: bool) -> torch
93
94
# This just pads the second ubatch slice out to the total number of tokens
95
# (num_tokens + padding) since we do `create_ubatch_slices` before applying DP padding.
96
-def _pad_out_ubatch_slice(ubatch_slices: UBatchSlices, num_total_tokens: int):
97
- padded_second_ubatch_slice = slice(
96
+def _pad_out_ubatch_slice(
97
+ ubatch_slices: UBatchSlices, num_total_tokens: int
98
+) -> UBatchSlices:
99
+ padded_second_token_slice = slice(
100
ubatch_slices[1].token_slice.start, num_total_tokens
101
)
102
ubatch_slices[1] = UBatchSlice(
101
- padded_second_ubatch_slice, padded_second_ubatch_slice
103
+ ubatch_slices[1].request_slice, padded_second_token_slice
104
)
105
+ return ubatch_slices
106
107
108
def _synchronize_dp_ranks(