A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 02:00 UTC
[BugFix] Fix async scheduling + chunked prefill + preemption (#28787)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Nov 16, 2025 at 14:46 UTC
80b6080ddcad0653daa6b776eb71a5a7029b70d8
3 files changed
+8
-9
tests/v1/e2e/test_async_scheduling.py
+4
-6
@@ -65,9 +65,8 @@ def test_without_spec_decoding(
65
(True, "mp", True, None, False),
66
(True, "uni", True, None, False),
67
(False, "mp", True, None, True),
68
- # Async scheduling + preemption + chunked prefill needs to be fixed (WIP)
69
- # (True, "mp", True, None, True),
70
- # (True, "uni", True, None, True),
68
+ (True, "mp", True, None, True),
69
+ (True, "uni", True, None, True),
70
]
71
72
run_tests(
@@ -103,9 +102,8 @@ def test_with_spec_decoding(monkeypatch: pytest.MonkeyPatch):
102
(False, "mp", True, spec_config_short, True),
103
(True, "uni", True, spec_config, False),
104
(True, "uni", True, spec_config_short, False),
106
- # Async scheduling + preemption + chunked prefill needs to be fixed (WIP)
107
- # (True, "mp", True, spec_config, True),
108
- # (True, "uni", True, spec_config_short, True),
105
+ (True, "mp", True, spec_config, True),
106
+ (True, "uni", True, spec_config_short, True),
107
]
108
109
run_tests(
vllm/v1/core/sched/scheduler.py
+1
-3
@@ -778,9 +778,7 @@ class Scheduler(SchedulerInterface):
778
assert not scheduled_in_prev_step
779
resumed_req_ids.add(req_id)
780
if not scheduled_in_prev_step:
781
- all_token_ids[req_id] = req.all_token_ids[
782
- : req.num_computed_tokens + num_tokens
783
- ]
781
+ all_token_ids[req_id] = req.all_token_ids.copy()
782
new_block_ids.append(
783
req_to_new_blocks[req_id].get_block_ids(allow_none=True)
784
)
vllm/v1/utils.py
+3
@@ -97,6 +97,9 @@ class ConstantList(Generic[T], Sequence):
97
def __repr__(self):
98
return f"ConstantList({self._x})"
99
100
+ def copy(self) -> list[T]:
101
+ return self._x.copy()
102
+
103
104
class CpuGpuBuffer:
105
"""Buffer to easily copy tensors between CPU and GPU."""