A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[BugFix] scheduler: Fix ordering preserving of skipped requests (#32173)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Or Ozeri committed
Jan 12, 2026 at 20:39 UTC
2be765b68ace5efe4cba4d03495f543c85784688
2 files changed
+31
-15
tests/v1/core/test_scheduler.py
+25
@@ -3349,3 +3349,28 @@ def test_ec_connector_allocate_encoder_tokens_with_external_load(use_kv_connecto
3349
# ==============================================================================
3350
# EPD (Encoder-Prefill-Decode) Encoder-cache-specific tests end
3351
# ==============================================================================
3352
+
3353
+
3354
+def test_prepend_skipped_requests_order():
3355
+ scheduler = create_scheduler(max_num_seqs=1, use_kv_connector=True)
3356
+ requests = create_requests(num_requests=4)
3357
+ for request in requests:
3358
+ scheduler.add_request(request)
3359
+
3360
+ # 4 requests waiting, capture their order
3361
+ expected_waiting_reqs = list(scheduler.waiting)
3362
+
3363
+ # simulate first 2 waiting requests are waiting for remote KVs
3364
+ for req in expected_waiting_reqs[:2]:
3365
+ req.status = RequestStatus.WAITING_FOR_REMOTE_KVS
3366
+
3367
+ # schedule step
3368
+ # expect the first 2 waiting to be skipped, the third running,
3369
+ # and the fourth waiting
3370
+ scheduler.schedule()
3371
+
3372
+ # pop the third request which is expected to be running
3373
+ expected_waiting_reqs.pop(2)
3374
+
3375
+ # verify waiting order is preserved
3376
+ assert list(scheduler.waiting) == expected_waiting_reqs
vllm/v1/core/sched/request_queue.py
+6
-15
@@ -71,11 +71,6 @@ class RequestQueue(ABC):
71
"""Iterate over the queue according to the policy."""
72
pass
73
74
- @abstractmethod
75
- def __reversed__(self) -> Iterator[Request]:
76
- """Iterate over the queue in reverse order."""
77
- pass
78
-
74
75
class FCFSRequestQueue(deque[Request], RequestQueue):
76
"""A first-come-first-served queue that supports deque operations."""
@@ -100,8 +95,12 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
95
96
def prepend_requests(self, requests: RequestQueue) -> None:
97
"""Prepend all requests from another queue to the front of this
103
- queue."""
104
- self.extendleft(reversed(requests))
98
+ queue.
99
+
100
+ Note: The requests will be prepended in reverse order of their
101
+ appearance in the `requests` queue.
102
+ """
103
+ self.extendleft(requests)
104
105
def remove_request(self, request: Request) -> None:
106
"""Remove a specific request from the queue."""
@@ -128,10 +127,6 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
127
"""Iterate over the queue according to FCFS policy."""
128
return super().__iter__()
129
131
- def __reversed__(self) -> Iterator[Request]:
132
- """Iterate over the queue in reverse order."""
133
- return super().__reversed__()
134
-
130
131
class PriorityRequestQueue(RequestQueue):
132
"""
@@ -202,10 +197,6 @@ class PriorityRequestQueue(RequestQueue):
197
while heap_copy:
198
yield heapq.heappop(heap_copy)
199
205
- def __reversed__(self) -> Iterator[Request]:
206
- """Iterate over the queue in reverse priority order."""
207
- return reversed(list(self))
208
-
200
201
def create_request_queue(policy: SchedulingPolicy) -> RequestQueue:
202
"""Create request queue based on scheduling policy."""