A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 04:45 UTC
[Bugfix] fix --scheduling-policy=priority & n>1 crashes engine (#29764)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Chauncey committed
Dec 3, 2025 at 06:42 UTC
0a9caca9f5e130acbf39d5acd0b79fb492d6c4a3
3 files changed
+34
-15
tests/v1/core/test_priority_scheduler_random.py
+11
-1
@@ -219,7 +219,17 @@ def test_priority_scheduling_blast(
219
vllm_config=scheduler.vllm_config,
220
)
221
scheduler.add_request(req)
222
-
222
+ num_initial_requests = 2
223
+ for _ in range(num_initial_requests):
224
+ req = _create_random_request(
225
+ max_tokens_range=(1, max_output_tokens),
226
+ num_tokens_range=(1, max_input_tokens),
227
+ arrival_time_range=(0, 0),
228
+ priority_range=(4, 4),
229
+ num_mm_item_range=(0, 2),
230
+ vllm_config=scheduler.vllm_config,
231
+ )
232
+ scheduler.add_request(req)
233
for _ in range(20000):
234
if len(scheduler.waiting) == 0:
235
num_new_requests = random.randint(0, 2)
vllm/v1/core/sched/request_queue.py
+10
-14
@@ -137,31 +137,30 @@ class PriorityRequestQueue(RequestQueue):
137
"""
138
A priority queue that supports heap operations.
139
140
- Requests with a smaller value of `priority` are processed first.
140
+ Respects the ordering defined in the Request class, where
141
+ requests with a smaller value of `priority` are processed first.
142
If multiple requests have the same priority, the one with the earlier
143
`arrival_time` is processed first.
144
"""
145
146
def __init__(self) -> None:
146
- self._heap: list[tuple[int, float, Request]] = []
147
+ self._heap: list[Request] = []
148
149
def add_request(self, request: Request) -> None:
150
"""Add a request to the queue according to priority policy."""
150
- heapq.heappush(self._heap, (request.priority, request.arrival_time, request))
151
+ heapq.heappush(self._heap, request)
152
153
def pop_request(self) -> Request:
154
"""Pop a request from the queue according to priority policy."""
155
if not self._heap:
156
raise IndexError("pop from empty heap")
156
- _, _, request = heapq.heappop(self._heap)
157
- return request
157
+ return heapq.heappop(self._heap)
158
159
def peek_request(self) -> Request:
160
"""Peek at the next request in the queue without removing it."""
161
if not self._heap:
162
raise IndexError("peek from empty heap")
163
- _, _, request = self._heap[0]
164
- return request
163
+ return self._heap[0]
164
165
def prepend_request(self, request: Request) -> None:
166
"""Add a request to the queue according to priority policy.
@@ -180,15 +179,13 @@ class PriorityRequestQueue(RequestQueue):
179
180
def remove_request(self, request: Request) -> None:
181
"""Remove a specific request from the queue."""
183
- self._heap = [(p, t, r) for p, t, r in self._heap if r != request]
182
+ self._heap.remove(request)
183
heapq.heapify(self._heap)
184
185
def remove_requests(self, requests: Iterable[Request]) -> None:
186
"""Remove multiple specific requests from the queue."""
188
- requests_to_remove = set(requests)
189
- self._heap = [
190
- (p, t, r) for p, t, r in self._heap if r not in requests_to_remove
191
- ]
187
+ requests_to_remove = requests if isinstance(requests, set) else set(requests)
188
+ self._heap = [r for r in self._heap if r not in requests_to_remove]
189
heapq.heapify(self._heap)
190
191
def __bool__(self) -> bool:
@@ -203,8 +200,7 @@ class PriorityRequestQueue(RequestQueue):
200
"""Iterate over the queue according to priority policy."""
201
heap_copy = self._heap[:]
202
while heap_copy:
206
- _, _, request = heapq.heappop(heap_copy)
207
- yield request
203
+ yield heapq.heappop(heap_copy)
204
205
def __reversed__(self) -> Iterator[Request]:
206
"""Iterate over the queue in reverse priority order."""
vllm/v1/request.py
+13
@@ -227,6 +227,19 @@ class Request:
227
events, self.events = self.events, []
228
return events
229
230
+ def __lt__(self, other: "Request") -> bool:
231
+ """
232
+ Compare two requests based on priority, arrival time, and request ID.
233
+ Used in priority scheduling.
234
+ """
235
+ if self.priority != other.priority:
236
+ return self.priority < other.priority
237
+ if self.arrival_time != other.arrival_time:
238
+ return self.arrival_time < other.arrival_time
239
+ if self.request_id != other.request_id:
240
+ return self.request_id < other.request_id
241
+ return id(self) < id(other)
242
+
243
244
class RequestStatus(enum.IntEnum):
245
"""Status of a request."""