A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1] Remove scheduling constraint on partial requests (#12674)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Feb 4, 2025 at 02:43 UTC
18a88fcccce73261d51a18aba17368236ceb2f8b
4 files changed
+350
-123
tests/v1/core/test_scheduler.py
new
+214
@@ -0,0 +1,214 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from typing import List, Optional
3
+
4
+from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
5
+from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
6
+from vllm.sampling_params import SamplingParams
7
+from vllm.v1.core.scheduler import Scheduler
8
+from vllm.v1.outputs import ModelRunnerOutput
9
+from vllm.v1.request import Request, RequestStatus
10
+
11
+
12
+def create_scheduler(
13
+ model: str = "facebook/opt-125m",
14
+ max_num_seqs: int = 16,
15
+ max_num_batched_tokens: int = 8192,
16
+) -> Scheduler:
17
+ scheduler_config = SchedulerConfig(
18
+ max_num_seqs=max_num_seqs,
19
+ max_num_batched_tokens=max_num_batched_tokens,
20
+ max_model_len=max_num_batched_tokens,
21
+ )
22
+ model_config = ModelConfig(
23
+ model=model,
24
+ task="auto",
25
+ tokenizer=model,
26
+ tokenizer_mode="auto",
27
+ trust_remote_code=True,
28
+ dtype="float16",
29
+ seed=42,
30
+ )
31
+ cache_config = CacheConfig(
32
+ block_size=16,
33
+ gpu_memory_utilization=0.9,
34
+ swap_space=0,
35
+ cache_dtype="auto",
36
+ )
37
+ cache_config.num_gpu_blocks = 10000
38
+ return Scheduler(scheduler_config,
39
+ model_config,
40
+ cache_config,
41
+ lora_config=None)
42
+
43
+
44
+def create_requests(
45
+ num_requests: int,
46
+ num_tokens: int = 10,
47
+ mm_positions: Optional[List[PlaceholderRange]] = None,
48
+):
49
+ sampling_params = SamplingParams()
50
+ requests = []
51
+ for i in range(num_requests):
52
+ if mm_positions is not None:
53
+ mm_position = mm_positions[i]
54
+ mm_inputs = [MultiModalKwargs({})] * len(mm_position)
55
+ else:
56
+ mm_position = None
57
+ mm_inputs = None
58
+ request = Request(
59
+ request_id=f"{i}",
60
+ prompt=None,
61
+ prompt_token_ids=[i] * num_tokens,
62
+ sampling_params=sampling_params,
63
+ multi_modal_inputs=mm_inputs,
64
+ multi_modal_placeholders=mm_position,
65
+ multi_modal_hashes=None,
66
+ eos_token_id=None,
67
+ arrival_time=0,
68
+ )
69
+ requests.append(request)
70
+ return requests
71
+
72
+
73
+def test_add_requests():
74
+ scheduler = create_scheduler()
75
+ requests = create_requests(num_requests=10)
76
+
77
+ for i, request in enumerate(requests):
78
+ scheduler.add_request(request)
79
+ assert request.request_id in scheduler.requests
80
+ assert len(scheduler.waiting) == i + 1
81
+
82
+
83
+def test_finish_request():
84
+ scheduler = create_scheduler()
85
+ requests = create_requests(num_requests=10)
86
+ for request in requests:
87
+ scheduler.add_request(request)
88
+
89
+ for i, request in enumerate(requests):
90
+ scheduler.finish_requests(request.request_id,
91
+ RequestStatus.FINISHED_ABORTED)
92
+ assert request.request_id not in scheduler.requests
93
+ assert len(scheduler.waiting) == 9 - i
94
+
95
+
96
+def test_get_num_unfinished_requests():
97
+ scheduler = create_scheduler()
98
+ requests = create_requests(num_requests=10)
99
+ for request in requests:
100
+ scheduler.add_request(request)
101
+
102
+ for i, request in enumerate(requests):
103
+ scheduler.finish_requests(request.request_id,
104
+ RequestStatus.FINISHED_STOPPED)
105
+ assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
106
+
107
+
108
+def test_schedule():
109
+ scheduler = create_scheduler()
110
+ requests = create_requests(num_requests=10)
111
+ for request in requests:
112
+ scheduler.add_request(request)
113
+
114
+ # Test initial scheduling
115
+ output = scheduler.schedule()
116
+ assert len(output.scheduled_new_reqs) == len(requests)
117
+ assert len(output.scheduled_cached_reqs) == 0
118
+ assert len(output.finished_req_ids) == 0
119
+ # Verify all requests are scheduled.
120
+ for req_id, num_tokens in output.num_scheduled_tokens.items():
121
+ assert num_tokens == len(requests[int(req_id)].prompt_token_ids)
122
+
123
+ # Verify requests moved from waiting to running
124
+ assert len(scheduler.waiting) == 0
125
+ assert len(scheduler.running) == len(requests)
126
+ for i, request in enumerate(requests):
127
+ assert scheduler.running[i] == request
128
+
129
+
130
+def test_schedule_multimodal_requests():
131
+ scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
132
+ mm_positions = [[PlaceholderRange(offset=i, length=100)]
133
+ for i in range(10)]
134
+ requests = create_requests(
135
+ num_requests=10,
136
+ num_tokens=200,
137
+ mm_positions=mm_positions,
138
+ )
139
+ for request in requests:
140
+ scheduler.add_request(request)
141
+
142
+ output = scheduler.schedule()
143
+ assert len(output.scheduled_new_reqs) == len(requests)
144
+ assert len(output.scheduled_cached_reqs) == 0
145
+ assert len(output.finished_req_ids) == 0
146
+ for req_id, num_tokens in output.num_scheduled_tokens.items():
147
+ assert num_tokens == len(requests[int(req_id)].prompt_token_ids)
148
+ assert len(output.scheduled_encoder_inputs) == 10
149
+ for req_id, encoder_input in output.scheduled_encoder_inputs.items():
150
+ assert len(encoder_input) == 1
151
+
152
+
153
+def test_schedule_partial_requests():
154
+ """Test scheduling behavior with partial requests.
155
+
156
+ This test verifies that:
157
+ 1. The scheduler can handle multiple partial requests in a single step when
158
+ constrained by encoder budget.
159
+ 2. A request in RUNNING state may be unscheduled in subsequent steps if
160
+ there is insufficient encoder budget.
161
+ """
162
+ scheduler = create_scheduler(
163
+ model="llava-hf/llava-1.5-7b-hf",
164
+ max_num_batched_tokens=1024,
165
+ )
166
+ mm_positions = [[PlaceholderRange(offset=100, length=600)]
167
+ for _ in range(3)]
168
+ requests = create_requests(
169
+ num_requests=3,
170
+ num_tokens=800,
171
+ mm_positions=mm_positions,
172
+ )
173
+ for request in requests:
174
+ scheduler.add_request(request)
175
+
176
+ output = scheduler.schedule()
177
+ assert len(output.scheduled_new_reqs) == 3
178
+ assert len(output.scheduled_cached_reqs) == 0
179
+ assert len(output.finished_req_ids) == 0
180
+
181
+ assert scheduler.max_num_encoder_input_tokens == 1024
182
+ # The first request is scheduled fully.
183
+ assert output.num_scheduled_tokens[requests[0].request_id] == 800
184
+ # The second request is scheduled partially.
185
+ # The <img> tokens are not scheduled because of the encoder budget.
186
+ assert output.num_scheduled_tokens[requests[1].request_id] == 100
187
+ # The third request is also scheduled partially.
188
+ # The <img> tokens are not scheduled because of the encoder budget.
189
+ assert output.num_scheduled_tokens[requests[2].request_id] == 100
190
+ req_to_index = {
191
+ request.request_id: i
192
+ for i, request in enumerate(requests)
193
+ }
194
+ model_runner_output = ModelRunnerOutput(
195
+ req_ids=[request.request_id for request in requests],
196
+ req_id_to_index=req_to_index,
197
+ sampled_token_ids=[0] * len(requests),
198
+ logprob_token_ids_cpu=None,
199
+ logprobs_cpu=None,
200
+ )
201
+ scheduler.update_from_output(output, model_runner_output)
202
+
203
+ # Schedule the next step.
204
+ # Only the first and second requests are scheduled.
205
+ # The third request is in the RUNNING state but not scheduled in this step
206
+ # because of the encoder budget.
207
+ output = scheduler.schedule()
208
+ assert len(scheduler.running) == 3
209
+ assert len(output.scheduled_new_reqs) == 0
210
+ assert len(output.scheduled_cached_reqs) == 2
211
+ assert len(output.finished_req_ids) == 0
212
+ assert output.num_scheduled_tokens[requests[0].request_id] == 1
213
+ assert output.num_scheduled_tokens[requests[1].request_id] == 700
214
+ assert requests[2].request_id not in output.num_scheduled_tokens
vllm/v1/core/scheduler.py
+59
-70
@@ -67,10 +67,10 @@ class Scheduler:
67
# This is flushed at the end of each scheduling step.
68
self.finished_req_ids: Set[str] = set()
69
70
- # OPTIMIZATION: Cache the RunningRequestData objects to avoid creating
70
+ # OPTIMIZATION: Cache the CachedRequestData objects to avoid creating
71
# them at each scheduling step.
72
- # Request id -> RunningRequestData
73
- self.running_reqs_data: Dict[str, RunningRequestData] = {}
72
+ # Request id -> CachedRequestData
73
+ self._cached_reqs_data: Dict[str, CachedRequestData] = {}
74
75
# Encoder-related.
76
# Calculate encoder cache size if applicable
@@ -115,17 +115,8 @@ class Scheduler:
115
encoder_budget = self.max_num_encoder_input_tokens
116
117
# First, schedule the RUNNING requests.
118
- # NOTE(woosuk): At most 1 request in the RUNNING queue is allowed to be
119
- # in the "partial" state, where the request has some tokens computed
120
- # but not all. The constraint is due to the persistent batch in the
121
- # V1 model runner.
122
- # TODO(woosuk): Remove this constraint after refactoring model runner.
123
- has_partial_request = False
118
req_index = 0
125
- while req_index < len(self.running):
126
- # Only the last request in the RUNNING queue can be "partial".
127
- assert not has_partial_request
128
- assert token_budget > 0
119
+ while req_index < len(self.running) and token_budget > 0:
120
request = self.running[req_index]
121
num_new_tokens = request.num_tokens - request.num_computed_tokens
122
num_new_tokens = min(num_new_tokens, token_budget)
@@ -137,7 +128,14 @@ class Scheduler:
128
request.num_computed_tokens,
129
num_new_tokens,
130
encoder_budget))
140
- assert num_new_tokens > 0
131
+ if num_new_tokens == 0:
132
+ # The request cannot be scheduled because the encoder budget
133
+ # or the encoder cache is exhausted.
134
+ # NOTE(woosuk): Here, by doing `continue` instead of `break`,
135
+ # we do not strictly follow the FCFS scheduling policy and
136
+ # allow the lower-priority requests to be scheduled.
137
+ req_index += 1
138
+ continue
139
140
while True:
141
new_blocks = self.kv_cache_manager.allocate_slots(
@@ -172,8 +170,6 @@ class Scheduler:
170
num_scheduled_tokens[request.request_id] = num_new_tokens
171
token_budget -= num_new_tokens
172
req_index += 1
175
- has_partial_request = (request.num_computed_tokens + num_new_tokens
176
- < request.num_tokens)
173
174
# Encoder-related.
175
if encoder_inputs_to_schedule:
@@ -186,13 +182,9 @@ class Scheduler:
182
183
# Next, schedule the WAITING requests.
184
if not preempted_reqs:
189
- while self.waiting:
190
- if has_partial_request:
191
- break
185
+ while self.waiting and token_budget > 0:
186
if len(self.running) == self.max_num_running_reqs:
187
break
194
- if token_budget == 0:
195
- break
188
189
request = self.waiting[0]
190
# Get already-cached tokens.
@@ -249,8 +241,6 @@ class Scheduler:
241
token_budget -= num_new_tokens
242
request.status = RequestStatus.RUNNING
243
request.num_computed_tokens = num_computed_tokens
252
- has_partial_request = (num_computed_tokens + num_new_tokens
253
- < request.num_tokens)
244
245
# Encoder-related.
246
if encoder_inputs_to_schedule:
@@ -266,8 +256,11 @@ class Scheduler:
256
assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
257
assert token_budget >= 0
258
assert len(self.running) <= self.max_num_running_reqs
259
+ # Since some requests in the RUNNING queue may not be scheduled in
260
+ # this step, the total number of scheduled requests can be smaller than
261
+ # len(self.running).
262
assert (len(scheduled_new_reqs) + len(scheduled_resumed_reqs) +
270
- len(scheduled_running_reqs) == len(self.running))
263
+ len(scheduled_running_reqs) <= len(self.running))
264
265
# Get the longest common prefix among all requests in the running queue.
266
# This can be potentially used for cascade attention.
@@ -286,25 +279,28 @@ class Scheduler:
279
for req in scheduled_new_reqs
280
]
281
resumed_reqs_data = [
289
- ResumedRequestData.from_request(
290
- req, req_to_new_block_ids[req.request_id],
291
- req.num_computed_tokens) for req in scheduled_resumed_reqs
282
+ self._make_cached_request_data(
283
+ req,
284
+ req_to_new_block_ids[req.request_id],
285
+ req.num_computed_tokens,
286
+ resumed_from_preemption=True,
287
+ ) for req in scheduled_resumed_reqs
288
]
289
running_reqs_data = [
294
- self._make_running_request_data(
295
- req, req_to_new_block_ids[req.request_id],
296
- req.num_computed_tokens) for req in scheduled_running_reqs
290
+ self._make_cached_request_data(
291
+ req,
292
+ req_to_new_block_ids[req.request_id],
293
+ req.num_computed_tokens,
294
+ resumed_from_preemption=False,
295
+ ) for req in scheduled_running_reqs
296
]
298
- preempted_req_ids = {req.request_id for req in preempted_reqs}
297
scheduler_output = SchedulerOutput(
298
scheduled_new_reqs=new_reqs_data,
301
- scheduled_resumed_reqs=resumed_reqs_data,
302
- scheduled_running_reqs=running_reqs_data,
299
+ scheduled_cached_reqs=resumed_reqs_data + running_reqs_data,
300
num_scheduled_tokens=num_scheduled_tokens,
301
total_num_scheduled_tokens=total_num_scheduled_tokens,
302
scheduled_encoder_inputs=scheduled_encoder_inputs,
303
num_common_prefix_blocks=num_common_prefix_blocks,
307
- preempted_req_ids=preempted_req_ids,
304
# finished_req_ids is an existing state in the scheduler,
305
# instead of being newly scheduled in this step.
306
# It contains the request IDs that are finished in between
@@ -316,22 +312,26 @@ class Scheduler:
312
self.finished_req_ids = set()
313
return scheduler_output
314
319
- def _make_running_request_data(
315
+ def _make_cached_request_data(
316
self,
317
request: Request,
318
new_block_ids: List[int],
319
num_computed_tokens: int,
324
- ) -> "RunningRequestData":
325
- # OPTIMIZATION: Cache the RunningRequestData objects to avoid creating
320
+ resumed_from_preemption: bool,
321
+ ) -> "CachedRequestData":
322
+ # OPTIMIZATION: Cache the CachedRequestData objects to avoid creating
323
# them at each scheduling step.
327
- if request.request_id in self.running_reqs_data:
328
- req_data = self.running_reqs_data[request.request_id]
324
+ if request.request_id in self._cached_reqs_data:
325
+ req_data = self._cached_reqs_data[request.request_id]
326
+ req_data.resumed_from_preemption = resumed_from_preemption
327
req_data.new_block_ids = new_block_ids
328
req_data.num_computed_tokens = num_computed_tokens
329
else:
332
- req_data = RunningRequestData.from_request(request, new_block_ids,
333
- num_computed_tokens)
334
- self.running_reqs_data[request.request_id] = req_data
330
+ req_data = CachedRequestData.from_request(request,
331
+ resumed_from_preemption,
332
+ new_block_ids,
333
+ num_computed_tokens)
334
+ self._cached_reqs_data[request.request_id] = req_data
335
return req_data
336
337
def _try_schedule_encoder_inputs(
@@ -420,7 +420,13 @@ class Scheduler:
420
# expensive operations inside the loop.
421
for request in self.running:
422
req_id = request.request_id
423
- request.num_computed_tokens += num_scheduled_tokens[req_id]
423
+ num_tokens_scheduled = num_scheduled_tokens.get(req_id, 0)
424
+ if num_tokens_scheduled == 0:
425
+ # The request was not scheduled in this step.
426
+ new_running.append(request)
427
+ continue
428
+
429
+ request.num_computed_tokens += num_tokens_scheduled
430
# When the request's num_computed_tokens catches up its num_tokens,
431
# the request generates output tokens. Otherwise, we ignore the
432
# sampler output for the request.
@@ -529,7 +535,7 @@ class Scheduler:
535
assert request.is_finished()
536
self.kv_cache_manager.free(request)
537
self.encoder_cache_manager.free(request)
532
- self.running_reqs_data.pop(request.request_id, None)
538
+ self._cached_reqs_data.pop(request.request_id, None)
539
del self.requests[request.request_id]
540
self.finished_req_ids.add(request.request_id)
541
@@ -584,30 +590,13 @@ class NewRequestData:
590
591
592
@dataclass
587
-class ResumedRequestData:
588
-
589
- req_id: str
590
- block_ids: List[int]
591
- num_computed_tokens: int
592
-
593
- @classmethod
594
- def from_request(
595
- cls,
596
- request: Request,
597
- block_ids: List[int],
598
- num_computed_tokens: int,
599
- ) -> "ResumedRequestData":
600
- return cls(
601
- req_id=request.request_id,
602
- block_ids=block_ids,
603
- num_computed_tokens=num_computed_tokens,
604
- )
605
-
606
-
607
-@dataclass
608
-class RunningRequestData:
593
+class CachedRequestData:
594
595
req_id: str
596
+ # If resumed_from_preemption is False, new_block_ids will be appended to
597
+ # the request's block IDs. If True, new_block_ids will be used as the
598
+ # request's block IDs instead of appending to the existing block IDs.
599
+ resumed_from_preemption: bool
600
new_block_ids: List[int]
601
num_computed_tokens: int
602
@@ -615,11 +604,13 @@ class RunningRequestData:
604
def from_request(
605
cls,
606
request: Request,
607
+ resumed_from_preemption: bool,
608
new_block_ids: List[int],
609
num_computed_tokens: int,
620
- ) -> "RunningRequestData":
610
+ ) -> "CachedRequestData":
611
return cls(
612
req_id=request.request_id,
613
+ resumed_from_preemption=resumed_from_preemption,
614
new_block_ids=new_block_ids,
615
num_computed_tokens=num_computed_tokens,
616
)
@@ -629,14 +620,12 @@ class RunningRequestData:
620
class SchedulerOutput:
621
622
scheduled_new_reqs: List[NewRequestData]
632
- scheduled_resumed_reqs: List[ResumedRequestData]
633
- scheduled_running_reqs: List[RunningRequestData]
623
+ scheduled_cached_reqs: List[CachedRequestData]
624
625
num_scheduled_tokens: Dict[str, int]
626
total_num_scheduled_tokens: int
627
scheduled_encoder_inputs: Dict[str, List[int]]
628
num_common_prefix_blocks: int
629
640
- preempted_req_ids: Set[str]
630
finished_req_ids: Set[str]
631
free_encoder_input_ids: List[Tuple[str, int]]
vllm/v1/worker/block_table.py
+2
@@ -46,6 +46,8 @@ class BlockTable:
46
start: int,
47
block_ids: List[int],
48
) -> None:
49
+ if not block_ids:
50
+ return
51
num_blocks = len(block_ids)
52
self.block_table_np[row_idx, start:start + num_blocks] = block_ids
53
self.num_blocks_per_row[row_idx] = start + num_blocks
vllm/v1/worker/gpu_model_runner.py
+75
-53
@@ -205,12 +205,32 @@ class GPUModelRunner:
205
pin_memory=self.pin_memory)
206
self.seq_lens_np = self.seq_lens_cpu.numpy()
207
208
- def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
209
- # Remove stopped requests from the cached states.
210
- # Keep the states of the preempted requests.
208
+ def _update_states(self, scheduler_output: "SchedulerOutput") -> bool:
209
+ """Update the cached states and the persistent batch with the scheduler
210
+ output.
211
+
212
+ The updated states are used by the `_prepare_inputs` function to create
213
+ the input GPU tensors for the model.
214
+
215
+ Returns:
216
+ True if there is a new/resumed/paused/finished request in the batch.
217
+ If False, we can skip copying SamplingMetadata to the GPU.
218
+ """
219
+ # Remove finished requests from the cached states.
220
for req_id in scheduler_output.finished_req_ids:
221
self.requests.pop(req_id, None)
222
self.encoder_cache.pop(req_id, None)
223
+ # Remove the finished requests from the persistent batch.
224
+ # NOTE(woosuk): There could be an edge case where finished_req_ids and
225
+ # scheduled_req_ids overlap. This happens when a request is aborted and
226
+ # then resubmitted with the same ID. In this case, we treat them as two
227
+ # distinct requests - clearing the cached states for the first request
228
+ # and handling the second as a new request.
229
+ removed_req_indices: List[int] = []
230
+ for req_id in scheduler_output.finished_req_ids:
231
+ req_index = self.input_batch.remove_request(req_id)
232
+ if req_index is not None:
233
+ removed_req_indices.append(req_index)
234
235
# Free the cached encoder outputs.
236
for req_id, input_id in scheduler_output.free_encoder_input_ids:
@@ -220,36 +240,22 @@ class GPUModelRunner:
240
if not encoder_outputs:
241
self.encoder_cache.pop(req_id, None)
242
223
- # Remove the requests from the persistent batch.
224
- stopped_req_ids = set().union(
225
- scheduler_output.preempted_req_ids,
226
- scheduler_output.finished_req_ids,
227
- )
228
- removed_req_indices: List[int] = []
229
- for req_id in stopped_req_ids:
243
+ # Remove the unscheduled requests from the persistent batch.
244
+ # NOTE(woosuk): The unscheduled requests are either preempted requests
245
+ # or running requests that are not scheduled in this step. We remove
246
+ # them from the persistent batch but keep their cached states since
247
+ # they will be scheduled again sometime in the future.
248
+ scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys()
249
+ cached_req_ids = self.input_batch.req_id_to_index.keys()
250
+ unscheduled_req_ids = cached_req_ids - scheduled_req_ids
251
+ # NOTE(woosuk): The persistent batch optimization assumes that
252
+ # consecutive batches contain mostly the same requests. If batches
253
+ # have low request overlap (e.g., alternating between two distinct
254
+ # sets of requests), this optimization becomes very inefficient.
255
+ for req_id in unscheduled_req_ids:
256
req_index = self.input_batch.remove_request(req_id)
231
- if req_index is not None:
232
- removed_req_indices.append(req_index)
233
-
234
- # Update the states of the running requests.
235
- for req_data in scheduler_output.scheduled_running_reqs:
236
- req_id = req_data.req_id
237
- req_state = self.requests[req_id]
238
- req_index = self.input_batch.req_id_to_index[req_id]
239
-
240
- # Update the num_computed_tokens.
241
- req_state.num_computed_tokens = req_data.num_computed_tokens
242
- self.input_batch.num_computed_tokens_cpu[req_index] = (
243
- req_data.num_computed_tokens)
244
-
245
- # Update the block table.
246
- num_new_blocks = len(req_data.new_block_ids)
247
- if num_new_blocks == 0:
248
- continue
249
- start_index = len(req_state.block_ids)
250
- req_state.block_ids.extend(req_data.new_block_ids)
251
- self.input_batch.block_table.append_row(req_index, start_index,
252
- req_data.new_block_ids)
257
+ assert req_index is not None
258
+ removed_req_indices.append(req_index)
259
260
req_ids_to_add: List[str] = []
261
# Add new requests to the cached states.
@@ -305,14 +311,36 @@ class GPUModelRunner:
311
312
req_ids_to_add.append(req_id)
313
308
- # Update the cached states of the resumed requests.
309
- for res_req_data in scheduler_output.scheduled_resumed_reqs:
310
- req_id = res_req_data.req_id
314
+ # Update the states of the running/resumed requests.
315
+ for req_data in scheduler_output.scheduled_cached_reqs:
316
+ req_id = req_data.req_id
317
req_state = self.requests[req_id]
318
313
- req_state.block_ids = res_req_data.block_ids
314
- req_state.num_computed_tokens = res_req_data.num_computed_tokens
315
- req_ids_to_add.append(req_id)
319
+ # Update the cached states.
320
+ req_state.num_computed_tokens = req_data.num_computed_tokens
321
+ if not req_data.resumed_from_preemption:
322
+ # Append the new blocks to the existing block IDs.
323
+ req_state.block_ids.extend(req_data.new_block_ids)
324
+ else:
325
+ # The request is resumed from preemption.
326
+ # Replace the existing block IDs with the new ones.
327
+ req_state.block_ids = req_data.new_block_ids
328
+
329
+ req_index = self.input_batch.req_id_to_index.get(req_id)
330
+ if req_index is None:
331
+ # The request is not in the persistent batch.
332
+ # The request was either preempted and resumed later, or was not
333
+ # scheduled in the previous step and needs to be added again.
334
+ req_ids_to_add.append(req_id)
335
+ continue
336
+
337
+ # Update the persistent batch.
338
+ self.input_batch.num_computed_tokens_cpu[req_index] = (
339
+ req_data.num_computed_tokens)
340
+ start_index = len(req_state.block_ids) - len(
341
+ req_data.new_block_ids)
342
+ self.input_batch.block_table.append_row(req_index, start_index,
343
+ req_data.new_block_ids)
344
345
# Add the new or resumed requests to the persistent batch.
346
# The smaller empty indices are filled first.
@@ -330,6 +358,7 @@ class GPUModelRunner:
358
# Condense the batched states if there are empty indices.
359
if removed_req_indices:
360
self.input_batch.condense(removed_req_indices)
361
+ return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
362
363
def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
364
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
@@ -536,10 +565,10 @@ class GPUModelRunner:
565
prefix_kv_lens=prefix_kv_lens,
566
suffix_kv_lens=suffix_kv_lens,
567
)
539
- # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
540
- # request in the batch. While we should not sample any token from this
541
- # partial request, we do so for simplicity. We will ignore the sampled
542
- # token from the partial request.
568
+ # NOTE(woosuk): Due to chunked prefills, the batch may contain partial
569
+ # requests. While we should not sample any token from these partial
570
+ # requests, we do so for simplicity. We will ignore the sampled
571
+ # tokens from the partial requests.
572
# TODO: Support prompt logprobs.
573
logits_indices = query_start_loc[1:] - 1
574
return attn_metadata, logits_indices
@@ -601,22 +630,15 @@ class GPUModelRunner:
630
631
def _prepare_sampling(
632
self,
604
- scheduler_output: "SchedulerOutput",
633
+ batch_changed: bool,
634
) -> SamplingMetadata:
606
- skip_copy = True
607
- if (scheduler_output.finished_req_ids
608
- or scheduler_output.preempted_req_ids):
609
- skip_copy = False
610
- if (scheduler_output.scheduled_new_reqs
611
- or scheduler_output.scheduled_resumed_reqs):
612
- skip_copy = False
635
# Create the sampling metadata.
636
req_id_output_token_ids: Dict[str, List[int]] = \
637
{req_id: req.output_token_ids \
638
for req_id, req in self.requests.items()}
639
640
sampling_metadata = self.input_batch.make_sampling_metadata(
619
- req_id_output_token_ids, skip_copy)
641
+ req_id_output_token_ids, skip_copy=not batch_changed)
642
return sampling_metadata
643
644
def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
@@ -715,7 +737,7 @@ class GPUModelRunner:
737
self,
738
scheduler_output: "SchedulerOutput",
739
) -> ModelRunnerOutput:
718
- self._update_states(scheduler_output)
740
+ batch_changed = self._update_states(scheduler_output)
741
742
if self.is_multimodal_model:
743
# Run the multimodal encoder if any.
@@ -778,7 +800,7 @@ class GPUModelRunner:
800
logits = self.model.compute_logits(hidden_states, None)
801
802
# Sample the next token and get logprobs if needed.
781
- sampling_metadata = self._prepare_sampling(scheduler_output)
803
+ sampling_metadata = self._prepare_sampling(batch_changed)
804
sampler_output = self.model.sample(
805
logits=logits,
806
sampling_metadata=sampling_metadata,