@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18a88fcccc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1] Remove scheduling constraint on partial requests (#12674)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Feb 4, 2025 at 02:43 UTC 18a88fcccce73261d51a18aba17368236ceb2f8b
4 files changed +350 -123
tests/v1/core/test_scheduler.py new
+214
@@ -0,0 +1,214 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +from typing import List, Optional
3 +
4 +from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
5 +from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
6 +from vllm.sampling_params import SamplingParams
7 +from vllm.v1.core.scheduler import Scheduler
8 +from vllm.v1.outputs import ModelRunnerOutput
9 +from vllm.v1.request import Request, RequestStatus
10 +
11 +
12 +def create_scheduler(
13 + model: str = "facebook/opt-125m",
14 + max_num_seqs: int = 16,
15 + max_num_batched_tokens: int = 8192,
16 +) -> Scheduler:
17 + scheduler_config = SchedulerConfig(
18 + max_num_seqs=max_num_seqs,
19 + max_num_batched_tokens=max_num_batched_tokens,
20 + max_model_len=max_num_batched_tokens,
21 + )
22 + model_config = ModelConfig(
23 + model=model,
24 + task="auto",
25 + tokenizer=model,
26 + tokenizer_mode="auto",
27 + trust_remote_code=True,
28 + dtype="float16",
29 + seed=42,
30 + )
31 + cache_config = CacheConfig(
32 + block_size=16,
33 + gpu_memory_utilization=0.9,
34 + swap_space=0,
35 + cache_dtype="auto",
36 + )
37 + cache_config.num_gpu_blocks = 10000
38 + return Scheduler(scheduler_config,
39 + model_config,
40 + cache_config,
41 + lora_config=None)
42 +
43 +
44 +def create_requests(
45 + num_requests: int,
46 + num_tokens: int = 10,
47 + mm_positions: Optional[List[PlaceholderRange]] = None,
48 +):
49 + sampling_params = SamplingParams()
50 + requests = []
51 + for i in range(num_requests):
52 + if mm_positions is not None:
53 + mm_position = mm_positions[i]
54 + mm_inputs = [MultiModalKwargs({})] * len(mm_position)
55 + else:
56 + mm_position = None
57 + mm_inputs = None
58 + request = Request(
59 + request_id=f"{i}",
60 + prompt=None,
61 + prompt_token_ids=[i] * num_tokens,
62 + sampling_params=sampling_params,
63 + multi_modal_inputs=mm_inputs,
64 + multi_modal_placeholders=mm_position,
65 + multi_modal_hashes=None,
66 + eos_token_id=None,
67 + arrival_time=0,
68 + )
69 + requests.append(request)
70 + return requests
71 +
72 +
73 +def test_add_requests():
74 + scheduler = create_scheduler()
75 + requests = create_requests(num_requests=10)
76 +
77 + for i, request in enumerate(requests):
78 + scheduler.add_request(request)
79 + assert request.request_id in scheduler.requests
80 + assert len(scheduler.waiting) == i + 1
81 +
82 +
83 +def test_finish_request():
84 + scheduler = create_scheduler()
85 + requests = create_requests(num_requests=10)
86 + for request in requests:
87 + scheduler.add_request(request)
88 +
89 + for i, request in enumerate(requests):
90 + scheduler.finish_requests(request.request_id,
91 + RequestStatus.FINISHED_ABORTED)
92 + assert request.request_id not in scheduler.requests
93 + assert len(scheduler.waiting) == 9 - i
94 +
95 +
96 +def test_get_num_unfinished_requests():
97 + scheduler = create_scheduler()
98 + requests = create_requests(num_requests=10)
99 + for request in requests:
100 + scheduler.add_request(request)
101 +
102 + for i, request in enumerate(requests):
103 + scheduler.finish_requests(request.request_id,
104 + RequestStatus.FINISHED_STOPPED)
105 + assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
106 +
107 +
108 +def test_schedule():
109 + scheduler = create_scheduler()
110 + requests = create_requests(num_requests=10)
111 + for request in requests:
112 + scheduler.add_request(request)
113 +
114 + # Test initial scheduling
115 + output = scheduler.schedule()
116 + assert len(output.scheduled_new_reqs) == len(requests)
117 + assert len(output.scheduled_cached_reqs) == 0
118 + assert len(output.finished_req_ids) == 0
119 + # Verify all requests are scheduled.
120 + for req_id, num_tokens in output.num_scheduled_tokens.items():
121 + assert num_tokens == len(requests[int(req_id)].prompt_token_ids)
122 +
123 + # Verify requests moved from waiting to running
124 + assert len(scheduler.waiting) == 0
125 + assert len(scheduler.running) == len(requests)
126 + for i, request in enumerate(requests):
127 + assert scheduler.running[i] == request
128 +
129 +
130 +def test_schedule_multimodal_requests():
131 + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
132 + mm_positions = [[PlaceholderRange(offset=i, length=100)]
133 + for i in range(10)]
134 + requests = create_requests(
135 + num_requests=10,
136 + num_tokens=200,
137 + mm_positions=mm_positions,
138 + )
139 + for request in requests:
140 + scheduler.add_request(request)
141 +
142 + output = scheduler.schedule()
143 + assert len(output.scheduled_new_reqs) == len(requests)
144 + assert len(output.scheduled_cached_reqs) == 0
145 + assert len(output.finished_req_ids) == 0
146 + for req_id, num_tokens in output.num_scheduled_tokens.items():
147 + assert num_tokens == len(requests[int(req_id)].prompt_token_ids)
148 + assert len(output.scheduled_encoder_inputs) == 10
149 + for req_id, encoder_input in output.scheduled_encoder_inputs.items():
150 + assert len(encoder_input) == 1
151 +
152 +
153 +def test_schedule_partial_requests():
154 + """Test scheduling behavior with partial requests.
155 +
156 + This test verifies that:
157 + 1. The scheduler can handle multiple partial requests in a single step when
158 + constrained by encoder budget.
159 + 2. A request in RUNNING state may be unscheduled in subsequent steps if
160 + there is insufficient encoder budget.
161 + """
162 + scheduler = create_scheduler(
163 + model="llava-hf/llava-1.5-7b-hf",
164 + max_num_batched_tokens=1024,
165 + )
166 + mm_positions = [[PlaceholderRange(offset=100, length=600)]
167 + for _ in range(3)]
168 + requests = create_requests(
169 + num_requests=3,
170 + num_tokens=800,
171 + mm_positions=mm_positions,
172 + )
173 + for request in requests:
174 + scheduler.add_request(request)
175 +
176 + output = scheduler.schedule()
177 + assert len(output.scheduled_new_reqs) == 3
178 + assert len(output.scheduled_cached_reqs) == 0
179 + assert len(output.finished_req_ids) == 0
180 +
181 + assert scheduler.max_num_encoder_input_tokens == 1024
182 + # The first request is scheduled fully.
183 + assert output.num_scheduled_tokens[requests[0].request_id] == 800
184 + # The second request is scheduled partially.
185 + # The <img> tokens are not scheduled because of the encoder budget.
186 + assert output.num_scheduled_tokens[requests[1].request_id] == 100
187 + # The third request is also scheduled partially.
188 + # The <img> tokens are not scheduled because of the encoder budget.
189 + assert output.num_scheduled_tokens[requests[2].request_id] == 100
190 + req_to_index = {
191 + request.request_id: i
192 + for i, request in enumerate(requests)
193 + }
194 + model_runner_output = ModelRunnerOutput(
195 + req_ids=[request.request_id for request in requests],
196 + req_id_to_index=req_to_index,
197 + sampled_token_ids=[0] * len(requests),
198 + logprob_token_ids_cpu=None,
199 + logprobs_cpu=None,
200 + )
201 + scheduler.update_from_output(output, model_runner_output)
202 +
203 + # Schedule the next step.
204 + # Only the first and second requests are scheduled.
205 + # The third request is in the RUNNING state but not scheduled in this step
206 + # because of the encoder budget.
207 + output = scheduler.schedule()
208 + assert len(scheduler.running) == 3
209 + assert len(output.scheduled_new_reqs) == 0
210 + assert len(output.scheduled_cached_reqs) == 2
211 + assert len(output.finished_req_ids) == 0
212 + assert output.num_scheduled_tokens[requests[0].request_id] == 1
213 + assert output.num_scheduled_tokens[requests[1].request_id] == 700
214 + assert requests[2].request_id not in output.num_scheduled_tokens
vllm/v1/core/scheduler.py
+59 -70
@@ -67,10 +67,10 @@ class Scheduler:
67 # This is flushed at the end of each scheduling step.
68 self.finished_req_ids: Set[str] = set()
69
70 - # OPTIMIZATION: Cache the RunningRequestData objects to avoid creating
70 + # OPTIMIZATION: Cache the CachedRequestData objects to avoid creating
71 # them at each scheduling step.
72 - # Request id -> RunningRequestData
73 - self.running_reqs_data: Dict[str, RunningRequestData] = {}
72 + # Request id -> CachedRequestData
73 + self._cached_reqs_data: Dict[str, CachedRequestData] = {}
74
75 # Encoder-related.
76 # Calculate encoder cache size if applicable
@@ -115,17 +115,8 @@ class Scheduler:
115 encoder_budget = self.max_num_encoder_input_tokens
116
117 # First, schedule the RUNNING requests.
118 - # NOTE(woosuk): At most 1 request in the RUNNING queue is allowed to be
119 - # in the "partial" state, where the request has some tokens computed
120 - # but not all. The constraint is due to the persistent batch in the
121 - # V1 model runner.
122 - # TODO(woosuk): Remove this constraint after refactoring model runner.
123 - has_partial_request = False
118 req_index = 0
125 - while req_index < len(self.running):
126 - # Only the last request in the RUNNING queue can be "partial".
127 - assert not has_partial_request
128 - assert token_budget > 0
119 + while req_index < len(self.running) and token_budget > 0:
120 request = self.running[req_index]
121 num_new_tokens = request.num_tokens - request.num_computed_tokens
122 num_new_tokens = min(num_new_tokens, token_budget)
@@ -137,7 +128,14 @@ class Scheduler:
128 request.num_computed_tokens,
129 num_new_tokens,
130 encoder_budget))
140 - assert num_new_tokens > 0
131 + if num_new_tokens == 0:
132 + # The request cannot be scheduled because the encoder budget
133 + # or the encoder cache is exhausted.
134 + # NOTE(woosuk): Here, by doing `continue` instead of `break`,
135 + # we do not strictly follow the FCFS scheduling policy and
136 + # allow the lower-priority requests to be scheduled.
137 + req_index += 1
138 + continue
139
140 while True:
141 new_blocks = self.kv_cache_manager.allocate_slots(
@@ -172,8 +170,6 @@ class Scheduler:
170 num_scheduled_tokens[request.request_id] = num_new_tokens
171 token_budget -= num_new_tokens
172 req_index += 1
175 - has_partial_request = (request.num_computed_tokens + num_new_tokens
176 - < request.num_tokens)
173
174 # Encoder-related.
175 if encoder_inputs_to_schedule:
@@ -186,13 +182,9 @@ class Scheduler:
182
183 # Next, schedule the WAITING requests.
184 if not preempted_reqs:
189 - while self.waiting:
190 - if has_partial_request:
191 - break
185 + while self.waiting and token_budget > 0:
186 if len(self.running) == self.max_num_running_reqs:
187 break
194 - if token_budget == 0:
195 - break
188
189 request = self.waiting[0]
190 # Get already-cached tokens.
@@ -249,8 +241,6 @@ class Scheduler:
241 token_budget -= num_new_tokens
242 request.status = RequestStatus.RUNNING
243 request.num_computed_tokens = num_computed_tokens
252 - has_partial_request = (num_computed_tokens + num_new_tokens
253 - < request.num_tokens)
244
245 # Encoder-related.
246 if encoder_inputs_to_schedule:
@@ -266,8 +256,11 @@ class Scheduler:
256 assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
257 assert token_budget >= 0
258 assert len(self.running) <= self.max_num_running_reqs
259 + # Since some requests in the RUNNING queue may not be scheduled in
260 + # this step, the total number of scheduled requests can be smaller than
261 + # len(self.running).
262 assert (len(scheduled_new_reqs) + len(scheduled_resumed_reqs) +
270 - len(scheduled_running_reqs) == len(self.running))
263 + len(scheduled_running_reqs) <= len(self.running))
264
265 # Get the longest common prefix among all requests in the running queue.
266 # This can be potentially used for cascade attention.
@@ -286,25 +279,28 @@ class Scheduler:
279 for req in scheduled_new_reqs
280 ]
281 resumed_reqs_data = [
289 - ResumedRequestData.from_request(
290 - req, req_to_new_block_ids[req.request_id],
291 - req.num_computed_tokens) for req in scheduled_resumed_reqs
282 + self._make_cached_request_data(
283 + req,
284 + req_to_new_block_ids[req.request_id],
285 + req.num_computed_tokens,
286 + resumed_from_preemption=True,
287 + ) for req in scheduled_resumed_reqs
288 ]
289 running_reqs_data = [
294 - self._make_running_request_data(
295 - req, req_to_new_block_ids[req.request_id],
296 - req.num_computed_tokens) for req in scheduled_running_reqs
290 + self._make_cached_request_data(
291 + req,
292 + req_to_new_block_ids[req.request_id],
293 + req.num_computed_tokens,
294 + resumed_from_preemption=False,
295 + ) for req in scheduled_running_reqs
296 ]
298 - preempted_req_ids = {req.request_id for req in preempted_reqs}
297 scheduler_output = SchedulerOutput(
298 scheduled_new_reqs=new_reqs_data,
301 - scheduled_resumed_reqs=resumed_reqs_data,
302 - scheduled_running_reqs=running_reqs_data,
299 + scheduled_cached_reqs=resumed_reqs_data + running_reqs_data,
300 num_scheduled_tokens=num_scheduled_tokens,
301 total_num_scheduled_tokens=total_num_scheduled_tokens,
302 scheduled_encoder_inputs=scheduled_encoder_inputs,
303 num_common_prefix_blocks=num_common_prefix_blocks,
307 - preempted_req_ids=preempted_req_ids,
304 # finished_req_ids is an existing state in the scheduler,
305 # instead of being newly scheduled in this step.
306 # It contains the request IDs that are finished in between
@@ -316,22 +312,26 @@ class Scheduler:
312 self.finished_req_ids = set()
313 return scheduler_output
314
319 - def _make_running_request_data(
315 + def _make_cached_request_data(
316 self,
317 request: Request,
318 new_block_ids: List[int],
319 num_computed_tokens: int,
324 - ) -> "RunningRequestData":
325 - # OPTIMIZATION: Cache the RunningRequestData objects to avoid creating
320 + resumed_from_preemption: bool,
321 + ) -> "CachedRequestData":
322 + # OPTIMIZATION: Cache the CachedRequestData objects to avoid creating
323 # them at each scheduling step.
327 - if request.request_id in self.running_reqs_data:
328 - req_data = self.running_reqs_data[request.request_id]
324 + if request.request_id in self._cached_reqs_data:
325 + req_data = self._cached_reqs_data[request.request_id]
326 + req_data.resumed_from_preemption = resumed_from_preemption
327 req_data.new_block_ids = new_block_ids
328 req_data.num_computed_tokens = num_computed_tokens
329 else:
332 - req_data = RunningRequestData.from_request(request, new_block_ids,
333 - num_computed_tokens)
334 - self.running_reqs_data[request.request_id] = req_data
330 + req_data = CachedRequestData.from_request(request,
331 + resumed_from_preemption,
332 + new_block_ids,
333 + num_computed_tokens)
334 + self._cached_reqs_data[request.request_id] = req_data
335 return req_data
336
337 def _try_schedule_encoder_inputs(
@@ -420,7 +420,13 @@ class Scheduler:
420 # expensive operations inside the loop.
421 for request in self.running:
422 req_id = request.request_id
423 - request.num_computed_tokens += num_scheduled_tokens[req_id]
423 + num_tokens_scheduled = num_scheduled_tokens.get(req_id, 0)
424 + if num_tokens_scheduled == 0:
425 + # The request was not scheduled in this step.
426 + new_running.append(request)
427 + continue
428 +
429 + request.num_computed_tokens += num_tokens_scheduled
430 # When the request's num_computed_tokens catches up its num_tokens,
431 # the request generates output tokens. Otherwise, we ignore the
432 # sampler output for the request.
@@ -529,7 +535,7 @@ class Scheduler:
535 assert request.is_finished()
536 self.kv_cache_manager.free(request)
537 self.encoder_cache_manager.free(request)
532 - self.running_reqs_data.pop(request.request_id, None)
538 + self._cached_reqs_data.pop(request.request_id, None)
539 del self.requests[request.request_id]
540 self.finished_req_ids.add(request.request_id)
541
@@ -584,30 +590,13 @@ class NewRequestData:
590
591
592 @dataclass
587 -class ResumedRequestData:
588 -
589 - req_id: str
590 - block_ids: List[int]
591 - num_computed_tokens: int
592 -
593 - @classmethod
594 - def from_request(
595 - cls,
596 - request: Request,
597 - block_ids: List[int],
598 - num_computed_tokens: int,
599 - ) -> "ResumedRequestData":
600 - return cls(
601 - req_id=request.request_id,
602 - block_ids=block_ids,
603 - num_computed_tokens=num_computed_tokens,
604 - )
605 -
606 -
607 -@dataclass
608 -class RunningRequestData:
593 +class CachedRequestData:
594
595 req_id: str
596 + # If resumed_from_preemption is False, new_block_ids will be appended to
597 + # the request's block IDs. If True, new_block_ids will be used as the
598 + # request's block IDs instead of appending to the existing block IDs.
599 + resumed_from_preemption: bool
600 new_block_ids: List[int]
601 num_computed_tokens: int
602
@@ -615,11 +604,13 @@ class RunningRequestData:
604 def from_request(
605 cls,
606 request: Request,
607 + resumed_from_preemption: bool,
608 new_block_ids: List[int],
609 num_computed_tokens: int,
620 - ) -> "RunningRequestData":
610 + ) -> "CachedRequestData":
611 return cls(
612 req_id=request.request_id,
613 + resumed_from_preemption=resumed_from_preemption,
614 new_block_ids=new_block_ids,
615 num_computed_tokens=num_computed_tokens,
616 )
@@ -629,14 +620,12 @@ class RunningRequestData:
620 class SchedulerOutput:
621
622 scheduled_new_reqs: List[NewRequestData]
632 - scheduled_resumed_reqs: List[ResumedRequestData]
633 - scheduled_running_reqs: List[RunningRequestData]
623 + scheduled_cached_reqs: List[CachedRequestData]
624
625 num_scheduled_tokens: Dict[str, int]
626 total_num_scheduled_tokens: int
627 scheduled_encoder_inputs: Dict[str, List[int]]
628 num_common_prefix_blocks: int
629
640 - preempted_req_ids: Set[str]
630 finished_req_ids: Set[str]
631 free_encoder_input_ids: List[Tuple[str, int]]
vllm/v1/worker/block_table.py
+2
@@ -46,6 +46,8 @@ class BlockTable:
46 start: int,
47 block_ids: List[int],
48 ) -> None:
49 + if not block_ids:
50 + return
51 num_blocks = len(block_ids)
52 self.block_table_np[row_idx, start:start + num_blocks] = block_ids
53 self.num_blocks_per_row[row_idx] = start + num_blocks
vllm/v1/worker/gpu_model_runner.py
+75 -53
@@ -205,12 +205,32 @@ class GPUModelRunner:
205 pin_memory=self.pin_memory)
206 self.seq_lens_np = self.seq_lens_cpu.numpy()
207
208 - def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
209 - # Remove stopped requests from the cached states.
210 - # Keep the states of the preempted requests.
208 + def _update_states(self, scheduler_output: "SchedulerOutput") -> bool:
209 + """Update the cached states and the persistent batch with the scheduler
210 + output.
211 +
212 + The updated states are used by the `_prepare_inputs` function to create
213 + the input GPU tensors for the model.
214 +
215 + Returns:
216 + True if there is a new/resumed/paused/finished request in the batch.
217 + If False, we can skip copying SamplingMetadata to the GPU.
218 + """
219 + # Remove finished requests from the cached states.
220 for req_id in scheduler_output.finished_req_ids:
221 self.requests.pop(req_id, None)
222 self.encoder_cache.pop(req_id, None)
223 + # Remove the finished requests from the persistent batch.
224 + # NOTE(woosuk): There could be an edge case where finished_req_ids and
225 + # scheduled_req_ids overlap. This happens when a request is aborted and
226 + # then resubmitted with the same ID. In this case, we treat them as two
227 + # distinct requests - clearing the cached states for the first request
228 + # and handling the second as a new request.
229 + removed_req_indices: List[int] = []
230 + for req_id in scheduler_output.finished_req_ids:
231 + req_index = self.input_batch.remove_request(req_id)
232 + if req_index is not None:
233 + removed_req_indices.append(req_index)
234
235 # Free the cached encoder outputs.
236 for req_id, input_id in scheduler_output.free_encoder_input_ids:
@@ -220,36 +240,22 @@ class GPUModelRunner:
240 if not encoder_outputs:
241 self.encoder_cache.pop(req_id, None)
242
223 - # Remove the requests from the persistent batch.
224 - stopped_req_ids = set().union(
225 - scheduler_output.preempted_req_ids,
226 - scheduler_output.finished_req_ids,
227 - )
228 - removed_req_indices: List[int] = []
229 - for req_id in stopped_req_ids:
243 + # Remove the unscheduled requests from the persistent batch.
244 + # NOTE(woosuk): The unscheduled requests are either preempted requests
245 + # or running requests that are not scheduled in this step. We remove
246 + # them from the persistent batch but keep their cached states since
247 + # they will be scheduled again sometime in the future.
248 + scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys()
249 + cached_req_ids = self.input_batch.req_id_to_index.keys()
250 + unscheduled_req_ids = cached_req_ids - scheduled_req_ids
251 + # NOTE(woosuk): The persistent batch optimization assumes that
252 + # consecutive batches contain mostly the same requests. If batches
253 + # have low request overlap (e.g., alternating between two distinct
254 + # sets of requests), this optimization becomes very inefficient.
255 + for req_id in unscheduled_req_ids:
256 req_index = self.input_batch.remove_request(req_id)
231 - if req_index is not None:
232 - removed_req_indices.append(req_index)
233 -
234 - # Update the states of the running requests.
235 - for req_data in scheduler_output.scheduled_running_reqs:
236 - req_id = req_data.req_id
237 - req_state = self.requests[req_id]
238 - req_index = self.input_batch.req_id_to_index[req_id]
239 -
240 - # Update the num_computed_tokens.
241 - req_state.num_computed_tokens = req_data.num_computed_tokens
242 - self.input_batch.num_computed_tokens_cpu[req_index] = (
243 - req_data.num_computed_tokens)
244 -
245 - # Update the block table.
246 - num_new_blocks = len(req_data.new_block_ids)
247 - if num_new_blocks == 0:
248 - continue
249 - start_index = len(req_state.block_ids)
250 - req_state.block_ids.extend(req_data.new_block_ids)
251 - self.input_batch.block_table.append_row(req_index, start_index,
252 - req_data.new_block_ids)
257 + assert req_index is not None
258 + removed_req_indices.append(req_index)
259
260 req_ids_to_add: List[str] = []
261 # Add new requests to the cached states.
@@ -305,14 +311,36 @@ class GPUModelRunner:
311
312 req_ids_to_add.append(req_id)
313
308 - # Update the cached states of the resumed requests.
309 - for res_req_data in scheduler_output.scheduled_resumed_reqs:
310 - req_id = res_req_data.req_id
314 + # Update the states of the running/resumed requests.
315 + for req_data in scheduler_output.scheduled_cached_reqs:
316 + req_id = req_data.req_id
317 req_state = self.requests[req_id]
318
313 - req_state.block_ids = res_req_data.block_ids
314 - req_state.num_computed_tokens = res_req_data.num_computed_tokens
315 - req_ids_to_add.append(req_id)
319 + # Update the cached states.
320 + req_state.num_computed_tokens = req_data.num_computed_tokens
321 + if not req_data.resumed_from_preemption:
322 + # Append the new blocks to the existing block IDs.
323 + req_state.block_ids.extend(req_data.new_block_ids)
324 + else:
325 + # The request is resumed from preemption.
326 + # Replace the existing block IDs with the new ones.
327 + req_state.block_ids = req_data.new_block_ids
328 +
329 + req_index = self.input_batch.req_id_to_index.get(req_id)
330 + if req_index is None:
331 + # The request is not in the persistent batch.
332 + # The request was either preempted and resumed later, or was not
333 + # scheduled in the previous step and needs to be added again.
334 + req_ids_to_add.append(req_id)
335 + continue
336 +
337 + # Update the persistent batch.
338 + self.input_batch.num_computed_tokens_cpu[req_index] = (
339 + req_data.num_computed_tokens)
340 + start_index = len(req_state.block_ids) - len(
341 + req_data.new_block_ids)
342 + self.input_batch.block_table.append_row(req_index, start_index,
343 + req_data.new_block_ids)
344
345 # Add the new or resumed requests to the persistent batch.
346 # The smaller empty indices are filled first.
@@ -330,6 +358,7 @@ class GPUModelRunner:
358 # Condense the batched states if there are empty indices.
359 if removed_req_indices:
360 self.input_batch.condense(removed_req_indices)
361 + return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
362
363 def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
364 total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
@@ -536,10 +565,10 @@ class GPUModelRunner:
565 prefix_kv_lens=prefix_kv_lens,
566 suffix_kv_lens=suffix_kv_lens,
567 )
539 - # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
540 - # request in the batch. While we should not sample any token from this
541 - # partial request, we do so for simplicity. We will ignore the sampled
542 - # token from the partial request.
568 + # NOTE(woosuk): Due to chunked prefills, the batch may contain partial
569 + # requests. While we should not sample any token from these partial
570 + # requests, we do so for simplicity. We will ignore the sampled
571 + # tokens from the partial requests.
572 # TODO: Support prompt logprobs.
573 logits_indices = query_start_loc[1:] - 1
574 return attn_metadata, logits_indices
@@ -601,22 +630,15 @@ class GPUModelRunner:
630
631 def _prepare_sampling(
632 self,
604 - scheduler_output: "SchedulerOutput",
633 + batch_changed: bool,
634 ) -> SamplingMetadata:
606 - skip_copy = True
607 - if (scheduler_output.finished_req_ids
608 - or scheduler_output.preempted_req_ids):
609 - skip_copy = False
610 - if (scheduler_output.scheduled_new_reqs
611 - or scheduler_output.scheduled_resumed_reqs):
612 - skip_copy = False
635 # Create the sampling metadata.
636 req_id_output_token_ids: Dict[str, List[int]] = \
637 {req_id: req.output_token_ids \
638 for req_id, req in self.requests.items()}
639
640 sampling_metadata = self.input_batch.make_sampling_metadata(
619 - req_id_output_token_ids, skip_copy)
641 + req_id_output_token_ids, skip_copy=not batch_changed)
642 return sampling_metadata
643
644 def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
@@ -715,7 +737,7 @@ class GPUModelRunner:
737 self,
738 scheduler_output: "SchedulerOutput",
739 ) -> ModelRunnerOutput:
718 - self._update_states(scheduler_output)
740 + batch_changed = self._update_states(scheduler_output)
741
742 if self.is_multimodal_model:
743 # Run the multimodal encoder if any.
@@ -778,7 +800,7 @@ class GPUModelRunner:
800 logits = self.model.compute_logits(hidden_states, None)
801
802 # Sample the next token and get logprobs if needed.
781 - sampling_metadata = self._prepare_sampling(scheduler_output)
803 + sampling_metadata = self._prepare_sampling(batch_changed)
804 sampler_output = self.model.sample(
805 logits=logits,
806 sampling_metadata=sampling_metadata,