A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] Support VLMs with fine-grained scheduling (#9871)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Roger Wang <ywang@roblox.com>
Woosuk Kwon committed
Nov 12, 2024 at 20:53 UTC
bbd3e86926f15e59e4c62246b4b3185e71fe7ff2
12 files changed
+542
-96
vllm/model_executor/models/gpt2.py
+9
-2
@@ -216,9 +216,11 @@ class GPT2Model(nn.Module):
216
kv_caches: List[torch.Tensor],
217
attn_metadata: AttentionMetadata,
218
intermediate_tensors: Optional[IntermediateTensors],
219
+ inputs_embeds: Optional[torch.Tensor],
220
) -> Union[torch.Tensor, IntermediateTensors]:
221
if get_pp_group().is_first_rank:
221
- inputs_embeds = self.wte(input_ids)
222
+ if inputs_embeds is None:
223
+ inputs_embeds = self.wte(input_ids)
224
position_embeds = self.wpe(position_ids)
225
hidden_states = inputs_embeds + position_embeds
226
else:
@@ -263,6 +265,9 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
265
self.make_empty_intermediate_tensors = (
266
self.transformer.make_empty_intermediate_tensors)
267
268
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
269
+ return self.transformer.wte(input_ids)
270
+
271
def forward(
272
self,
273
input_ids: torch.Tensor,
@@ -270,9 +275,11 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
275
kv_caches: List[torch.Tensor],
276
attn_metadata: AttentionMetadata,
277
intermediate_tensors: Optional[IntermediateTensors] = None,
278
+ inputs_embeds: Optional[torch.Tensor] = None,
279
) -> Union[torch.Tensor, IntermediateTensors]:
280
hidden_states = self.transformer(input_ids, positions, kv_caches,
275
- attn_metadata, intermediate_tensors)
281
+ attn_metadata, intermediate_tensors,
282
+ inputs_embeds)
283
return hidden_states
284
285
def compute_logits(
vllm/model_executor/models/llama.py
+6
-1
@@ -538,6 +538,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
538
normalize=False,
539
softmax=False)
540
541
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
542
+ return self.model.get_input_embeddings(input_ids)
543
+
544
def forward(
545
self,
546
input_ids: torch.Tensor,
@@ -545,9 +548,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
548
kv_caches: List[torch.Tensor],
549
attn_metadata: AttentionMetadata,
550
intermediate_tensors: Optional[IntermediateTensors] = None,
551
+ inputs_embeds: Optional[torch.Tensor] = None,
552
) -> Union[torch.Tensor, IntermediateTensors]:
553
model_output = self.model(input_ids, positions, kv_caches,
550
- attn_metadata, intermediate_tensors)
554
+ attn_metadata, intermediate_tensors,
555
+ inputs_embeds)
556
return model_output
557
558
def compute_logits(
vllm/model_executor/models/llava.py
+28
-18
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
+from vllm.multimodal.base import NestedTensors
21
from vllm.sequence import IntermediateTensors
22
from vllm.utils import is_list_of
23
@@ -448,6 +449,25 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
449
image_features = self._process_image_pixels(image_input)
450
return self.multi_modal_projector(image_features)
451
452
+ def process_mm_inputs(self, **kwargs):
453
+ image_input = self._parse_and_validate_image_input(**kwargs)
454
+ if image_input is None:
455
+ return None
456
+ vision_embeddings = self._process_image_input(image_input)
457
+ return vision_embeddings
458
+
459
+ def get_input_embeddings(
460
+ self,
461
+ input_ids: torch.Tensor,
462
+ vision_embeddings: Optional[NestedTensors] = None,
463
+ ) -> torch.Tensor:
464
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
465
+ if vision_embeddings is not None:
466
+ inputs_embeds = merge_multimodal_embeddings(
467
+ input_ids, inputs_embeds, vision_embeddings,
468
+ self.config.image_token_index)
469
+ return inputs_embeds
470
+
471
def forward(
472
self,
473
input_ids: torch.Tensor,
@@ -455,6 +475,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
475
kv_caches: List[torch.Tensor],
476
attn_metadata: AttentionMetadata,
477
intermediate_tensors: Optional[IntermediateTensors] = None,
478
+ inputs_embeds: Optional[torch.Tensor] = None,
479
**kwargs: object,
480
) -> Union[torch.Tensor, IntermediateTensors]:
481
"""Run forward pass for LLaVA-1.5.
@@ -494,24 +515,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
515
"""
516
if intermediate_tensors is not None:
517
inputs_embeds = None
497
- else:
498
- image_input = self._parse_and_validate_image_input(**kwargs)
499
- if image_input is not None:
500
- vision_embeddings = self._process_image_input(image_input)
501
- inputs_embeds = self.language_model.model.get_input_embeddings(
502
- input_ids)
503
-
504
- inputs_embeds = merge_multimodal_embeddings(
505
- input_ids, inputs_embeds, vision_embeddings,
506
- self.config.image_token_index)
507
- else:
508
- inputs_embeds = self.language_model.model.get_input_embeddings(
509
- input_ids)
510
-
511
- # always pass the input via `inputs_embeds`
512
- # to make sure the computation graph is consistent
513
- # for `torch.compile` integration
514
- input_ids = None
518
+ elif inputs_embeds is None:
519
+ vision_embeddings = self.process_mm_inputs(**kwargs)
520
+ # always pass the input via `inputs_embeds`
521
+ # to make sure the computation graph is consistent
522
+ inputs_embeds = self.get_input_embeddings(input_ids,
523
+ vision_embeddings)
524
+ input_ids = None
525
526
hidden_states = self.language_model.model(input_ids,
527
positions,
vllm/model_executor/models/opt.py
+6
-1
@@ -360,6 +360,9 @@ class OPTForCausalLM(nn.Module, SupportsPP):
360
self.make_empty_intermediate_tensors = (
361
self.model.make_empty_intermediate_tensors)
362
363
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
364
+ return self.model.get_input_embeddings(input_ids)
365
+
366
def forward(
367
self,
368
input_ids: torch.Tensor,
@@ -367,9 +370,11 @@ class OPTForCausalLM(nn.Module, SupportsPP):
370
kv_caches: List[torch.Tensor],
371
attn_metadata: AttentionMetadata,
372
intermediate_tensors: Optional[IntermediateTensors] = None,
373
+ inputs_embeds: Optional[torch.Tensor] = None,
374
) -> Union[torch.Tensor, IntermediateTensors]:
375
hidden_states = self.model(input_ids, positions, kv_caches,
372
- attn_metadata, intermediate_tensors)
376
+ attn_metadata, intermediate_tensors,
377
+ inputs_embeds)
378
return hidden_states
379
380
def compute_logits(
vllm/model_executor/models/phi3v.py
+40
-23
@@ -39,6 +39,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
39
from vllm.model_executor.pooling_metadata import PoolingMetadata
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
from vllm.multimodal import MULTIMODAL_REGISTRY
42
+from vllm.multimodal.base import NestedTensors, PlaceholderRange
43
from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
44
from vllm.sequence import IntermediateTensors, PoolerOutput
45
from vllm.utils import is_list_of
@@ -500,15 +501,20 @@ def input_processor_for_phi3v(ctx: InputContext,
501
502
# TODO: Move this to utils or integrate with clip.
503
new_token_ids: List[int] = []
504
+ placeholder_ranges: List[PlaceholderRange] = []
505
placeholder_idx = 0
506
while merged_token_ids:
507
token_id = merged_token_ids.pop(0)
508
if token_id == _IMAGE_TOKEN_ID:
507
- new_token_ids.extend(
508
- repeat_and_pad_token(
509
- _IMAGE_TOKEN_ID,
510
- repeat_count=image_feature_size[placeholder_idx],
511
- ))
509
+ replacement_ids = repeat_and_pad_token(
510
+ _IMAGE_TOKEN_ID,
511
+ repeat_count=image_feature_size[placeholder_idx],
512
+ )
513
+ placeholder_ranges.append({
514
+ "offset": len(new_token_ids),
515
+ "length": len(replacement_ids)
516
+ })
517
+ new_token_ids.extend(replacement_ids)
518
placeholder_idx += 1
519
else:
520
new_token_ids.append(token_id)
@@ -516,7 +522,8 @@ def input_processor_for_phi3v(ctx: InputContext,
522
# NOTE: Create a defensive copy of the original inputs
523
return token_inputs(prompt_token_ids=new_token_ids,
524
prompt=new_prompt,
519
- multi_modal_data=multi_modal_data)
525
+ multi_modal_data=multi_modal_data,
526
+ multi_modal_placeholders={"image": placeholder_ranges})
527
528
529
@MULTIMODAL_REGISTRY.register_image_input_mapper()
@@ -669,32 +676,42 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
676
677
return image_embeds
678
679
+ def process_mm_inputs(self, **kwargs):
680
+ image_input = self._parse_and_validate_image_input(**kwargs)
681
+ if image_input is None:
682
+ return None
683
+ vision_embeddings = self._process_image_input(image_input)
684
+ return vision_embeddings
685
+
686
+ def get_input_embeddings(
687
+ self,
688
+ input_ids: torch.Tensor,
689
+ vision_embeddings: Optional[NestedTensors] = None,
690
+ ) -> torch.Tensor:
691
+ inputs_embeds = self.embed_tokens(input_ids)
692
+ if vision_embeddings is not None:
693
+ inputs_embeds = merge_multimodal_embeddings(
694
+ input_ids, inputs_embeds, vision_embeddings,
695
+ self.image_token_id)
696
+ return inputs_embeds
697
+
698
def forward(self,
699
input_ids: torch.Tensor,
700
positions: torch.Tensor,
701
kv_caches: List[torch.Tensor],
702
attn_metadata: AttentionMetadata,
703
intermediate_tensors: Optional[IntermediateTensors] = None,
704
+ inputs_embeds: Optional[torch.Tensor] = None,
705
**kwargs: object):
706
if intermediate_tensors is not None:
707
inputs_embeds = None
681
- else:
682
- image_input = self._parse_and_validate_image_input(**kwargs)
683
-
684
- if image_input is not None:
685
- vision_embeddings = self._process_image_input(image_input)
686
- inputs_embeds = self.embed_tokens(input_ids)
687
- inputs_embeds = merge_multimodal_embeddings(
688
- input_ids, inputs_embeds, vision_embeddings,
689
- self.image_token_id)
690
- else:
691
- inputs_embeds = self.language_model.model.embed_tokens(
692
- input_ids)
693
-
694
- # always pass the input via `inputs_embeds`
695
- # to make sure the computation graph is consistent
696
- # for `torch.compile` integration
697
- input_ids = None
708
+ elif inputs_embeds is None:
709
+ vision_embeddings = self.process_mm_inputs(**kwargs)
710
+ # always pass the input via `inputs_embeds`
711
+ # to make sure the computation graph is consistent
712
+ inputs_embeds = self.get_input_embeddings(input_ids,
713
+ vision_embeddings)
714
+ input_ids = None
715
716
hidden_states = self.language_model.model(input_ids,
717
positions,
vllm/model_executor/models/qwen2.py
+6
-1
@@ -441,6 +441,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
441
self.make_empty_intermediate_tensors = (
442
self.model.make_empty_intermediate_tensors)
443
444
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
445
+ return self.model.get_input_embeddings(input_ids)
446
+
447
def forward(
448
self,
449
input_ids: torch.Tensor,
@@ -448,9 +451,11 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
451
kv_caches: List[torch.Tensor],
452
attn_metadata: AttentionMetadata,
453
intermediate_tensors: Optional[IntermediateTensors] = None,
454
+ inputs_embeds: Optional[torch.Tensor] = None,
455
) -> Union[torch.Tensor, IntermediateTensors]:
456
hidden_states = self.model(input_ids, positions, kv_caches,
453
- attn_metadata, intermediate_tensors)
457
+ attn_metadata, intermediate_tensors,
458
+ inputs_embeds)
459
return hidden_states
460
461
def compute_logits(
vllm/v1/core/encoder_cache_manager.py
new
+48
@@ -0,0 +1,48 @@
1
+from typing import Dict, List, Set, Tuple
2
+
3
+from vllm.v1.request import Request
4
+
5
+
6
+class EncoderCacheManager:
7
+
8
+ def __init__(self, cache_size: int):
9
+ self.cache_size = cache_size
10
+ self.num_free_slots = cache_size
11
+ # req_id -> cached input ids
12
+ self.cached: Dict[str, Set[int]] = {}
13
+ # List of [req_id, input_id]
14
+ self.freed: List[Tuple[str, int]] = []
15
+
16
+ def has_cache(self, request: Request, input_id: int) -> bool:
17
+ req_id = request.request_id
18
+ return req_id in self.cached and input_id in self.cached[req_id]
19
+
20
+ def can_allocate(self, request: Request, input_id: int) -> bool:
21
+ num_tokens = request.get_num_encoder_tokens(input_id)
22
+ return num_tokens <= self.num_free_slots
23
+
24
+ def allocate(self, request: Request, input_id: int) -> None:
25
+ req_id = request.request_id
26
+ if req_id not in self.cached:
27
+ self.cached[req_id] = set()
28
+ self.cached[req_id].add(input_id)
29
+ self.num_free_slots -= request.get_num_encoder_tokens(input_id)
30
+
31
+ def get_cached_input_ids(self, request: Request) -> Set[int]:
32
+ return self.cached.get(request.request_id, set())
33
+
34
+ def free(self, request: Request, input_id: int) -> None:
35
+ req_id = request.request_id
36
+ if req_id not in self.cached:
37
+ return
38
+
39
+ self.cached[req_id].discard(input_id)
40
+ if len(self.cached[req_id]) == 0:
41
+ del self.cached[req_id]
42
+ self.num_free_slots += request.get_num_encoder_tokens(input_id)
43
+ self.freed.append((req_id, input_id))
44
+
45
+ def get_freed_ids(self) -> List[Tuple[str, int]]:
46
+ freed = self.freed
47
+ self.freed = []
48
+ return freed
vllm/v1/core/scheduler.py
+181
-24
@@ -1,16 +1,21 @@
1
from collections import deque
2
from dataclasses import dataclass
3
-from typing import Deque, Dict, Iterable, List, Optional, Set, Union
3
+from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
4
+ Tuple, Union)
5
6
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7
from vllm.logger import init_logger
7
-from vllm.multimodal import MultiModalDataDict
8
from vllm.sampling_params import SamplingParams
9
+from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
10
from vllm.v1.core.kv_cache_manager import KVCacheManager
11
from vllm.v1.engine import EngineCoreOutput
12
from vllm.v1.outputs import ModelRunnerOutput
13
from vllm.v1.request import Request, RequestStatus
14
15
+if TYPE_CHECKING:
16
+ from vllm.multimodal import MultiModalKwargs
17
+ from vllm.multimodal.base import PlaceholderRange
18
+
19
logger = init_logger(__name__)
20
21
@@ -61,12 +66,20 @@ class Scheduler:
66
# Request id -> RunningRequestData
67
self.running_reqs_data: Dict[str, RunningRequestData] = {}
68
64
- def schedule(self) -> "SchedulerOutput":
65
- scheduled_new_reqs: List[Request] = []
66
- scheduled_resumed_reqs: List[Request] = []
67
- scheduled_running_reqs: List[Request] = []
68
- preempted_reqs: List[Request] = []
69
+ # Encoder-related.
70
+ # NOTE(woosuk): Here, "encoder" includes the vision encoder (and
71
+ # projector if needed). Currently, we assume that the encoder also
72
+ # has the Transformer architecture (e.g., ViT).
73
+ # FIXME(woosuk): Below are placeholder values. We need to calculate the
74
+ # actual values from the configurations.
75
+ self.max_num_encoder_input_tokens = 2048
76
+ # NOTE(woosuk): For the models without encoder (e.g., text-only models),
77
+ # the encoder cache will not be initialized and used, regardless of
78
+ # the cache size. This is because the memory space for the encoder cache
79
+ # is preallocated in the profiling run.
80
+ self.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
81
82
+ def schedule(self) -> "SchedulerOutput":
83
# NOTE(woosuk) on the scheduling algorithm:
84
# There's no "decoding phase" nor "prefill phase" in the scheduler.
85
# Each request just has the num_computed_tokens and num_tokens,
@@ -74,23 +87,45 @@ class Scheduler:
87
# At each step, the scheduler tries to assign tokens to the requests
88
# so that each request's num_computed_tokens can catch up its
89
# num_tokens. This is general enough to cover chunked prefills,
77
- # prefix caching, and the "jump forward" optimization in the future.
90
+ # prefix caching, and the "jump decoding" optimization in the future.
91
+
92
+ scheduled_new_reqs: List[Request] = []
93
+ scheduled_resumed_reqs: List[Request] = []
94
+ scheduled_running_reqs: List[Request] = []
95
+ preempted_reqs: List[Request] = []
96
97
req_to_new_block_ids: Dict[str, List[int]] = {}
98
num_scheduled_tokens: Dict[str, int] = {}
99
token_budget = self.max_num_scheduled_tokens
100
+ # Encoder-related.
101
+ scheduled_encoder_inputs: Dict[str, List[int]] = {}
102
+ encoder_budget = self.max_num_encoder_input_tokens
103
104
# First, schedule the RUNNING requests.
105
+ # NOTE(woosuk): At most 1 request in the RUNNING queue is allowed to be
106
+ # in the "partial" state, where the request has some tokens computed
107
+ # but not all. The constraint is due to the persistent batch in the
108
+ # V1 model runner.
109
+ # TODO(woosuk): Remove this constraint after refactoring model runner.
110
+ has_partial_request = False
111
req_index = 0
112
while req_index < len(self.running):
86
- if token_budget == 0:
87
- break
88
-
113
+ # Only the last request in the RUNNING queue can be "partial".
114
+ assert not has_partial_request
115
+ assert token_budget > 0
116
request = self.running[req_index]
117
num_new_tokens = request.num_tokens - request.num_computed_tokens
118
num_new_tokens = min(num_new_tokens, token_budget)
119
assert num_new_tokens > 0
120
121
+ # Schedule encoder inputs.
122
+ encoder_inputs_to_schedule, num_new_tokens, new_encoder_budget = (
123
+ self._try_schedule_encoder_inputs(request,
124
+ request.num_computed_tokens,
125
+ num_new_tokens,
126
+ encoder_budget))
127
+ assert num_new_tokens > 0
128
+
129
while True:
130
new_blocks = self.kv_cache_manager.append_slots(
131
request, num_new_tokens)
@@ -106,22 +141,40 @@ class Scheduler:
141
preempted_reqs.append(preempted_req)
142
if preempted_req == request:
143
# No more request to preempt.
144
+ can_schedule = False
145
break
146
else:
147
# The request can be scheduled.
112
- scheduled_running_reqs.append(request)
113
-
114
- req_to_new_block_ids[request.request_id] = [
115
- b.block_id for b in new_blocks
116
- ]
117
- num_scheduled_tokens[request.request_id] = num_new_tokens
118
- token_budget -= num_new_tokens
119
- req_index += 1
148
+ can_schedule = True
149
break
150
+ if not can_schedule:
151
+ break
152
+
153
+ # Schedule the request.
154
+ scheduled_running_reqs.append(request)
155
+ req_to_new_block_ids[request.request_id] = [
156
+ b.block_id for b in new_blocks
157
+ ]
158
+ num_scheduled_tokens[request.request_id] = num_new_tokens
159
+ token_budget -= num_new_tokens
160
+ req_index += 1
161
+ has_partial_request = (request.num_computed_tokens + num_new_tokens
162
+ < request.num_tokens)
163
+
164
+ # Encoder-related.
165
+ if encoder_inputs_to_schedule:
166
+ scheduled_encoder_inputs[request.request_id] = (
167
+ encoder_inputs_to_schedule)
168
+ # Allocate the encoder cache.
169
+ for i in encoder_inputs_to_schedule:
170
+ self.encoder_cache_manager.allocate(request, i)
171
+ encoder_budget = new_encoder_budget
172
173
# Next, schedule the WAITING requests.
174
if not preempted_reqs:
175
while self.waiting:
176
+ if has_partial_request:
177
+ break
178
if len(self.running) == self.max_num_running_reqs:
179
break
180
if token_budget == 0:
@@ -149,12 +202,21 @@ class Scheduler:
202
computed_blocks.pop()
203
num_new_tokens = min(num_new_tokens, token_budget)
204
assert num_new_tokens > 0
205
+
206
+ # Schedule encoder inputs.
207
+ (encoder_inputs_to_schedule, num_new_tokens,
208
+ new_encoder_budget) = self._try_schedule_encoder_inputs(
209
+ request, num_computed_tokens, num_new_tokens,
210
+ encoder_budget)
211
+ if num_new_tokens == 0:
212
+ # The request cannot be scheduled.
213
+ break
214
+
215
new_blocks = self.kv_cache_manager.allocate_slots(
216
request, num_new_tokens, computed_blocks)
217
if new_blocks is None:
218
# The request cannot be scheduled.
219
break
157
- request.num_computed_tokens = num_computed_tokens
220
221
self.waiting.popleft()
222
self.running.append(request)
@@ -172,6 +234,18 @@ class Scheduler:
234
num_scheduled_tokens[request.request_id] = num_new_tokens
235
token_budget -= num_new_tokens
236
request.status = RequestStatus.RUNNING
237
+ request.num_computed_tokens = num_computed_tokens
238
+ has_partial_request = (num_computed_tokens + num_new_tokens <
239
+ request.num_tokens)
240
+
241
+ # Encoder-related.
242
+ if encoder_inputs_to_schedule:
243
+ scheduled_encoder_inputs[request.request_id] = (
244
+ encoder_inputs_to_schedule)
245
+ # Allocate the encoder cache.
246
+ for i in encoder_inputs_to_schedule:
247
+ self.encoder_cache_manager.allocate(request, i)
248
+ encoder_budget = new_encoder_budget
249
250
# Check if the scheduling constraints are satisfied.
251
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
@@ -205,12 +279,14 @@ class Scheduler:
279
scheduled_running_reqs=running_reqs_data,
280
num_scheduled_tokens=num_scheduled_tokens,
281
total_num_scheduled_tokens=total_num_scheduled_tokens,
282
+ scheduled_encoder_inputs=scheduled_encoder_inputs,
283
preempted_req_ids=preempted_req_ids,
284
# finished_req_ids is an existing state in the scheduler,
285
# instead of being newly scheduled in this step.
286
# It contains the request IDs that are finished in between
287
# the previous and the current steps.
288
finished_req_ids=self.finished_req_ids,
289
+ free_encoder_input_ids=self.encoder_cache_manager.get_freed_ids(),
290
)
291
292
self.finished_req_ids = set()
@@ -234,6 +310,72 @@ class Scheduler:
310
self.running_reqs_data[request.request_id] = req_data
311
return req_data
312
313
+ def _try_schedule_encoder_inputs(
314
+ self,
315
+ request: Request,
316
+ num_computed_tokens: int,
317
+ num_new_tokens: int,
318
+ encoder_budget: int,
319
+ ) -> Tuple[List[int], int, int]:
320
+ """
321
+ Determine which encoder inputs need to be scheduled in the current step,
322
+ and update `num_new_tokens` and encoder token budget accordingly.
323
+
324
+ An encoder input will be scheduled if:
325
+ - Its output tokens overlap with the range of tokens being computed
326
+ in this step, i.e.,
327
+ [num_computed_tokens, num_computed_tokens + num_new_tokens).
328
+ - It is not already computed and stored in the encoder cache.
329
+ - There is sufficient encoder token budget to process it.
330
+ - The encoder cache has space to store it.
331
+
332
+ If an encoder input cannot be scheduled due to cache or budget
333
+ limitations, the method adjusts `num_new_tokens` to schedule only the
334
+ decoder tokens up to just before the unschedulable encoder input.
335
+ """
336
+ if not request.has_encoder_inputs():
337
+ return [], num_new_tokens, encoder_budget
338
+
339
+ encoder_inputs_to_schedule: List[int] = []
340
+ mm_positions = request.mm_positions
341
+ assert mm_positions is not None
342
+ assert len(mm_positions) > 0
343
+ for i, pos_info in enumerate(mm_positions):
344
+ start_pos = pos_info["offset"]
345
+ num_encoder_tokens = pos_info["length"]
346
+
347
+ # The encoder output is needed if the two ranges overlap:
348
+ # [num_computed_tokens, num_computed_tokens + num_new_tokens) and
349
+ # [start_pos, start_pos + num_encoder_tokens)
350
+ if start_pos >= num_computed_tokens + num_new_tokens:
351
+ # The encoder input is not needed in this step.
352
+ break
353
+ if start_pos + num_encoder_tokens <= num_computed_tokens:
354
+ # The encoder input is already computed and stored
355
+ # in the decoder's KV cache.
356
+ continue
357
+
358
+ if self.encoder_cache_manager.has_cache(request, i):
359
+ # The encoder input is already computed and cached.
360
+ continue
361
+ if not self.encoder_cache_manager.can_allocate(request, i):
362
+ # The encoder cache is full. We can only schedule the decoder
363
+ # tokens just before the encoder input.
364
+ num_new_tokens = start_pos - num_computed_tokens
365
+ break
366
+ if num_encoder_tokens > encoder_budget:
367
+ # The encoder budget is exhausted. We can only schedule the
368
+ # decoder tokens up until the encoder input.
369
+ # NOTE(woosuk): We assume that the encoder tokens should be
370
+ # processed altogether, as the encoder usually uses
371
+ # bidirectional attention.
372
+ num_new_tokens = start_pos - num_computed_tokens
373
+ break
374
+
375
+ encoder_budget -= num_encoder_tokens
376
+ encoder_inputs_to_schedule.append(i)
377
+ return encoder_inputs_to_schedule, num_new_tokens, encoder_budget
378
+
379
def update_from_output(
380
self,
381
scheduler_output: "SchedulerOutput",
@@ -251,6 +393,17 @@ class Scheduler:
393
# the request generates output tokens. Otherwise, we ignore the
394
# sampler output for the request.
395
assert request.num_computed_tokens <= request.num_tokens
396
+
397
+ cached_encoder_input_ids = (
398
+ self.encoder_cache_manager.get_cached_input_ids(request))
399
+ for input_id in list(cached_encoder_input_ids):
400
+ start_pos = request.mm_positions[input_id]["offset"]
401
+ num_tokens = request.mm_positions[input_id]["length"]
402
+ if start_pos + num_tokens <= request.num_computed_tokens:
403
+ # The encoder output is already processed and stored
404
+ # in the decoder's KV cache.
405
+ self.encoder_cache_manager.free(request, input_id)
406
+
407
if request.num_computed_tokens == request.num_tokens:
408
req_index = model_runner_output.req_id_to_index[req_id]
409
# NOTE(woosuk): Currently, we assume that each request
@@ -355,7 +508,8 @@ class NewRequestData:
508
req_id: str
509
prompt_token_ids: List[int]
510
prompt: Optional[str]
358
- multi_modal_data: Optional[MultiModalDataDict]
511
+ mm_inputs: List["MultiModalKwargs"]
512
+ mm_positions: List["PlaceholderRange"]
513
sampling_params: SamplingParams
514
block_ids: List[int]
515
num_computed_tokens: int
@@ -369,9 +523,10 @@ class NewRequestData:
523
) -> "NewRequestData":
524
return cls(
525
req_id=request.request_id,
372
- prompt_token_ids=request.inputs["prompt_token_ids"],
373
- prompt=request.inputs.get("prompt"),
374
- multi_modal_data=request.inputs.get("multi_modal_data"),
526
+ prompt_token_ids=request.prompt_token_ids,
527
+ prompt=request.prompt,
528
+ mm_inputs=request.mm_inputs,
529
+ mm_positions=request.mm_positions,
530
sampling_params=request.sampling_params,
531
block_ids=block_ids,
532
num_computed_tokens=num_computed_tokens,
@@ -429,6 +584,8 @@ class SchedulerOutput:
584
585
num_scheduled_tokens: Dict[str, int]
586
total_num_scheduled_tokens: int
587
+ scheduled_encoder_inputs: Dict[str, List[int]]
588
589
preempted_req_ids: Set[str]
590
finished_req_ids: Set[str]
591
+ free_encoder_input_ids: List[Tuple[str, int]]
vllm/v1/engine/core.py
+10
@@ -17,6 +17,7 @@ from vllm.usage.usage_lib import UsageContext
17
from vllm.v1.core.scheduler import Scheduler
18
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19
EngineCoreRequest, EngineCoreRequestType)
20
+from vllm.v1.engine.mm_input_mapper import MMInputMapper
21
from vllm.v1.executor.gpu_executor import GPUExecutor
22
from vllm.v1.request import Request, RequestStatus
23
from vllm.v1.serial_utils import PickleEncoder
@@ -65,6 +66,9 @@ class EngineCore:
66
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
67
vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
68
69
+ # Set up multimodal input mapper (e.g., convert PIL images to tensors).
70
+ self.mm_input_mapper = MMInputMapper(vllm_config.model_config)
71
+
72
# Setup scheduler.
73
self.scheduler = Scheduler(vllm_config.scheduler_config,
74
vllm_config.cache_config,
@@ -93,6 +97,12 @@ class EngineCore:
97
"""Add request to the scheduler."""
98
99
req = Request.from_engine_core_request(request)
100
+ # FIXME(woosuk): The input mapping (e.g., PIL images to tensors) may
101
+ # take 10-50 ms, which can cause a spike in the latency. We should
102
+ # consider moving this to a separate thread.
103
+ if req.mm_data:
104
+ req.mm_inputs = self.mm_input_mapper.process_inputs(
105
+ req.mm_data, req.mm_processor_kwargs)
106
self.scheduler.add_request(req)
107
108
def abort_requests(self, request_ids: List[str]):
vllm/v1/engine/mm_input_mapper.py
new
+39
@@ -0,0 +1,39 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+from vllm.config import ModelConfig
4
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
5
+ MultiModalKwargs, MultiModalRegistry)
6
+
7
+
8
+class MMInputMapper:
9
+
10
+ def __init__(
11
+ self,
12
+ model_config: ModelConfig,
13
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
14
+ ):
15
+ self.mm_registry = mm_registry
16
+ self.multi_modal_input_mapper = mm_registry.create_input_mapper(
17
+ model_config)
18
+ self.mm_registry.init_mm_limits_per_prompt(model_config)
19
+
20
+ def process_inputs(
21
+ self,
22
+ mm_data: MultiModalDataDict,
23
+ mm_processor_kwargs: Optional[Dict[str, Any]],
24
+ ) -> List[MultiModalKwargs]:
25
+ image_inputs = mm_data["image"]
26
+ if not isinstance(image_inputs, list):
27
+ image_inputs = [image_inputs]
28
+
29
+ # Process each image input separately so that later we can schedule
30
+ # them in a fine-grained manner.
31
+ mm_inputs: List[MultiModalKwargs] = []
32
+ num_images = len(image_inputs)
33
+ for i in range(num_images):
34
+ mm_input = self.multi_modal_input_mapper(
35
+ {"image": [image_inputs[i]]},
36
+ mm_processor_kwargs=mm_processor_kwargs,
37
+ )
38
+ mm_inputs.append(mm_input)
39
+ return mm_inputs
vllm/v1/request.py
+35
-6
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
3
4
from vllm.inputs.data import DecoderOnlyInputs
5
from vllm.lora.request import LoRARequest
6
+from vllm.multimodal import MultiModalKwargs
7
from vllm.sampling_params import SamplingParams
8
from vllm.sequence import RequestMetrics
9
from vllm.v1.engine import EngineCoreRequest
@@ -47,14 +48,30 @@ class Request:
48
self._all_token_ids: List[int] = self.prompt_token_ids.copy()
49
self.num_computed_tokens = 0
50
51
+ # Raw multimodal data before the mm input mapper (e.g., PIL images).
52
+ self.mm_data = inputs.get("multi_modal_data")
53
+ self.mm_processor_kwargs = inputs.get("mm_processor_kwargs")
54
+ mm_positions = inputs.get("multi_modal_placeholders")
55
+ if mm_positions:
56
+ # FIXME(woosuk): Support other modalities.
57
+ self.mm_positions = mm_positions.get("image", [])
58
+ else:
59
+ self.mm_positions = []
60
+ # Output of the mm input mapper (e.g., image tensors).
61
+ self.mm_inputs: List[MultiModalKwargs] = []
62
+
63
@classmethod
64
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
52
-
65
return cls(
66
request_id=request.request_id,
55
- inputs=DecoderOnlyInputs(type="token",
56
- prompt_token_ids=request.prompt_token_ids,
57
- prompt=request.prompt),
67
+ inputs=DecoderOnlyInputs(
68
+ type="token",
69
+ prompt_token_ids=request.prompt_token_ids,
70
+ prompt=request.prompt,
71
+ multi_modal_data=request.mm_data,
72
+ multi_modal_placeholders=request.mm_placeholders,
73
+ mm_processor_kwargs=request.mm_processor_kwargs,
74
+ ),
75
sampling_params=request.sampling_params,
76
eos_token_id=request.eos_token_id,
77
arrival_time=request.arrival_time,
@@ -96,9 +113,21 @@ class Request:
113
def get_finished_reason(self) -> Union[str, None]:
114
return RequestStatus.get_finished_reason(self.status)
115
116
+ def has_encoder_inputs(self) -> bool:
117
+ return self.mm_data is not None
118
+
119
+ @property
120
+ def num_encoder_inputs(self) -> int:
121
+ return len(self.mm_positions)
122
+
123
+ def get_num_encoder_tokens(self, input_id: int) -> int:
124
+ assert input_id < len(self.mm_positions)
125
+ num_tokens = self.mm_positions[input_id]["length"]
126
+ return num_tokens
127
+
128
129
class RequestStatus(enum.IntEnum):
101
- """Status of a sequence."""
130
+ """Status of a request."""
131
WAITING = 0
132
RUNNING = 1
133
PREEMPTED = 2
@@ -119,7 +148,7 @@ class RequestStatus(enum.IntEnum):
148
149
150
# Mapping of finished statuses to their finish reasons.
122
-# NOTE: The ignored sequences are the sequences whose prompt lengths
151
+# NOTE: The ignored requests are the requests whose prompt lengths
152
# are longer than the model's length cap. Therefore, the stop
153
# reason should also be "length" as in OpenAI API.
154
_FINISHED_REASON_MAP = {
vllm/v1/worker/gpu_model_runner.py
+134
-20
@@ -1,7 +1,7 @@
1
import os
2
import time
3
from dataclasses import dataclass
4
-from typing import TYPE_CHECKING, Dict, List, Optional, Set
4
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
5
6
import numpy as np
7
import torch
@@ -14,9 +14,10 @@ from vllm.compilation.config import CompilationConfig
14
from vllm.compilation.levels import CompilationLevel
15
from vllm.config import VllmConfig
16
from vllm.forward_context import set_forward_context
17
+from vllm.inputs import INPUT_REGISTRY, InputRegistry
18
from vllm.logger import init_logger
19
from vllm.model_executor.model_loader import get_model
19
-from vllm.multimodal import MultiModalDataDict
20
+from vllm.multimodal import MultiModalKwargs
21
from vllm.plugins import set_compilation_config
22
from vllm.sampling_params import SamplingParams, SamplingType
23
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
@@ -27,6 +28,7 @@ from vllm.v1.outputs import ModelRunnerOutput
28
from vllm.v1.sample.metadata import SamplingMetadata
29
30
if TYPE_CHECKING:
31
+ from vllm.multimodal.base import PlaceholderRange
32
from vllm.v1.core.scheduler import SchedulerOutput
33
34
logger = init_logger(__name__)
@@ -37,8 +39,8 @@ class GPUModelRunner:
39
def __init__(
40
self,
41
vllm_config: VllmConfig,
42
+ input_registry: InputRegistry = INPUT_REGISTRY,
43
):
41
- # TODO: use ModelRunnerBase.__init__(self, vllm_config=vllm_config)
44
self.vllm_config = vllm_config
45
self.model_config = vllm_config.model_config
46
self.cache_config = vllm_config.cache_config
@@ -75,10 +77,16 @@ class GPUModelRunner:
77
parallel_config)
78
self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
79
self.head_size = model_config.get_head_size()
80
+ self.hidden_size = model_config.get_hidden_size()
81
+
82
+ # Multi-modal data support
83
+ self.input_registry = input_registry
84
85
# Lazy initialization
86
# self.model: nn.Module # Set after load_model
87
self.kv_caches: List[torch.Tensor] = []
88
+ # req_id -> (input_id -> encoder_output)
89
+ self.encoder_cache: Dict[str, Dict[int, torch.Tensor]] = {}
90
91
# Request states.
92
self.requests: Dict[str, CachedRequestState] = {}
@@ -96,18 +104,28 @@ class GPUModelRunner:
104
and not self.model_config.enforce_eager)
105
# TODO(woosuk): Provide an option to tune the max cudagraph batch size.
106
self.cudagraph_batch_sizes = [1, 2, 4] + [i for i in range(8, 513, 8)]
99
- self.input_ids = torch.zeros(self.max_num_tokens,
100
- dtype=torch.int32,
101
- device=self.device)
107
self.positions = torch.zeros(self.max_num_tokens,
108
dtype=torch.int64,
109
device=self.device)
110
+ self.inputs_embeds = torch.zeros(
111
+ (self.max_num_tokens, self.hidden_size),
112
+ dtype=self.dtype,
113
+ device=self.device)
114
115
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
116
# Remove stopped requests from the cached states.
117
# Keep the states of the pre-empted requests.
118
for req_id in scheduler_output.finished_req_ids:
119
self.requests.pop(req_id, None)
120
+ self.encoder_cache.pop(req_id, None)
121
+
122
+ # Free the cached encoder outputs.
123
+ for req_id, input_id in scheduler_output.free_encoder_input_ids:
124
+ encoder_outputs = self.encoder_cache.get(req_id)
125
+ if encoder_outputs is not None:
126
+ encoder_outputs.pop(input_id, None)
127
+ if not encoder_outputs:
128
+ self.encoder_cache.pop(req_id, None)
129
130
# Remove the requests from the persistent batch.
131
stopped_req_ids = set().union(
@@ -156,7 +174,8 @@ class GPUModelRunner:
174
req_id=req_id,
175
prompt_token_ids=req_data.prompt_token_ids,
176
prompt=req_data.prompt,
159
- multi_modal_data=req_data.multi_modal_data,
177
+ mm_inputs=req_data.mm_inputs,
178
+ mm_positions=req_data.mm_positions,
179
sampling_params=sampling_params,
180
generator=generator,
181
block_ids=req_data.block_ids,
@@ -285,11 +304,9 @@ class GPUModelRunner:
304
seq_start_loc_np[0] = 0
305
np.cumsum(seq_lens, out=seq_start_loc_np[1:])
306
288
- self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
289
- non_blocking=True)
307
+ input_ids = input_ids.to(self.device, non_blocking=True)
308
self.positions[:total_num_scheduled_tokens].copy_(positions,
309
non_blocking=True)
292
-
310
query_start_loc = query_start_loc.to(self.device, non_blocking=True)
311
seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
312
slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
@@ -308,7 +325,7 @@ class GPUModelRunner:
325
# token from the partial request.
326
# TODO: Support prompt logprobs.
327
logits_indices = query_start_loc[1:] - 1
311
- return attn_metadata, logits_indices
328
+ return input_ids, attn_metadata, logits_indices
329
330
def _prepare_sampling(
331
self,
@@ -325,13 +342,91 @@ class GPUModelRunner:
342
sampling_metadata = self.input_batch.make_sampling_metadata(skip_copy)
343
return sampling_metadata
344
345
+ def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
346
+ scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
347
+ if not scheduled_encoder_inputs:
348
+ return
349
+
350
+ # Batch the multi-modal inputs.
351
+ mm_inputs: List[MultiModalKwargs] = []
352
+ req_input_ids: List[Tuple[int, int]] = []
353
+ for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
354
+ req_state = self.requests[req_id]
355
+ for input_id in encoder_input_ids:
356
+ mm_inputs.append(req_state.mm_inputs[input_id])
357
+ req_input_ids.append((req_id, input_id))
358
+ batched_mm_inputs = MultiModalKwargs.batch(mm_inputs)
359
+ batched_mm_inputs = MultiModalKwargs.as_kwargs(batched_mm_inputs,
360
+ device=self.device)
361
+
362
+ # Run the encoder.
363
+ # `encoder_outputs` is either of the following:
364
+ # 1. A tensor of shape [num_images, feature_size, hidden_size]
365
+ # in case when feature_size is fixed across all images.
366
+ # 2. A list (length: num_images) of tensors, each of shape
367
+ # [feature_size, hidden_size] in case when the feature size is
368
+ # dynamic depending on input images.
369
+ encoder_outputs = self.model.process_mm_inputs(**batched_mm_inputs)
370
+
371
+ # Cache the encoder outputs.
372
+ for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):
373
+ if req_id not in self.encoder_cache:
374
+ self.encoder_cache[req_id] = {}
375
+ self.encoder_cache[req_id][input_id] = output
376
+
377
+ def _gather_encoder_outputs(
378
+ self,
379
+ scheduler_output: "SchedulerOutput",
380
+ ) -> List[torch.Tensor]:
381
+ encoder_outputs: List[torch.Tensor] = []
382
+ num_reqs = self.input_batch.num_reqs
383
+ for req_id in self.input_batch.req_ids[:num_reqs]:
384
+ num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
385
+ req_id]
386
+ req_state = self.requests[req_id]
387
+ num_computed_tokens = req_state.num_computed_tokens
388
+ mm_positions = req_state.mm_positions
389
+ for i, pos_info in enumerate(mm_positions):
390
+ start_pos = pos_info["offset"]
391
+ num_encoder_tokens = pos_info["length"]
392
+
393
+ # The encoder output is needed if the two ranges overlap:
394
+ # [num_computed_tokens,
395
+ # num_computed_tokens + num_scheduled_tokens) and
396
+ # [start_pos, start_pos + num_encoder_tokens)
397
+ if start_pos >= num_computed_tokens + num_scheduled_tokens:
398
+ # The encoder output is not needed in this step.
399
+ break
400
+ if start_pos + num_encoder_tokens <= num_computed_tokens:
401
+ # The encoder output is already processed and stored
402
+ # in the decoder's KV cache.
403
+ continue
404
+
405
+ start_idx = max(num_computed_tokens - start_pos, 0)
406
+ end_idx = min(
407
+ num_computed_tokens - start_pos + num_scheduled_tokens,
408
+ num_encoder_tokens)
409
+ assert start_idx < end_idx
410
+ assert req_id in self.encoder_cache
411
+ assert i in self.encoder_cache[req_id]
412
+ encoder_output = self.encoder_cache[req_id][i]
413
+ encoder_outputs.append(encoder_output[start_idx:end_idx])
414
+ return encoder_outputs
415
+
416
@torch.inference_mode()
417
def execute_model(
418
self,
419
scheduler_output: "SchedulerOutput",
420
) -> ModelRunnerOutput:
421
self._update_states(scheduler_output)
334
- attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
422
+
423
+ # Run the encoder.
424
+ self._execute_encoder(scheduler_output)
425
+ encoder_outputs = self._gather_encoder_outputs(scheduler_output)
426
+
427
+ # Prepare the decoder inputs.
428
+ input_ids, attn_metadata, logits_indices = self._prepare_inputs(
429
+ scheduler_output)
430
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
431
if (self.use_cuda_graph
432
and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
@@ -343,12 +438,26 @@ class GPUModelRunner:
438
# Eager mode.
439
num_input_tokens = num_scheduled_tokens
440
441
+ # Get the inputs embeds.
442
+ if encoder_outputs:
443
+ inputs_embeds = self.model.get_input_embeddings(
444
+ input_ids, encoder_outputs)
445
+ else:
446
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
447
+ # NOTE(woosuk): To unify token ids and soft tokens (vision embeddings),
448
+ # always use embeddings (rather than token ids) as input to the model.
449
+ # TODO(woosuk): Avoid the copy. Optimize.
450
+ self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
451
+
452
+ # Run the decoder.
453
+ # Use persistent buffers for CUDA graphs.
454
with set_forward_context(attn_metadata):
455
hidden_states = self.model(
348
- input_ids=self.input_ids[:num_input_tokens],
456
+ input_ids=None,
457
positions=self.positions[:num_input_tokens],
458
kv_caches=self.kv_caches,
459
attn_metadata=None,
460
+ inputs_embeds=self.inputs_embeds[:num_input_tokens],
461
)
462
hidden_states = hidden_states[:num_scheduled_tokens]
463
hidden_states = hidden_states[logits_indices]
@@ -440,13 +549,16 @@ class GPUModelRunner:
549
with set_forward_context(None): # noqa: SIM117
550
with set_compile_context(self.cudagraph_batch_sizes):
551
# Trigger compilation for general shape.
443
- model(self.input_ids,
444
- self.positions,
445
- dummy_kv_caches,
446
- attn_metadata=None)
552
+ model(input_ids=None,
553
+ positions=self.positions,
554
+ kv_caches=dummy_kv_caches,
555
+ attn_metadata=None,
556
+ inputs_embeds=self.inputs_embeds)
557
558
@torch.inference_mode()
559
def profile_run(self) -> None:
560
+ # TODO(woosuk): Profile the max memory usage of the encoder and
561
+ # the encoder cache.
562
self._dummy_run(self.model, self.max_num_tokens)
563
torch.cuda.synchronize()
564
@@ -468,10 +580,11 @@ class GPUModelRunner:
580
# can reuse the memory pool allocated for the large shapes.
581
for num_tokens in reversed(self.cudagraph_batch_sizes):
582
self.model(
471
- self.input_ids[:num_tokens],
472
- self.positions[:num_tokens],
583
+ input_ids=None,
584
+ positions=self.positions[:num_tokens],
585
kv_caches=self.kv_caches,
586
attn_metadata=None,
587
+ inputs_embeds=self.inputs_embeds[:num_tokens],
588
)
589
590
end_time = time.perf_counter()
@@ -506,7 +619,8 @@ class CachedRequestState:
619
req_id: str
620
prompt_token_ids: List[int]
621
prompt: Optional[str]
509
- multi_modal_data: Optional["MultiModalDataDict"]
622
+ mm_inputs: List[MultiModalKwargs]
623
+ mm_positions: List["PlaceholderRange"]
624
sampling_params: SamplingParams
625
generator: Optional[torch.Generator]
626