A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1] Add V1 support of Qwen2-VL (#12128)
Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: imkero <kerorek@outlook.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Roger Wang committed
Jan 19, 2025 at 03:52 UTC
81763c58a01eda9205f3750177358acc79613e65
9 files changed
+292
-85
docs/source/models/supported_models.md
+1
-1
@@ -754,7 +754,7 @@ See [this page](#generative-models) for more information on how to use generativ
754
- `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc.
755
- ✅︎
756
- ✅︎
757
- -
757
+ - ✅︎
758
* - `UltravoxModel`
759
- Ultravox
760
- T + A<sup>E+</sup>
tests/models/decoder_only/vision_language/test_qwen2_vl.py
+8
-10
@@ -105,7 +105,7 @@ def batch_make_image_embeddings(
105
pixel_values = preprocess_result["pixel_values"]
106
image_grid_thw = preprocess_result["image_grid_thw"]
107
108
- # pixel values to embeddinds & grid_thws
108
+ # pixel values to embeddings & grid_thws
109
with torch.no_grad():
110
visual = llm.llm_engine.model_executor.driver_worker. \
111
model_runner.model.visual
@@ -124,11 +124,10 @@ def batch_make_image_embeddings(
124
for image_batch in image_batches_:
125
cur_batch_image_count = len(image_batch)
126
merge_size = image_processor.merge_size
127
- cur_batch_embed_len = sum([
128
- grid_thw.prod() // merge_size // merge_size
127
+ cur_batch_embed_len = sum(
128
+ grid_thw.prod(-1) // merge_size // merge_size
129
for grid_thw in image_grid_thw[image_counter:image_counter +
130
- cur_batch_image_count]
131
- ])
130
+ cur_batch_image_count])
131
132
result.append({
133
"image_embeds":
@@ -187,7 +186,7 @@ def batch_make_video_embeddings(
186
pixel_values = preprocess_result["pixel_values_videos"]
187
video_grid_thw = preprocess_result["video_grid_thw"]
188
190
- # pixel values to embeddinds & grid_thws
189
+ # pixel values to embeddings & grid_thws
190
with torch.no_grad():
191
visual = llm.llm_engine.model_executor.driver_worker.\
192
model_runner.model.visual
@@ -206,11 +205,10 @@ def batch_make_video_embeddings(
205
for video_batch in video_batches_:
206
cur_batch_video_count = len(video_batch)
207
merge_size = image_processor.merge_size
209
- cur_batch_embed_len = sum([
210
- grid_thw.prod() // merge_size // merge_size
208
+ cur_batch_embed_len = sum(
209
+ grid_thw.prod(-1) // merge_size // merge_size
210
for grid_thw in video_grid_thw[video_counter:video_counter +
212
- cur_batch_video_count]
213
- ])
211
+ cur_batch_video_count])
212
213
result.append({
214
"video_embeds":
vllm/compilation/decorators.py
+12
-2
@@ -76,8 +76,8 @@ def support_torch_compile(
76
During runtime, when we actually mark dimensions of tensors,
77
it depends on the value of arguments:
78
79
- - if it is a single integer, the corresponding dimension of the argument
80
- will be marked as dynamic.
79
+ - if it is a single integer (can be negative), the corresponding dimension
80
+ of the argument will be marked as dynamic.
81
- if it is `None`, ignored.
82
- if it is `IntermediateTensors`, all the tensors in the intermediate
83
tensors will be marked as dynamic.
@@ -177,10 +177,20 @@ def _support_torch_compile(
177
for k, dims in dynamic_arg_dims.items():
178
arg = bound_args.arguments.get(k)
179
if arg is not None:
180
+ dims = [dims] if isinstance(dims, int) else dims
181
if isinstance(arg, torch.Tensor):
182
+ # In case dims is specified with negative indexing
183
+ dims = [
184
+ arg.ndim + dim if dim < 0 else dim for dim in dims
185
+ ]
186
torch._dynamo.mark_dynamic(arg, dims)
187
elif isinstance(arg, IntermediateTensors):
188
for tensor in arg.tensors.values():
189
+ # In case dims is specified with negative indexing
190
+ dims = [
191
+ tensor.ndim + dim if dim < 0 else dim
192
+ for dim in dims
193
+ ]
194
torch._dynamo.mark_dynamic(tensor, dims)
195
else:
196
raise ValueError(
vllm/model_executor/layers/rotary_embedding.py
+43
-1
@@ -841,6 +841,37 @@ class MRotaryEmbedding(RotaryEmbedding):
841
) -> Tuple[List[List[int]], int]:
842
"""Get mrope input positions and delta value."""
843
844
+ llm_positions, mrope_position_delta = \
845
+ MRotaryEmbedding.get_input_positions_tensor(
846
+ input_tokens,
847
+ image_grid_thw,
848
+ video_grid_thw,
849
+ image_token_id,
850
+ video_token_id,
851
+ vision_start_token_id,
852
+ vision_end_token_id,
853
+ spatial_merge_size,
854
+ context_len,
855
+ seq_len,
856
+ )
857
+
858
+ return llm_positions.tolist(), mrope_position_delta
859
+
860
+ @staticmethod
861
+ def get_input_positions_tensor(
862
+ input_tokens: List[int],
863
+ image_grid_thw: Union[List[List[int]], torch.Tensor],
864
+ video_grid_thw: Union[List[List[int]], torch.Tensor],
865
+ image_token_id: int,
866
+ video_token_id: int,
867
+ vision_start_token_id: int,
868
+ vision_end_token_id: int,
869
+ spatial_merge_size: int,
870
+ context_len: int = 0,
871
+ seq_len: Optional[int] = None,
872
+ ) -> Tuple[torch.Tensor, int]:
873
+ """Get mrope input positions and delta value."""
874
+
875
if isinstance(image_grid_thw, torch.Tensor):
876
image_grid_thw = image_grid_thw.tolist()
877
if isinstance(video_grid_thw, torch.Tensor):
@@ -916,7 +947,7 @@ class MRotaryEmbedding(RotaryEmbedding):
947
len(input_tokens)).item()
948
llm_positions = llm_positions[:, context_len:seq_len]
949
919
- return llm_positions.tolist(), mrope_position_delta
950
+ return llm_positions, mrope_position_delta
951
952
@staticmethod
953
def get_next_input_positions(
@@ -930,6 +961,17 @@ class MRotaryEmbedding(RotaryEmbedding):
961
seq_len + mrope_position_delta)) for _ in range(3)
962
]
963
964
+ @staticmethod
965
+ def get_next_input_positions_tensor(
966
+ mrope_position_delta: int,
967
+ context_len: int,
968
+ seq_len: int,
969
+ ) -> torch.Tensor:
970
+ return torch.arange(
971
+ mrope_position_delta + context_len,
972
+ mrope_position_delta + seq_len,
973
+ ).expand(3, -1)
974
+
975
976
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
977
vllm/model_executor/models/llava_onevision.py
+4
-2
@@ -554,10 +554,12 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
554
# Preserve the order of modalities if there are multiple of them
555
# from the order of kwargs.
556
for input_key in kwargs:
557
- if input_key == "pixel_values" and "images" not in modalities:
557
+ if input_key in ("pixel_values",
558
+ "image_embeds") and "images" not in modalities:
559
modalities["images"] = self._parse_and_validate_image_input(
560
**kwargs)
560
- if input_key == "pixel_values_videos" and "videos" not in modalities: # noqa E501
561
+ if input_key in ("pixel_values_videos",
562
+ "video_embeds") and "videos" not in modalities:
563
modalities["videos"] = self._parse_and_validate_video_input(
564
**kwargs)
565
vllm/model_executor/models/qwen2.py
+9
-1
@@ -256,7 +256,15 @@ class Qwen2DecoderLayer(nn.Module):
256
return hidden_states, residual
257
258
259
-@support_torch_compile
259
+@support_torch_compile(
260
+ dynamic_arg_dims={
261
+ "input_ids": 0,
262
+ # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
263
+ # otherwise (seq_len, ).
264
+ "positions": -1,
265
+ "intermediate_tensors": 0,
266
+ "inputs_embeds": 0,
267
+ })
268
class Qwen2Model(nn.Module):
269
270
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/qwen2_vl.py
+78
-64
@@ -67,11 +67,15 @@ from vllm.transformers_utils.config import uses_mrope
67
68
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69
from .utils import (AutoWeightsLoader, WeightsMapper,
70
- init_vllm_registered_model, maybe_prefix)
70
+ init_vllm_registered_model, maybe_prefix,
71
+ merge_multimodal_embeddings)
72
from .vision import get_vit_attn_backend
73
74
logger = init_logger(__name__)
75
76
+# For profile run
77
+_MAX_FRAMES_PER_VIDEO = 16
78
+
79
# === Vision Inputs === #
80
81
@@ -135,7 +139,7 @@ class Qwen2VLVideoEmbeddingInputs(TypedDict):
139
- List[`torch.Tensor`]: A list of tensors holding all videos' features.
140
Each tensor holds an video's features.
141
- `torch.Tensor`: A tensor holding all videos' features
138
- (concatenation of all videos' feature tensors).
142
+ (concatenation of all videos' feature tensors).
143
144
Tensor shape: `(num_image_features, hidden_size)`
145
- `num_image_features` varies based on
@@ -611,6 +615,7 @@ class Qwen2VisionTransformer(nn.Module):
615
616
# adapter
617
x = self.merger(x)
618
+
619
return x
620
621
def load_weights(self, weights: Iterable[Tuple[str,
@@ -874,8 +879,8 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
879
max_image_tokens = self.get_max_image_tokens() * max_images
880
max_total_frames = self._get_max_video_frames(seq_len -
881
max_image_tokens)
877
-
878
- num_frames = max(max_total_frames // max(max_videos, 1), 1)
882
+ num_frames = min(max(max_total_frames // max(max_videos, 1), 1),
883
+ _MAX_FRAMES_PER_VIDEO)
884
885
# Temporary workaround for https://github.com/huggingface/transformers/issues/35412
886
if num_frames > 1 and num_frames % 2 == 1:
@@ -955,13 +960,14 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
960
"image": hf_processor.image_token,
961
"video": hf_processor.video_token,
962
}
963
+
964
merge_length = image_processor.merge_size**2
965
966
def get_replacement_qwen2vl(item_idx: int, modality: str):
967
grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
968
assert isinstance(grid_thw, torch.Tensor)
969
964
- num_tokens = grid_thw.prod() // merge_length
970
+ num_tokens = grid_thw.prod().item() // merge_length
971
return placeholder[modality] * num_tokens
972
973
return [
@@ -1047,11 +1053,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1053
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1054
super().__init__()
1055
config: Qwen2VLConfig = vllm_config.model_config.hf_config
1050
- cache_config = vllm_config.cache_config
1056
quant_config = vllm_config.quant_config
1057
multimodal_config = vllm_config.model_config.multimodal_config
1053
- assert not cache_config.enable_prefix_caching, \
1054
- "Qwen2-VL currently does not support prefix caching"
1058
1059
self.config = config
1060
self.multimodal_config = multimodal_config
@@ -1173,59 +1176,82 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1176
video_embeds=video_embeds,
1177
video_grid_thw=video_grid_thw)
1178
1176
- def _process_image_input(self,
1177
- image_input: Qwen2VLImageInputs) -> torch.Tensor:
1179
+ def _process_image_input(
1180
+ self, image_input: Qwen2VLImageInputs) -> tuple[torch.Tensor, ...]:
1181
+
1182
+ grid_thw = image_input["image_grid_thw"]
1183
+ assert grid_thw.ndim == 2
1184
+
1185
if image_input["type"] == "image_embeds":
1179
- return image_input["image_embeds"].type(self.visual.dtype)
1186
+ image_embeds = image_input["image_embeds"].type(self.visual.dtype)
1187
+ else:
1188
+ pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1189
+ image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
1190
+
1191
+ # Split concatenated embeddings for each image item.
1192
+ merge_size = self.visual.spatial_merge_size
1193
+ sizes = grid_thw.prod(-1) // merge_size // merge_size
1194
+
1195
+ return image_embeds.split(sizes.tolist())
1196
+
1197
+ def _process_video_input(
1198
+ self, video_input: Qwen2VLVideoInputs) -> tuple[torch.Tensor, ...]:
1199
1181
- pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1182
- image_embeds = self.visual(pixel_values,
1183
- grid_thw=image_input["image_grid_thw"])
1184
- return image_embeds
1200
+ grid_thw = video_input["video_grid_thw"]
1201
+ assert grid_thw.ndim == 2
1202
1186
- def _process_video_input(self,
1187
- video_input: Qwen2VLVideoInputs) -> torch.Tensor:
1203
if video_input["type"] == "video_embeds":
1189
- return video_input["video_embeds"].type(self.visual.dtype)
1204
+ video_embeds = video_input["video_embeds"].type(self.visual.dtype)
1205
+ else:
1206
+ pixel_values_videos = video_input["pixel_values_videos"].type(
1207
+ self.visual.dtype)
1208
+ video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
1209
1191
- pixel_values_videos = video_input["pixel_values_videos"].type(
1192
- self.visual.dtype)
1193
- video_embeds = self.visual(pixel_values_videos,
1194
- grid_thw=video_input["video_grid_thw"])
1195
- return video_embeds
1210
+ # Split concatenated embeddings for each video item.
1211
+ merge_size = self.visual.spatial_merge_size
1212
+ sizes = grid_thw.prod(-1) // merge_size // merge_size
1213
1197
- def _merge_multimodal_embeddings(
1198
- self,
1199
- input_ids: torch.Tensor,
1200
- inputs_embeds: torch.Tensor,
1201
- multimodal_embeddings: torch.Tensor,
1202
- placeholder_token_id: int,
1203
- ) -> torch.Tensor:
1204
- mask = (input_ids == placeholder_token_id)
1205
- inputs_embeds[mask, :] = multimodal_embeddings
1206
- return inputs_embeds
1214
+ return video_embeds.split(sizes.tolist())
1215
+
1216
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1217
+ modalities = {}
1218
+
1219
+ # Preserve the order of modalities if there are multiple of them
1220
+ # from the order of kwargs.
1221
+ for input_key in kwargs:
1222
+ if input_key in ("pixel_values",
1223
+ "image_embeds") and "images" not in modalities:
1224
+ modalities["images"] = self._parse_and_validate_image_input(
1225
+ **kwargs)
1226
+ if input_key in ("pixel_values_videos",
1227
+ "video_embeds") and "videos" not in modalities:
1228
+ modalities["videos"] = self._parse_and_validate_video_input(
1229
+ **kwargs)
1230
+
1231
+ return modalities
1232
1233
def get_multimodal_embeddings(
1234
self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
1235
1211
- image_input = self._parse_and_validate_image_input(**kwargs)
1212
- video_input = self._parse_and_validate_video_input(**kwargs)
1213
- if image_input is None and video_input is None:
1236
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1237
+ if not modalities:
1238
return None
1239
1216
- # We make a tuple of each embedding with its modality string. This is a
1217
- # temporary workaround for models to handle mixed modalities when
1218
- # get_multimodal_embeddings and get_input_embeddings are called
1219
- # separately.
1220
- # TODO(ywang96): Add support for mixed-modality inference for v1.
1221
- multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
1222
-
1223
- if image_input is not None:
1224
- image_embeds = self._process_image_input(image_input)
1225
- multimodal_embeddings.append((image_embeds, "image"))
1226
- if video_input is not None:
1227
- video_embeds = self._process_video_input(video_input)
1228
- multimodal_embeddings.append((video_embeds, "video"))
1240
+ # The result multimodal_embeddings is tuple of tensors, with each
1241
+ # tensor correspoending to a multimodal data item (image or video).
1242
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1243
+
1244
+ # NOTE: It is important to iterate over the keys in this dictionary
1245
+ # to preserve the order of the modalities.
1246
+ for modality in modalities:
1247
+ if modality == "images":
1248
+ image_input = modalities["images"]
1249
+ vision_embeddings = self._process_image_input(image_input)
1250
+ multimodal_embeddings += vision_embeddings
1251
+ if modality == "videos":
1252
+ video_input = modalities["videos"]
1253
+ video_embeddings = self._process_video_input(video_input)
1254
+ multimodal_embeddings += video_embeddings
1255
1256
return multimodal_embeddings
1257
@@ -1237,21 +1263,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1263
) -> torch.Tensor:
1264
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1265
if multimodal_embeddings is not None:
1240
- for embeddings, modality in multimodal_embeddings:
1241
- if modality == "image":
1242
- inputs_embeds = self._merge_multimodal_embeddings(
1243
- input_ids,
1244
- inputs_embeds,
1245
- embeddings,
1246
- placeholder_token_id=self.config.image_token_id,
1247
- )
1248
- if modality == "video":
1249
- inputs_embeds = self._merge_multimodal_embeddings(
1250
- input_ids,
1251
- inputs_embeds,
1252
- embeddings,
1253
- placeholder_token_id=self.config.video_token_id,
1254
- )
1266
+ inputs_embeds = merge_multimodal_embeddings(
1267
+ input_ids, inputs_embeds, multimodal_embeddings,
1268
+ [self.config.image_token_id, self.config.video_token_id])
1269
return inputs_embeds
1270
1271
def forward(
vllm/v1/worker/gpu_input_batch.py
+3
@@ -30,6 +30,9 @@ class CachedRequestState:
30
num_computed_tokens: int
31
output_token_ids: List[int]
32
33
+ mrope_positions: Optional[torch.Tensor] = None
34
+ mrope_position_delta: Optional[int] = None
35
+
36
@property
37
def num_tokens(self) -> int:
38
return len(self.prompt_token_ids) + len(self.output_token_ids)
vllm/v1/worker/gpu_model_runner.py
+134
-4
@@ -14,6 +14,7 @@ from vllm.distributed.parallel_state import graph_capture
14
from vllm.forward_context import set_forward_context
15
from vllm.inputs import INPUT_REGISTRY
16
from vllm.logger import init_logger
17
+from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
18
from vllm.model_executor.model_loader import get_model
19
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
20
from vllm.sampling_params import SamplingType
@@ -139,6 +140,32 @@ class GPUModelRunner:
140
self.positions = torch.zeros(self.max_num_tokens,
141
dtype=torch.int64,
142
device=self.device)
143
+
144
+ # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
145
+ if self.model_config.uses_mrope:
146
+ # NOTE: `mrope_positions` is implemented as a permuted tensor to
147
+ # satisfy the following properties to allow `torch.compile` to work
148
+ # properly:
149
+ # - shape: (3, <variable>)
150
+ # - stride: (1, 3)
151
+ # See detailed explanation in https://github.com/vllm-project/vllm/pull/12128#discussion_r1921022256
152
+
153
+ # NOTE: When M-RoPE is enabled, position ids are 3D regardless of
154
+ # the modality of inputs. For text-only inputs, each dimension has
155
+ # identical position IDs, making M-RoPE functionally equivalent to
156
+ # 1D-RoPE.
157
+ # See page 5 of https://arxiv.org/abs/2409.12191
158
+ self.mrope_positions = torch.zeros((self.max_num_tokens, 3),
159
+ dtype=torch.int64,
160
+ device=self.device)
161
+ self.mrope_positions_cpu = torch.zeros((self.max_num_tokens, 3),
162
+ dtype=torch.int64,
163
+ device="cpu",
164
+ pin_memory=self.pin_memory)
165
+
166
+ self.mrope_positions = self.mrope_positions.permute((1, 0))
167
+ self.mrope_positions_cpu = self.mrope_positions_cpu.permute((1, 0))
168
+
169
self.inputs_embeds = torch.zeros(
170
(self.max_num_tokens, self.hidden_size),
171
dtype=self.dtype,
@@ -246,6 +273,35 @@ class GPUModelRunner:
273
num_computed_tokens=new_req_data.num_computed_tokens,
274
output_token_ids=[],
275
)
276
+
277
+ # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
278
+ if self.model_config.uses_mrope:
279
+ image_grid_thw = []
280
+ video_grid_thw = []
281
+ for mm_input in self.requests[req_id].mm_inputs:
282
+ if mm_input.get("image_grid_thw") is not None:
283
+ image_grid_thw.extend(
284
+ mm_input["image_grid_thw"].tolist())
285
+ if mm_input.get("video_grid_thw") is not None:
286
+ video_grid_thw.extend(
287
+ mm_input["video_grid_thw"].tolist())
288
+
289
+ hf_config = self.model_config.hf_config
290
+
291
+ self.requests[req_id].mrope_positions, \
292
+ self.requests[req_id].mrope_position_delta = \
293
+ MRotaryEmbedding.get_input_positions_tensor(
294
+ self.requests[req_id].prompt_token_ids,
295
+ image_grid_thw=image_grid_thw,
296
+ video_grid_thw=video_grid_thw,
297
+ image_token_id=hf_config.image_token_id,
298
+ video_token_id=hf_config.video_token_id,
299
+ vision_start_token_id=hf_config.vision_start_token_id,
300
+ vision_end_token_id=hf_config.vision_end_token_id,
301
+ spatial_merge_size=hf_config.vision_config.
302
+ spatial_merge_size,
303
+ )
304
+
305
req_ids_to_add.append(req_id)
306
307
# Update the cached states of the resumed requests.
@@ -313,6 +369,11 @@ class GPUModelRunner:
369
arange,
370
out=positions_np)
371
372
+ # Calculate M-RoPE positions.
373
+ # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
374
+ if self.model_config.uses_mrope:
375
+ self._calc_mrope_positions(scheduler_output)
376
+
377
# Get token indices.
378
# E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
379
# -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
@@ -359,8 +420,16 @@ class GPUModelRunner:
420
# Copy the tensors to the GPU.
421
self.input_ids[:total_num_scheduled_tokens].copy_(
422
self.input_ids_cpu[:total_num_scheduled_tokens], non_blocking=True)
362
- self.positions[:total_num_scheduled_tokens].copy_(
363
- self.positions_cpu[:total_num_scheduled_tokens], non_blocking=True)
423
+ if self.model_config.uses_mrope:
424
+ # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
425
+ self.mrope_positions[:, :total_num_scheduled_tokens].copy_(
426
+ self.mrope_positions_cpu[:, :total_num_scheduled_tokens],
427
+ non_blocking=True)
428
+ else:
429
+ # Common case (1D positions)
430
+ self.positions[:total_num_scheduled_tokens].copy_(
431
+ self.positions_cpu[:total_num_scheduled_tokens],
432
+ non_blocking=True)
433
query_start_loc = self.query_start_loc_cpu[:num_reqs + 1].to(
434
self.device, non_blocking=True)
435
seq_start_loc = self.seq_start_loc_cpu[:num_reqs + 1].to(
@@ -472,6 +541,61 @@ class GPUModelRunner:
541
logits_indices = query_start_loc[1:] - 1
542
return attn_metadata, logits_indices
543
544
+ def _calc_mrope_positions(self, scheduler_output: "SchedulerOutput"):
545
+ mrope_pos_ptr = 0
546
+ num_reqs = self.input_batch.num_reqs
547
+ for index, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
548
+ assert req_id is not None
549
+
550
+ req = self.requests[req_id]
551
+ assert req.mrope_positions is not None
552
+
553
+ num_computed_tokens = \
554
+ self.input_batch.num_computed_tokens_cpu[index]
555
+ num_scheduled_tokens = \
556
+ scheduler_output.num_scheduled_tokens[req_id]
557
+ num_prompt_tokens = len(req.prompt_token_ids)
558
+
559
+ if num_computed_tokens + num_scheduled_tokens > num_prompt_tokens:
560
+ prompt_part_len = max(0,
561
+ num_prompt_tokens - num_computed_tokens)
562
+ completion_part_len = max(
563
+ 0, num_scheduled_tokens - prompt_part_len)
564
+ else:
565
+ prompt_part_len = num_scheduled_tokens
566
+ completion_part_len = 0
567
+
568
+ assert num_scheduled_tokens == prompt_part_len + completion_part_len
569
+
570
+ if prompt_part_len > 0:
571
+ # prompt's mrope_positions are pre-computed
572
+ dst_start = mrope_pos_ptr
573
+ dst_end = mrope_pos_ptr + prompt_part_len
574
+ src_start = num_computed_tokens
575
+ src_end = num_computed_tokens + prompt_part_len
576
+
577
+ self.mrope_positions_cpu[:, dst_start:dst_end] = \
578
+ req.mrope_positions[:,src_start:src_end]
579
+
580
+ mrope_pos_ptr += prompt_part_len
581
+
582
+ if completion_part_len > 0:
583
+ # compute completion's mrope_positions on-the-fly
584
+ dst_start = mrope_pos_ptr
585
+ dst_end = mrope_pos_ptr + completion_part_len
586
+
587
+ self.mrope_positions_cpu[:, dst_start:dst_end] = \
588
+ MRotaryEmbedding.get_next_input_positions_tensor(
589
+ req.mrope_position_delta,
590
+ context_len=num_computed_tokens +
591
+ prompt_part_len,
592
+ seq_len=num_computed_tokens +
593
+ prompt_part_len +
594
+ completion_part_len,
595
+ )
596
+
597
+ mrope_pos_ptr += completion_part_len
598
+
599
def _prepare_sampling(
600
self,
601
scheduler_output: "SchedulerOutput",
@@ -618,9 +742,12 @@ class GPUModelRunner:
742
# Run the decoder.
743
# Use persistent buffers for CUDA graphs.
744
with set_forward_context(attn_metadata, self.vllm_config):
745
+ positions = self.mrope_positions[:, :num_input_tokens] \
746
+ if self.model_config.uses_mrope \
747
+ else self.positions[:num_input_tokens]
748
hidden_states = self.model(
749
input_ids=input_ids,
623
- positions=self.positions[:num_input_tokens],
750
+ positions=positions,
751
kv_caches=self.kv_caches,
752
attn_metadata=None,
753
inputs_embeds=inputs_embeds,
@@ -707,9 +834,12 @@ class GPUModelRunner:
834
input_ids = self.input_ids[:num_tokens]
835
inputs_embeds = None
836
with set_forward_context(None, self.vllm_config):
837
+ positions = self.mrope_positions[:, :num_tokens] \
838
+ if self.model_config.uses_mrope \
839
+ else self.positions[:num_tokens]
840
hidden_states = model(
841
input_ids=input_ids,
712
- positions=self.positions[:num_tokens],
842
+ positions=positions,
843
kv_caches=kv_caches,
844
attn_metadata=None,
845
inputs_embeds=inputs_embeds,