@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81763c58a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Add V1 support of Qwen2-VL (#12128)

Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: imkero <kerorek@outlook.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Roger Wang committed Jan 19, 2025 at 03:52 UTC 81763c58a01eda9205f3750177358acc79613e65
9 files changed +292 -85
docs/source/models/supported_models.md
+1 -1
@@ -754,7 +754,7 @@ See [this page](#generative-models) for more information on how to use generativ
754 - `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc.
755 - ✅︎
756 - ✅︎
757 - -
757 + - ✅︎
758 * - `UltravoxModel`
759 - Ultravox
760 - T + A<sup>E+</sup>
tests/models/decoder_only/vision_language/test_qwen2_vl.py
+8 -10
@@ -105,7 +105,7 @@ def batch_make_image_embeddings(
105 pixel_values = preprocess_result["pixel_values"]
106 image_grid_thw = preprocess_result["image_grid_thw"]
107
108 - # pixel values to embeddinds & grid_thws
108 + # pixel values to embeddings & grid_thws
109 with torch.no_grad():
110 visual = llm.llm_engine.model_executor.driver_worker. \
111 model_runner.model.visual
@@ -124,11 +124,10 @@ def batch_make_image_embeddings(
124 for image_batch in image_batches_:
125 cur_batch_image_count = len(image_batch)
126 merge_size = image_processor.merge_size
127 - cur_batch_embed_len = sum([
128 - grid_thw.prod() // merge_size // merge_size
127 + cur_batch_embed_len = sum(
128 + grid_thw.prod(-1) // merge_size // merge_size
129 for grid_thw in image_grid_thw[image_counter:image_counter +
130 - cur_batch_image_count]
131 - ])
130 + cur_batch_image_count])
131
132 result.append({
133 "image_embeds":
@@ -187,7 +186,7 @@ def batch_make_video_embeddings(
186 pixel_values = preprocess_result["pixel_values_videos"]
187 video_grid_thw = preprocess_result["video_grid_thw"]
188
190 - # pixel values to embeddinds & grid_thws
189 + # pixel values to embeddings & grid_thws
190 with torch.no_grad():
191 visual = llm.llm_engine.model_executor.driver_worker.\
192 model_runner.model.visual
@@ -206,11 +205,10 @@ def batch_make_video_embeddings(
205 for video_batch in video_batches_:
206 cur_batch_video_count = len(video_batch)
207 merge_size = image_processor.merge_size
209 - cur_batch_embed_len = sum([
210 - grid_thw.prod() // merge_size // merge_size
208 + cur_batch_embed_len = sum(
209 + grid_thw.prod(-1) // merge_size // merge_size
210 for grid_thw in video_grid_thw[video_counter:video_counter +
212 - cur_batch_video_count]
213 - ])
211 + cur_batch_video_count])
212
213 result.append({
214 "video_embeds":
vllm/compilation/decorators.py
+12 -2
@@ -76,8 +76,8 @@ def support_torch_compile(
76 During runtime, when we actually mark dimensions of tensors,
77 it depends on the value of arguments:
78
79 - - if it is a single integer, the corresponding dimension of the argument
80 - will be marked as dynamic.
79 + - if it is a single integer (can be negative), the corresponding dimension
80 + of the argument will be marked as dynamic.
81 - if it is `None`, ignored.
82 - if it is `IntermediateTensors`, all the tensors in the intermediate
83 tensors will be marked as dynamic.
@@ -177,10 +177,20 @@ def _support_torch_compile(
177 for k, dims in dynamic_arg_dims.items():
178 arg = bound_args.arguments.get(k)
179 if arg is not None:
180 + dims = [dims] if isinstance(dims, int) else dims
181 if isinstance(arg, torch.Tensor):
182 + # In case dims is specified with negative indexing
183 + dims = [
184 + arg.ndim + dim if dim < 0 else dim for dim in dims
185 + ]
186 torch._dynamo.mark_dynamic(arg, dims)
187 elif isinstance(arg, IntermediateTensors):
188 for tensor in arg.tensors.values():
189 + # In case dims is specified with negative indexing
190 + dims = [
191 + tensor.ndim + dim if dim < 0 else dim
192 + for dim in dims
193 + ]
194 torch._dynamo.mark_dynamic(tensor, dims)
195 else:
196 raise ValueError(
vllm/model_executor/layers/rotary_embedding.py
+43 -1
@@ -841,6 +841,37 @@ class MRotaryEmbedding(RotaryEmbedding):
841 ) -> Tuple[List[List[int]], int]:
842 """Get mrope input positions and delta value."""
843
844 + llm_positions, mrope_position_delta = \
845 + MRotaryEmbedding.get_input_positions_tensor(
846 + input_tokens,
847 + image_grid_thw,
848 + video_grid_thw,
849 + image_token_id,
850 + video_token_id,
851 + vision_start_token_id,
852 + vision_end_token_id,
853 + spatial_merge_size,
854 + context_len,
855 + seq_len,
856 + )
857 +
858 + return llm_positions.tolist(), mrope_position_delta
859 +
860 + @staticmethod
861 + def get_input_positions_tensor(
862 + input_tokens: List[int],
863 + image_grid_thw: Union[List[List[int]], torch.Tensor],
864 + video_grid_thw: Union[List[List[int]], torch.Tensor],
865 + image_token_id: int,
866 + video_token_id: int,
867 + vision_start_token_id: int,
868 + vision_end_token_id: int,
869 + spatial_merge_size: int,
870 + context_len: int = 0,
871 + seq_len: Optional[int] = None,
872 + ) -> Tuple[torch.Tensor, int]:
873 + """Get mrope input positions and delta value."""
874 +
875 if isinstance(image_grid_thw, torch.Tensor):
876 image_grid_thw = image_grid_thw.tolist()
877 if isinstance(video_grid_thw, torch.Tensor):
@@ -916,7 +947,7 @@ class MRotaryEmbedding(RotaryEmbedding):
947 len(input_tokens)).item()
948 llm_positions = llm_positions[:, context_len:seq_len]
949
919 - return llm_positions.tolist(), mrope_position_delta
950 + return llm_positions, mrope_position_delta
951
952 @staticmethod
953 def get_next_input_positions(
@@ -930,6 +961,17 @@ class MRotaryEmbedding(RotaryEmbedding):
961 seq_len + mrope_position_delta)) for _ in range(3)
962 ]
963
964 + @staticmethod
965 + def get_next_input_positions_tensor(
966 + mrope_position_delta: int,
967 + context_len: int,
968 + seq_len: int,
969 + ) -> torch.Tensor:
970 + return torch.arange(
971 + mrope_position_delta + context_len,
972 + mrope_position_delta + seq_len,
973 + ).expand(3, -1)
974 +
975
976 _ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
977
vllm/model_executor/models/llava_onevision.py
+4 -2
@@ -554,10 +554,12 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
554 # Preserve the order of modalities if there are multiple of them
555 # from the order of kwargs.
556 for input_key in kwargs:
557 - if input_key == "pixel_values" and "images" not in modalities:
557 + if input_key in ("pixel_values",
558 + "image_embeds") and "images" not in modalities:
559 modalities["images"] = self._parse_and_validate_image_input(
560 **kwargs)
560 - if input_key == "pixel_values_videos" and "videos" not in modalities: # noqa E501
561 + if input_key in ("pixel_values_videos",
562 + "video_embeds") and "videos" not in modalities:
563 modalities["videos"] = self._parse_and_validate_video_input(
564 **kwargs)
565
vllm/model_executor/models/qwen2.py
+9 -1
@@ -256,7 +256,15 @@ class Qwen2DecoderLayer(nn.Module):
256 return hidden_states, residual
257
258
259 -@support_torch_compile
259 +@support_torch_compile(
260 + dynamic_arg_dims={
261 + "input_ids": 0,
262 + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
263 + # otherwise (seq_len, ).
264 + "positions": -1,
265 + "intermediate_tensors": 0,
266 + "inputs_embeds": 0,
267 + })
268 class Qwen2Model(nn.Module):
269
270 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/qwen2_vl.py
+78 -64
@@ -67,11 +67,15 @@ from vllm.transformers_utils.config import uses_mrope
67
68 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69 from .utils import (AutoWeightsLoader, WeightsMapper,
70 - init_vllm_registered_model, maybe_prefix)
70 + init_vllm_registered_model, maybe_prefix,
71 + merge_multimodal_embeddings)
72 from .vision import get_vit_attn_backend
73
74 logger = init_logger(__name__)
75
76 +# For profile run
77 +_MAX_FRAMES_PER_VIDEO = 16
78 +
79 # === Vision Inputs === #
80
81
@@ -135,7 +139,7 @@ class Qwen2VLVideoEmbeddingInputs(TypedDict):
139 - List[`torch.Tensor`]: A list of tensors holding all videos' features.
140 Each tensor holds an video's features.
141 - `torch.Tensor`: A tensor holding all videos' features
138 - (concatenation of all videos' feature tensors).
142 + (concatenation of all videos' feature tensors).
143
144 Tensor shape: `(num_image_features, hidden_size)`
145 - `num_image_features` varies based on
@@ -611,6 +615,7 @@ class Qwen2VisionTransformer(nn.Module):
615
616 # adapter
617 x = self.merger(x)
618 +
619 return x
620
621 def load_weights(self, weights: Iterable[Tuple[str,
@@ -874,8 +879,8 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
879 max_image_tokens = self.get_max_image_tokens() * max_images
880 max_total_frames = self._get_max_video_frames(seq_len -
881 max_image_tokens)
877 -
878 - num_frames = max(max_total_frames // max(max_videos, 1), 1)
882 + num_frames = min(max(max_total_frames // max(max_videos, 1), 1),
883 + _MAX_FRAMES_PER_VIDEO)
884
885 # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
886 if num_frames > 1 and num_frames % 2 == 1:
@@ -955,13 +960,14 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
960 "image": hf_processor.image_token,
961 "video": hf_processor.video_token,
962 }
963 +
964 merge_length = image_processor.merge_size**2
965
966 def get_replacement_qwen2vl(item_idx: int, modality: str):
967 grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
968 assert isinstance(grid_thw, torch.Tensor)
969
964 - num_tokens = grid_thw.prod() // merge_length
970 + num_tokens = grid_thw.prod().item() // merge_length
971 return placeholder[modality] * num_tokens
972
973 return [
@@ -1047,11 +1053,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1053 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1054 super().__init__()
1055 config: Qwen2VLConfig = vllm_config.model_config.hf_config
1050 - cache_config = vllm_config.cache_config
1056 quant_config = vllm_config.quant_config
1057 multimodal_config = vllm_config.model_config.multimodal_config
1053 - assert not cache_config.enable_prefix_caching, \
1054 - "Qwen2-VL currently does not support prefix caching"
1058
1059 self.config = config
1060 self.multimodal_config = multimodal_config
@@ -1173,59 +1176,82 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1176 video_embeds=video_embeds,
1177 video_grid_thw=video_grid_thw)
1178
1176 - def _process_image_input(self,
1177 - image_input: Qwen2VLImageInputs) -> torch.Tensor:
1179 + def _process_image_input(
1180 + self, image_input: Qwen2VLImageInputs) -> tuple[torch.Tensor, ...]:
1181 +
1182 + grid_thw = image_input["image_grid_thw"]
1183 + assert grid_thw.ndim == 2
1184 +
1185 if image_input["type"] == "image_embeds":
1179 - return image_input["image_embeds"].type(self.visual.dtype)
1186 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)
1187 + else:
1188 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1189 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
1190 +
1191 + # Split concatenated embeddings for each image item.
1192 + merge_size = self.visual.spatial_merge_size
1193 + sizes = grid_thw.prod(-1) // merge_size // merge_size
1194 +
1195 + return image_embeds.split(sizes.tolist())
1196 +
1197 + def _process_video_input(
1198 + self, video_input: Qwen2VLVideoInputs) -> tuple[torch.Tensor, ...]:
1199
1181 - pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1182 - image_embeds = self.visual(pixel_values,
1183 - grid_thw=image_input["image_grid_thw"])
1184 - return image_embeds
1200 + grid_thw = video_input["video_grid_thw"]
1201 + assert grid_thw.ndim == 2
1202
1186 - def _process_video_input(self,
1187 - video_input: Qwen2VLVideoInputs) -> torch.Tensor:
1203 if video_input["type"] == "video_embeds":
1189 - return video_input["video_embeds"].type(self.visual.dtype)
1204 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)
1205 + else:
1206 + pixel_values_videos = video_input["pixel_values_videos"].type(
1207 + self.visual.dtype)
1208 + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
1209
1191 - pixel_values_videos = video_input["pixel_values_videos"].type(
1192 - self.visual.dtype)
1193 - video_embeds = self.visual(pixel_values_videos,
1194 - grid_thw=video_input["video_grid_thw"])
1195 - return video_embeds
1210 + # Split concatenated embeddings for each video item.
1211 + merge_size = self.visual.spatial_merge_size
1212 + sizes = grid_thw.prod(-1) // merge_size // merge_size
1213
1197 - def _merge_multimodal_embeddings(
1198 - self,
1199 - input_ids: torch.Tensor,
1200 - inputs_embeds: torch.Tensor,
1201 - multimodal_embeddings: torch.Tensor,
1202 - placeholder_token_id: int,
1203 - ) -> torch.Tensor:
1204 - mask = (input_ids == placeholder_token_id)
1205 - inputs_embeds[mask, :] = multimodal_embeddings
1206 - return inputs_embeds
1214 + return video_embeds.split(sizes.tolist())
1215 +
1216 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1217 + modalities = {}
1218 +
1219 + # Preserve the order of modalities if there are multiple of them
1220 + # from the order of kwargs.
1221 + for input_key in kwargs:
1222 + if input_key in ("pixel_values",
1223 + "image_embeds") and "images" not in modalities:
1224 + modalities["images"] = self._parse_and_validate_image_input(
1225 + **kwargs)
1226 + if input_key in ("pixel_values_videos",
1227 + "video_embeds") and "videos" not in modalities:
1228 + modalities["videos"] = self._parse_and_validate_video_input(
1229 + **kwargs)
1230 +
1231 + return modalities
1232
1233 def get_multimodal_embeddings(
1234 self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
1235
1211 - image_input = self._parse_and_validate_image_input(**kwargs)
1212 - video_input = self._parse_and_validate_video_input(**kwargs)
1213 - if image_input is None and video_input is None:
1236 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1237 + if not modalities:
1238 return None
1239
1216 - # We make a tuple of each embedding with its modality string. This is a
1217 - # temporary workaround for models to handle mixed modalities when
1218 - # get_multimodal_embeddings and get_input_embeddings are called
1219 - # separately.
1220 - # TODO(ywang96): Add support for mixed-modality inference for v1.
1221 - multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
1222 -
1223 - if image_input is not None:
1224 - image_embeds = self._process_image_input(image_input)
1225 - multimodal_embeddings.append((image_embeds, "image"))
1226 - if video_input is not None:
1227 - video_embeds = self._process_video_input(video_input)
1228 - multimodal_embeddings.append((video_embeds, "video"))
1240 + # The result multimodal_embeddings is tuple of tensors, with each
1241 + # tensor correspoending to a multimodal data item (image or video).
1242 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1243 +
1244 + # NOTE: It is important to iterate over the keys in this dictionary
1245 + # to preserve the order of the modalities.
1246 + for modality in modalities:
1247 + if modality == "images":
1248 + image_input = modalities["images"]
1249 + vision_embeddings = self._process_image_input(image_input)
1250 + multimodal_embeddings += vision_embeddings
1251 + if modality == "videos":
1252 + video_input = modalities["videos"]
1253 + video_embeddings = self._process_video_input(video_input)
1254 + multimodal_embeddings += video_embeddings
1255
1256 return multimodal_embeddings
1257
@@ -1237,21 +1263,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1263 ) -> torch.Tensor:
1264 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1265 if multimodal_embeddings is not None:
1240 - for embeddings, modality in multimodal_embeddings:
1241 - if modality == "image":
1242 - inputs_embeds = self._merge_multimodal_embeddings(
1243 - input_ids,
1244 - inputs_embeds,
1245 - embeddings,
1246 - placeholder_token_id=self.config.image_token_id,
1247 - )
1248 - if modality == "video":
1249 - inputs_embeds = self._merge_multimodal_embeddings(
1250 - input_ids,
1251 - inputs_embeds,
1252 - embeddings,
1253 - placeholder_token_id=self.config.video_token_id,
1254 - )
1266 + inputs_embeds = merge_multimodal_embeddings(
1267 + input_ids, inputs_embeds, multimodal_embeddings,
1268 + [self.config.image_token_id, self.config.video_token_id])
1269 return inputs_embeds
1270
1271 def forward(
vllm/v1/worker/gpu_input_batch.py
+3
@@ -30,6 +30,9 @@ class CachedRequestState:
30 num_computed_tokens: int
31 output_token_ids: List[int]
32
33 + mrope_positions: Optional[torch.Tensor] = None
34 + mrope_position_delta: Optional[int] = None
35 +
36 @property
37 def num_tokens(self) -> int:
38 return len(self.prompt_token_ids) + len(self.output_token_ids)
vllm/v1/worker/gpu_model_runner.py
+134 -4
@@ -14,6 +14,7 @@ from vllm.distributed.parallel_state import graph_capture
14 from vllm.forward_context import set_forward_context
15 from vllm.inputs import INPUT_REGISTRY
16 from vllm.logger import init_logger
17 +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
18 from vllm.model_executor.model_loader import get_model
19 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
20 from vllm.sampling_params import SamplingType
@@ -139,6 +140,32 @@ class GPUModelRunner:
140 self.positions = torch.zeros(self.max_num_tokens,
141 dtype=torch.int64,
142 device=self.device)
143 +
144 + # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
145 + if self.model_config.uses_mrope:
146 + # NOTE: `mrope_positions` is implemented as a permuted tensor to
147 + # satisfy the following properties to allow `torch.compile` to work
148 + # properly:
149 + # - shape: (3, <variable>)
150 + # - stride: (1, 3)
151 + # See detailed explanation in https://github.com/vllm-project/vllm/pull/12128#discussion_r1921022256
152 +
153 + # NOTE: When M-RoPE is enabled, position ids are 3D regardless of
154 + # the modality of inputs. For text-only inputs, each dimension has
155 + # identical position IDs, making M-RoPE functionally equivalent to
156 + # 1D-RoPE.
157 + # See page 5 of https://arxiv.org/abs/2409.12191
158 + self.mrope_positions = torch.zeros((self.max_num_tokens, 3),
159 + dtype=torch.int64,
160 + device=self.device)
161 + self.mrope_positions_cpu = torch.zeros((self.max_num_tokens, 3),
162 + dtype=torch.int64,
163 + device="cpu",
164 + pin_memory=self.pin_memory)
165 +
166 + self.mrope_positions = self.mrope_positions.permute((1, 0))
167 + self.mrope_positions_cpu = self.mrope_positions_cpu.permute((1, 0))
168 +
169 self.inputs_embeds = torch.zeros(
170 (self.max_num_tokens, self.hidden_size),
171 dtype=self.dtype,
@@ -246,6 +273,35 @@ class GPUModelRunner:
273 num_computed_tokens=new_req_data.num_computed_tokens,
274 output_token_ids=[],
275 )
276 +
277 + # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
278 + if self.model_config.uses_mrope:
279 + image_grid_thw = []
280 + video_grid_thw = []
281 + for mm_input in self.requests[req_id].mm_inputs:
282 + if mm_input.get("image_grid_thw") is not None:
283 + image_grid_thw.extend(
284 + mm_input["image_grid_thw"].tolist())
285 + if mm_input.get("video_grid_thw") is not None:
286 + video_grid_thw.extend(
287 + mm_input["video_grid_thw"].tolist())
288 +
289 + hf_config = self.model_config.hf_config
290 +
291 + self.requests[req_id].mrope_positions, \
292 + self.requests[req_id].mrope_position_delta = \
293 + MRotaryEmbedding.get_input_positions_tensor(
294 + self.requests[req_id].prompt_token_ids,
295 + image_grid_thw=image_grid_thw,
296 + video_grid_thw=video_grid_thw,
297 + image_token_id=hf_config.image_token_id,
298 + video_token_id=hf_config.video_token_id,
299 + vision_start_token_id=hf_config.vision_start_token_id,
300 + vision_end_token_id=hf_config.vision_end_token_id,
301 + spatial_merge_size=hf_config.vision_config.
302 + spatial_merge_size,
303 + )
304 +
305 req_ids_to_add.append(req_id)
306
307 # Update the cached states of the resumed requests.
@@ -313,6 +369,11 @@ class GPUModelRunner:
369 arange,
370 out=positions_np)
371
372 + # Calculate M-RoPE positions.
373 + # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
374 + if self.model_config.uses_mrope:
375 + self._calc_mrope_positions(scheduler_output)
376 +
377 # Get token indices.
378 # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
379 # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
@@ -359,8 +420,16 @@ class GPUModelRunner:
420 # Copy the tensors to the GPU.
421 self.input_ids[:total_num_scheduled_tokens].copy_(
422 self.input_ids_cpu[:total_num_scheduled_tokens], non_blocking=True)
362 - self.positions[:total_num_scheduled_tokens].copy_(
363 - self.positions_cpu[:total_num_scheduled_tokens], non_blocking=True)
423 + if self.model_config.uses_mrope:
424 + # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
425 + self.mrope_positions[:, :total_num_scheduled_tokens].copy_(
426 + self.mrope_positions_cpu[:, :total_num_scheduled_tokens],
427 + non_blocking=True)
428 + else:
429 + # Common case (1D positions)
430 + self.positions[:total_num_scheduled_tokens].copy_(
431 + self.positions_cpu[:total_num_scheduled_tokens],
432 + non_blocking=True)
433 query_start_loc = self.query_start_loc_cpu[:num_reqs + 1].to(
434 self.device, non_blocking=True)
435 seq_start_loc = self.seq_start_loc_cpu[:num_reqs + 1].to(
@@ -472,6 +541,61 @@ class GPUModelRunner:
541 logits_indices = query_start_loc[1:] - 1
542 return attn_metadata, logits_indices
543
544 + def _calc_mrope_positions(self, scheduler_output: "SchedulerOutput"):
545 + mrope_pos_ptr = 0
546 + num_reqs = self.input_batch.num_reqs
547 + for index, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
548 + assert req_id is not None
549 +
550 + req = self.requests[req_id]
551 + assert req.mrope_positions is not None
552 +
553 + num_computed_tokens = \
554 + self.input_batch.num_computed_tokens_cpu[index]
555 + num_scheduled_tokens = \
556 + scheduler_output.num_scheduled_tokens[req_id]
557 + num_prompt_tokens = len(req.prompt_token_ids)
558 +
559 + if num_computed_tokens + num_scheduled_tokens > num_prompt_tokens:
560 + prompt_part_len = max(0,
561 + num_prompt_tokens - num_computed_tokens)
562 + completion_part_len = max(
563 + 0, num_scheduled_tokens - prompt_part_len)
564 + else:
565 + prompt_part_len = num_scheduled_tokens
566 + completion_part_len = 0
567 +
568 + assert num_scheduled_tokens == prompt_part_len + completion_part_len
569 +
570 + if prompt_part_len > 0:
571 + # prompt's mrope_positions are pre-computed
572 + dst_start = mrope_pos_ptr
573 + dst_end = mrope_pos_ptr + prompt_part_len
574 + src_start = num_computed_tokens
575 + src_end = num_computed_tokens + prompt_part_len
576 +
577 + self.mrope_positions_cpu[:, dst_start:dst_end] = \
578 + req.mrope_positions[:,src_start:src_end]
579 +
580 + mrope_pos_ptr += prompt_part_len
581 +
582 + if completion_part_len > 0:
583 + # compute completion's mrope_positions on-the-fly
584 + dst_start = mrope_pos_ptr
585 + dst_end = mrope_pos_ptr + completion_part_len
586 +
587 + self.mrope_positions_cpu[:, dst_start:dst_end] = \
588 + MRotaryEmbedding.get_next_input_positions_tensor(
589 + req.mrope_position_delta,
590 + context_len=num_computed_tokens +
591 + prompt_part_len,
592 + seq_len=num_computed_tokens +
593 + prompt_part_len +
594 + completion_part_len,
595 + )
596 +
597 + mrope_pos_ptr += completion_part_len
598 +
599 def _prepare_sampling(
600 self,
601 scheduler_output: "SchedulerOutput",
@@ -618,9 +742,12 @@ class GPUModelRunner:
742 # Run the decoder.
743 # Use persistent buffers for CUDA graphs.
744 with set_forward_context(attn_metadata, self.vllm_config):
745 + positions = self.mrope_positions[:, :num_input_tokens] \
746 + if self.model_config.uses_mrope \
747 + else self.positions[:num_input_tokens]
748 hidden_states = self.model(
749 input_ids=input_ids,
623 - positions=self.positions[:num_input_tokens],
750 + positions=positions,
751 kv_caches=self.kv_caches,
752 attn_metadata=None,
753 inputs_embeds=inputs_embeds,
@@ -707,9 +834,12 @@ class GPUModelRunner:
834 input_ids = self.input_ids[:num_tokens]
835 inputs_embeds = None
836 with set_forward_context(None, self.vllm_config):
837 + positions = self.mrope_positions[:, :num_tokens] \
838 + if self.model_config.uses_mrope \
839 + else self.positions[:num_tokens]
840 hidden_states = model(
841 input_ids=input_ids,
712 - positions=self.positions[:num_tokens],
842 + positions=positions,
843 kv_caches=kv_caches,
844 attn_metadata=None,
845 inputs_embeds=inputs_embeds,