@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24700c346b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1] Cache `uses_mrope` in GPUModelRunner (#12969)

Woosuk Kwon committed Feb 8, 2025 at 15:32 UTC 24700c346bee5760f015bf41cdc6fd9ffb5d6aaf
1 file changed +13 -10
vllm/v1/worker/gpu_model_runner.py
+13 -10
@@ -92,6 +92,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
92 # Multi-modal data support
93 self.input_registry = INPUT_REGISTRY
94 self.mm_registry = MULTIMODAL_REGISTRY
95 + self.uses_mrope = model_config.uses_mrope
96
97 # NOTE: Initialized input mapper is only used for processing dummy
98 # multimodal data into multimodal kwargs for GPU memory profiling.
@@ -147,7 +148,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
148 device=self.device)
149
150 # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
150 - if self.model_config.uses_mrope:
151 + if self.uses_mrope:
152 # NOTE: `mrope_positions` is implemented with one additional dummy
153 # position on purpose to make it non-contiguous so that it can work
154 # with torch compile.
@@ -284,7 +285,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
285 )
286
287 # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
287 - if self.model_config.uses_mrope:
288 + if self.uses_mrope:
289 image_grid_thw = []
290 video_grid_thw = []
291 second_per_grid_ts = []
@@ -411,7 +412,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
412
413 # Calculate M-RoPE positions.
414 # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
414 - if self.model_config.uses_mrope:
415 + if self.uses_mrope:
416 self._calc_mrope_positions(scheduler_output)
417
418 # Get token indices.
@@ -458,7 +459,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
459 # Copy the tensors to the GPU.
460 self.input_ids[:total_num_scheduled_tokens].copy_(
461 self.input_ids_cpu[:total_num_scheduled_tokens], non_blocking=True)
461 - if self.model_config.uses_mrope:
462 + if self.uses_mrope:
463 # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
464 self.mrope_positions[:, :total_num_scheduled_tokens].copy_(
465 self.mrope_positions_cpu[:, :total_num_scheduled_tokens],
@@ -817,13 +818,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
818 # then the embedding layer is not included in the CUDA graph.
819 input_ids = self.input_ids[:num_input_tokens]
820 inputs_embeds = None
821 + if self.uses_mrope:
822 + positions = self.mrope_positions[:, :num_input_tokens]
823 + else:
824 + positions = self.positions[:num_input_tokens]
825
826 # Run the decoder.
827 # Use persistent buffers for CUDA graphs.
828 with set_forward_context(attn_metadata, self.vllm_config):
824 - positions = self.mrope_positions[:, :num_input_tokens] \
825 - if self.model_config.uses_mrope \
826 - else self.positions[:num_input_tokens]
829 hidden_states = self.model(
830 input_ids=input_ids,
831 positions=positions,
@@ -1001,10 +1003,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1003 else:
1004 input_ids = self.input_ids[:num_tokens]
1005 inputs_embeds = None
1006 + if self.uses_mrope:
1007 + positions = self.mrope_positions[:, :num_tokens]
1008 + else:
1009 + positions = self.positions[:num_tokens]
1010 with set_forward_context(None, self.vllm_config):
1005 - positions = self.mrope_positions[:, :num_tokens] \
1006 - if self.model_config.uses_mrope \
1007 - else self.positions[:num_tokens]
1011 hidden_states = model(
1012 input_ids=input_ids,
1013 positions=positions,