A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1] Cache `uses_mrope` in GPUModelRunner (#12969)
Woosuk Kwon committed
Feb 8, 2025 at 15:32 UTC
24700c346bee5760f015bf41cdc6fd9ffb5d6aaf
1 file changed
+13
-10
vllm/v1/worker/gpu_model_runner.py
+13
-10
@@ -92,6 +92,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
92
# Multi-modal data support
93
self.input_registry = INPUT_REGISTRY
94
self.mm_registry = MULTIMODAL_REGISTRY
95
+ self.uses_mrope = model_config.uses_mrope
96
97
# NOTE: Initialized input mapper is only used for processing dummy
98
# multimodal data into multimodal kwargs for GPU memory profiling.
@@ -147,7 +148,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
148
device=self.device)
149
150
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
150
- if self.model_config.uses_mrope:
151
+ if self.uses_mrope:
152
# NOTE: `mrope_positions` is implemented with one additional dummy
153
# position on purpose to make it non-contiguous so that it can work
154
# with torch compile.
@@ -284,7 +285,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
285
)
286
287
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
287
- if self.model_config.uses_mrope:
288
+ if self.uses_mrope:
289
image_grid_thw = []
290
video_grid_thw = []
291
second_per_grid_ts = []
@@ -411,7 +412,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
412
413
# Calculate M-RoPE positions.
414
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
414
- if self.model_config.uses_mrope:
415
+ if self.uses_mrope:
416
self._calc_mrope_positions(scheduler_output)
417
418
# Get token indices.
@@ -458,7 +459,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
459
# Copy the tensors to the GPU.
460
self.input_ids[:total_num_scheduled_tokens].copy_(
461
self.input_ids_cpu[:total_num_scheduled_tokens], non_blocking=True)
461
- if self.model_config.uses_mrope:
462
+ if self.uses_mrope:
463
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
464
self.mrope_positions[:, :total_num_scheduled_tokens].copy_(
465
self.mrope_positions_cpu[:, :total_num_scheduled_tokens],
@@ -817,13 +818,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
818
# then the embedding layer is not included in the CUDA graph.
819
input_ids = self.input_ids[:num_input_tokens]
820
inputs_embeds = None
821
+ if self.uses_mrope:
822
+ positions = self.mrope_positions[:, :num_input_tokens]
823
+ else:
824
+ positions = self.positions[:num_input_tokens]
825
826
# Run the decoder.
827
# Use persistent buffers for CUDA graphs.
828
with set_forward_context(attn_metadata, self.vllm_config):
824
- positions = self.mrope_positions[:, :num_input_tokens] \
825
- if self.model_config.uses_mrope \
826
- else self.positions[:num_input_tokens]
829
hidden_states = self.model(
830
input_ids=input_ids,
831
positions=positions,
@@ -1001,10 +1003,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1003
else:
1004
input_ids = self.input_ids[:num_tokens]
1005
inputs_embeds = None
1006
+ if self.uses_mrope:
1007
+ positions = self.mrope_positions[:, :num_tokens]
1008
+ else:
1009
+ positions = self.positions[:num_tokens]
1010
with set_forward_context(None, self.vllm_config):
1005
- positions = self.mrope_positions[:, :num_tokens] \
1006
- if self.model_config.uses_mrope \
1007
- else self.positions[:num_tokens]
1011
hidden_states = model(
1012
input_ids=input_ids,
1013
positions=positions,