@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99d01a5e3d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Simplify M-RoPE (#12352)

Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: imkero <kerorek@outlook.com>

Roger Wang committed Jan 23, 2025 at 07:13 UTC 99d01a5e3d5278284bad359ac8b87ee7a551afda
1 file changed +10 -14
vllm/v1/worker/gpu_model_runner.py
+10 -14
@@ -144,28 +144,24 @@ class GPUModelRunner:
144
145 # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
146 if self.model_config.uses_mrope:
147 - # NOTE: `mrope_positions` is implemented as a permuted tensor to
148 - # satisfy the following properties to allow `torch.compile` to work
149 - # properly:
150 - # - shape: (3, <variable>)
151 - # - stride: (1, 3)
152 - # See detailed explanation in https://github.com/vllm-project/vllm/pull/12128#discussion_r1921022256
147 + # NOTE: `mrope_positions` is implemented with one additional dummy
148 + # position on purpose to make it non-contiguous so that it can work
149 + # with torch compile.
150 + # See detailed explanation in https://github.com/vllm-project/vllm/pull/12128#discussion_r1926431923
151
152 # NOTE: When M-RoPE is enabled, position ids are 3D regardless of
153 # the modality of inputs. For text-only inputs, each dimension has
154 # identical position IDs, making M-RoPE functionally equivalent to
155 # 1D-RoPE.
156 # See page 5 of https://arxiv.org/abs/2409.12191
159 - self.mrope_positions = torch.zeros((self.max_num_tokens, 3),
157 + self.mrope_positions = torch.zeros((3, self.max_num_tokens + 1),
158 dtype=torch.int64,
159 device=self.device)
162 - self.mrope_positions_cpu = torch.zeros((self.max_num_tokens, 3),
163 - dtype=torch.int64,
164 - device="cpu",
165 - pin_memory=self.pin_memory)
166 -
167 - self.mrope_positions = self.mrope_positions.permute((1, 0))
168 - self.mrope_positions_cpu = self.mrope_positions_cpu.permute((1, 0))
160 + self.mrope_positions_cpu = torch.zeros(
161 + (3, self.max_num_tokens + 1),
162 + dtype=torch.int64,
163 + device="cpu",
164 + pin_memory=self.pin_memory)
165
166 self.inputs_embeds = torch.zeros(
167 (self.max_num_tokens, self.hidden_size),