@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c23953675f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Hardware][CPU] Enable mrope and support Qwen2-VL on CPU backend (#8770)

Isotr0py committed Sep 25, 2024 at 14:16 UTC c23953675f78bc85045d66fa98aea7d0581c2167
2 files changed +99 -9
vllm/model_executor/models/qwen2_vl.py
+16
@@ -67,6 +67,7 @@ from vllm.multimodal.image import cached_get_image_processor
67 from vllm.platforms import current_platform
68 from vllm.sequence import IntermediateTensors, SequenceData
69 from vllm.transformers_utils.processor import get_processor
70 +from vllm.utils import is_cpu
71
72 from .utils import (PPMissingLayer, is_pp_missing_parameter,
73 make_empty_intermediate_tensors_factory)
@@ -281,6 +282,21 @@ class Qwen2VisionAttention(nn.Module):
282 context_layer = rearrange(output,
283 "(b s) ... -> b s ...",
284 b=batch_size)
285 + elif is_cpu():
286 + seq_length = q.size(1)
287 + q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
288 + attention_mask = torch.zeros([1, seq_length, seq_length],
289 + device=q.device,
290 + dtype=torch.bool)
291 + for i in range(1, len(cu_seqlens)):
292 + attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
293 + cu_seqlens[i - 1]:cu_seqlens[i]] = True
294 + output = F.scaled_dot_product_attention(q,
295 + k,
296 + v,
297 + attention_mask,
298 + dropout_p=0.0)
299 + context_layer = rearrange(output, "b h s d -> b s h d ")
300 else:
301 from xformers import ops as xops
302 from xformers.ops.fmha.attn_bias import BlockDiagonalMask
vllm/worker/cpu_model_runner.py
+83 -9
@@ -12,11 +12,13 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 SchedulerConfig)
13 from vllm.logger import init_logger
14 from vllm.model_executor import SamplingMetadata
15 +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
16 from vllm.model_executor.layers.sampler import SamplerOutput
17 from vllm.model_executor.model_loader import get_model
18 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19 MultiModalInputs)
19 -from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
20 +from vllm.sequence import (IntermediateTensors, SequenceData,
21 + SequenceGroupMetadata)
22 from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
23 from vllm.worker.model_runner_base import (
24 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -145,6 +147,38 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
147 query_lens=seq_lens,
148 )
149
150 + def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data,
151 + computed_len: int):
152 + mm_kwargs = self.multi_modal_input_mapper(mm_data)
153 +
154 + # special processing for mrope position deltas.
155 + mrope_positions = None
156 + if self.runner.model_is_mrope:
157 + image_grid_thw = mm_kwargs.get("image_grid_thw", None)
158 + video_grid_thw = mm_kwargs.get("video_grid_thw", None)
159 + assert image_grid_thw is not None or video_grid_thw is not None, (
160 + "mrope embedding type requires multi-modal input mapper "
161 + "returns 'image_grid_thw' or 'video_grid_thw'.")
162 +
163 + hf_config = self.runner.model_config.hf_config
164 + token_ids = seq_data.get_token_ids()
165 +
166 + mrope_positions, mrope_position_delta = \
167 + MRotaryEmbedding.get_input_positions(
168 + token_ids,
169 + image_grid_thw=image_grid_thw,
170 + video_grid_thw=video_grid_thw,
171 + image_token_id=hf_config.image_token_id,
172 + video_token_id=hf_config.video_token_id,
173 + vision_start_token_id=hf_config.vision_start_token_id,
174 + vision_end_token_id=hf_config.vision_end_token_id,
175 + spatial_merge_size=hf_config.vision_config.
176 + spatial_merge_size,
177 + context_len=computed_len,
178 + )
179 + seq_data.mrope_position_delta = mrope_position_delta
180 + return mm_kwargs, mrope_positions
181 +
182 def _prepare_prompt(
183 self,
184 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -153,6 +187,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
187 assert len(seq_group_metadata_list) > 0
188 input_tokens: List[int] = []
189 input_positions: List[int] = []
190 + input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
191 +
192 slot_mapping: List[int] = []
193 seq_lens: List[int] = []
194 multi_modal_inputs_list: List[MultiModalInputs] = []
@@ -171,14 +207,20 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
207 seq_lens.append(seq_len) # Prompt token num
208 input_tokens.extend(prompt_tokens) # Token ids
209
210 + mrope_positions = None
211 + if (mm_data := seq_group_metadata.multi_modal_data):
212 + mm_kwargs, mrope_positions = self._compute_multi_modal_input(
213 + seq_data, mm_data, computed_len)
214 + multi_modal_inputs_list.append(mm_kwargs)
215 +
216 # Token position ids
217 # NOTE(woosuk): Here we assume that the first token in the prompt
218 # is always the first token in the sequence.
177 - input_positions.extend(list(range(computed_len, seq_len)))
178 -
179 - if (mm_data := seq_group_metadata.multi_modal_data):
180 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
181 - multi_modal_inputs_list.append(mm_kwargs)
219 + if mrope_positions:
220 + for idx in range(3):
221 + input_mrope_positions[idx].extend(mrope_positions[idx])
222 + else:
223 + input_positions.extend(list(range(computed_len, seq_len)))
224
225 # Compute the slot mapping.
226 block_table = seq_group_metadata.block_tables[seq_id]
@@ -202,12 +244,18 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
244 slot = block_number * self.block_size + block_offset
245 slot_mapping.append(slot)
246
247 + if any(input_mrope_positions):
248 + input_positions = None # type: ignore
249 + else:
250 + input_mrope_positions = None # type: ignore
251 +
252 num_prompt_tokens = len(input_tokens)
253
254 input_tokens = torch.tensor(input_tokens,
255 dtype=torch.long,
256 device=self.device) # type: ignore
210 - input_positions = torch.tensor(input_positions,
257 + input_positions = torch.tensor(input_positions
258 + or input_mrope_positions,
259 dtype=torch.long,
260 device=self.device) # type: ignore
261 slot_mapping = torch.tensor(slot_mapping,
@@ -238,6 +286,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
286 assert len(seq_group_metadata_list) > 0
287 input_tokens: List[int] = []
288 input_positions: List[int] = []
289 + input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
290 slot_mapping: List[int] = []
291 seq_lens: List[int] = []
292 block_tables: List[List[int]] = []
@@ -255,7 +304,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
304
305 seq_len = seq_data.get_len()
306 position = seq_len - 1
258 - input_positions.append(position)
307 + if seq_data.mrope_position_delta is not None:
308 + context_len = seq_data.get_num_computed_tokens()
309 + next_pos = MRotaryEmbedding.get_next_input_positions(
310 + seq_data.mrope_position_delta,
311 + context_len,
312 + seq_len,
313 + )
314 + for idx in range(3):
315 + input_mrope_positions[idx].extend(next_pos[idx])
316 + else:
317 + input_positions.append(position)
318
319 seq_len = seq_len if self.sliding_window is None else min(
320 seq_len, self.sliding_window)
@@ -273,12 +332,18 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
332 block_table = block_table[-sliding_window_blocks:]
333 block_tables.append(block_table)
334
335 + if any(input_mrope_positions):
336 + input_positions = None # type: ignore
337 + else:
338 + input_mrope_positions = None # type: ignore
339 +
340 max_decode_seq_len = max(seq_lens)
341
342 input_tokens = torch.tensor(input_tokens,
343 dtype=torch.long,
344 device=self.device)
281 - input_positions = torch.tensor(input_positions,
345 + input_positions = torch.tensor(input_positions
346 + or input_mrope_positions,
347 dtype=torch.long,
348 device=self.device)
349 slot_mapping = torch.tensor(slot_mapping,
@@ -373,6 +438,15 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
438 raise NotImplementedError(
439 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
440
441 + @property
442 + def model_is_mrope(self) -> bool:
443 + """Detect if the model has "mrope" rope_scaling type.
444 + mrope requires keep "rope_deltas" between prompt and decoding phases."""
445 + rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
446 + if rope_scaling is None:
447 + return False
448 + return rope_scaling.get("type", None) == "mrope"
449 +
450 def load_model(self) -> None:
451 self.model = get_model(model_config=self.model_config,
452 load_config=self.load_config,