A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Hardware][CPU] Enable mrope and support Qwen2-VL on CPU backend (#8770)
Isotr0py committed
Sep 25, 2024 at 14:16 UTC
c23953675f78bc85045d66fa98aea7d0581c2167
2 files changed
+99
-9
vllm/model_executor/models/qwen2_vl.py
+16
@@ -67,6 +67,7 @@ from vllm.multimodal.image import cached_get_image_processor
67
from vllm.platforms import current_platform
68
from vllm.sequence import IntermediateTensors, SequenceData
69
from vllm.transformers_utils.processor import get_processor
70
+from vllm.utils import is_cpu
71
72
from .utils import (PPMissingLayer, is_pp_missing_parameter,
73
make_empty_intermediate_tensors_factory)
@@ -281,6 +282,21 @@ class Qwen2VisionAttention(nn.Module):
282
context_layer = rearrange(output,
283
"(b s) ... -> b s ...",
284
b=batch_size)
285
+ elif is_cpu():
286
+ seq_length = q.size(1)
287
+ q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
288
+ attention_mask = torch.zeros([1, seq_length, seq_length],
289
+ device=q.device,
290
+ dtype=torch.bool)
291
+ for i in range(1, len(cu_seqlens)):
292
+ attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
293
+ cu_seqlens[i - 1]:cu_seqlens[i]] = True
294
+ output = F.scaled_dot_product_attention(q,
295
+ k,
296
+ v,
297
+ attention_mask,
298
+ dropout_p=0.0)
299
+ context_layer = rearrange(output, "b h s d -> b s h d ")
300
else:
301
from xformers import ops as xops
302
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
vllm/worker/cpu_model_runner.py
+83
-9
@@ -12,11 +12,13 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
SchedulerConfig)
13
from vllm.logger import init_logger
14
from vllm.model_executor import SamplingMetadata
15
+from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
16
from vllm.model_executor.layers.sampler import SamplerOutput
17
from vllm.model_executor.model_loader import get_model
18
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19
MultiModalInputs)
19
-from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
20
+from vllm.sequence import (IntermediateTensors, SequenceData,
21
+ SequenceGroupMetadata)
22
from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
23
from vllm.worker.model_runner_base import (
24
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -145,6 +147,38 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
147
query_lens=seq_lens,
148
)
149
150
+ def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data,
151
+ computed_len: int):
152
+ mm_kwargs = self.multi_modal_input_mapper(mm_data)
153
+
154
+ # special processing for mrope position deltas.
155
+ mrope_positions = None
156
+ if self.runner.model_is_mrope:
157
+ image_grid_thw = mm_kwargs.get("image_grid_thw", None)
158
+ video_grid_thw = mm_kwargs.get("video_grid_thw", None)
159
+ assert image_grid_thw is not None or video_grid_thw is not None, (
160
+ "mrope embedding type requires multi-modal input mapper "
161
+ "returns 'image_grid_thw' or 'video_grid_thw'.")
162
+
163
+ hf_config = self.runner.model_config.hf_config
164
+ token_ids = seq_data.get_token_ids()
165
+
166
+ mrope_positions, mrope_position_delta = \
167
+ MRotaryEmbedding.get_input_positions(
168
+ token_ids,
169
+ image_grid_thw=image_grid_thw,
170
+ video_grid_thw=video_grid_thw,
171
+ image_token_id=hf_config.image_token_id,
172
+ video_token_id=hf_config.video_token_id,
173
+ vision_start_token_id=hf_config.vision_start_token_id,
174
+ vision_end_token_id=hf_config.vision_end_token_id,
175
+ spatial_merge_size=hf_config.vision_config.
176
+ spatial_merge_size,
177
+ context_len=computed_len,
178
+ )
179
+ seq_data.mrope_position_delta = mrope_position_delta
180
+ return mm_kwargs, mrope_positions
181
+
182
def _prepare_prompt(
183
self,
184
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -153,6 +187,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
187
assert len(seq_group_metadata_list) > 0
188
input_tokens: List[int] = []
189
input_positions: List[int] = []
190
+ input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
191
+
192
slot_mapping: List[int] = []
193
seq_lens: List[int] = []
194
multi_modal_inputs_list: List[MultiModalInputs] = []
@@ -171,14 +207,20 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
207
seq_lens.append(seq_len) # Prompt token num
208
input_tokens.extend(prompt_tokens) # Token ids
209
210
+ mrope_positions = None
211
+ if (mm_data := seq_group_metadata.multi_modal_data):
212
+ mm_kwargs, mrope_positions = self._compute_multi_modal_input(
213
+ seq_data, mm_data, computed_len)
214
+ multi_modal_inputs_list.append(mm_kwargs)
215
+
216
# Token position ids
217
# NOTE(woosuk): Here we assume that the first token in the prompt
218
# is always the first token in the sequence.
177
- input_positions.extend(list(range(computed_len, seq_len)))
178
-
179
- if (mm_data := seq_group_metadata.multi_modal_data):
180
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
181
- multi_modal_inputs_list.append(mm_kwargs)
219
+ if mrope_positions:
220
+ for idx in range(3):
221
+ input_mrope_positions[idx].extend(mrope_positions[idx])
222
+ else:
223
+ input_positions.extend(list(range(computed_len, seq_len)))
224
225
# Compute the slot mapping.
226
block_table = seq_group_metadata.block_tables[seq_id]
@@ -202,12 +244,18 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
244
slot = block_number * self.block_size + block_offset
245
slot_mapping.append(slot)
246
247
+ if any(input_mrope_positions):
248
+ input_positions = None # type: ignore
249
+ else:
250
+ input_mrope_positions = None # type: ignore
251
+
252
num_prompt_tokens = len(input_tokens)
253
254
input_tokens = torch.tensor(input_tokens,
255
dtype=torch.long,
256
device=self.device) # type: ignore
210
- input_positions = torch.tensor(input_positions,
257
+ input_positions = torch.tensor(input_positions
258
+ or input_mrope_positions,
259
dtype=torch.long,
260
device=self.device) # type: ignore
261
slot_mapping = torch.tensor(slot_mapping,
@@ -238,6 +286,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
286
assert len(seq_group_metadata_list) > 0
287
input_tokens: List[int] = []
288
input_positions: List[int] = []
289
+ input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
290
slot_mapping: List[int] = []
291
seq_lens: List[int] = []
292
block_tables: List[List[int]] = []
@@ -255,7 +304,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
304
305
seq_len = seq_data.get_len()
306
position = seq_len - 1
258
- input_positions.append(position)
307
+ if seq_data.mrope_position_delta is not None:
308
+ context_len = seq_data.get_num_computed_tokens()
309
+ next_pos = MRotaryEmbedding.get_next_input_positions(
310
+ seq_data.mrope_position_delta,
311
+ context_len,
312
+ seq_len,
313
+ )
314
+ for idx in range(3):
315
+ input_mrope_positions[idx].extend(next_pos[idx])
316
+ else:
317
+ input_positions.append(position)
318
319
seq_len = seq_len if self.sliding_window is None else min(
320
seq_len, self.sliding_window)
@@ -273,12 +332,18 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
332
block_table = block_table[-sliding_window_blocks:]
333
block_tables.append(block_table)
334
335
+ if any(input_mrope_positions):
336
+ input_positions = None # type: ignore
337
+ else:
338
+ input_mrope_positions = None # type: ignore
339
+
340
max_decode_seq_len = max(seq_lens)
341
342
input_tokens = torch.tensor(input_tokens,
343
dtype=torch.long,
344
device=self.device)
281
- input_positions = torch.tensor(input_positions,
345
+ input_positions = torch.tensor(input_positions
346
+ or input_mrope_positions,
347
dtype=torch.long,
348
device=self.device)
349
slot_mapping = torch.tensor(slot_mapping,
@@ -373,6 +438,15 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
438
raise NotImplementedError(
439
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
440
441
+ @property
442
+ def model_is_mrope(self) -> bool:
443
+ """Detect if the model has "mrope" rope_scaling type.
444
+ mrope requires keep "rope_deltas" between prompt and decoding phases."""
445
+ rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
446
+ if rope_scaling is None:
447
+ return False
448
+ return rope_scaling.get("type", None) == "mrope"
449
+
450
def load_model(self) -> None:
451
self.model = get_model(model_config=self.model_config,
452
load_config=self.load_config,