A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Clean up MiniCPMV (#10751)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 29, 2024 at 12:47 UTC
fa6ecb9aa7a55a99f87fdec7a75011f87af2176c
7 files changed
+149
-215
tests/models/decoder_only/vision_language/test_models.py
+16
-3
@@ -295,16 +295,29 @@ VLM_TEST_SETTINGS = {
295
)
296
],
297
),
298
- "minicpmv": VLMTestInfo(
298
+ "minicpmv_25": VLMTestInfo(
299
models=["openbmb/MiniCPM-Llama3-V-2_5"],
300
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
300
+ test_type=VLMTestType.IMAGE,
301
prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
302
img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
303
max_model_len=4096,
304
max_num_seqs=2,
305
get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
306
postprocess_inputs=model_utils.wrap_inputs_post_processor,
307
- hf_output_post_proc=model_utils.minicmpv_trunc_hf_output,
307
+ hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
308
+ ),
309
+ "minicpmv_26": VLMTestInfo(
310
+ models=["openbmb/MiniCPM-V-2_6"],
311
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
312
+ prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
313
+ img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
314
+ max_model_len=4096,
315
+ max_num_seqs=2,
316
+ get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
317
+ postprocess_inputs=model_utils.ignore_inputs_post_processor(
318
+ "image_sizes"
319
+ ),
320
+ hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
321
),
322
# Tests for phi3v currently live in another file because of a bug in
323
# transformers. Once this issue is fixed, we can enable them here instead.
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+12
-1
@@ -170,7 +170,7 @@ def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput,
170
171
172
####### Post-processors for HF outputs
173
-def minicmpv_trunc_hf_output(hf_output: RunnerOutput,
173
+def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
174
model: str) -> RunnerOutput:
175
output_ids, output_str, out_logprobs = hf_output
176
if output_str.endswith("<|eot_id|>"):
@@ -197,6 +197,17 @@ def get_key_type_post_processor(
197
return process
198
199
200
+def ignore_inputs_post_processor(
201
+ hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
202
+ """Gets a handle to a post processor which ignores a given key."""
203
+
204
+ def process(hf_inputs: BatchEncoding, dtype: str):
205
+ del hf_inputs[hf_inp_key]
206
+ return hf_inputs
207
+
208
+ return process
209
+
210
+
211
def wrap_inputs_post_processor(hf_inputs: BatchEncoding, dtype: str):
212
return {"model_inputs": hf_inputs}
213
vllm/model_executor/layers/fused_moe/layer.py
+5
-5
@@ -242,7 +242,7 @@ class FusedMoE(torch.nn.Module):
242
def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
243
expert_data: torch.Tensor,
244
shard_id: str,
245
- loaded_weight: torch.tensor,
245
+ loaded_weight: torch.Tensor,
246
tp_rank: int):
247
# Load grouped weight scales for group quantization
248
# or model weights
@@ -261,7 +261,7 @@ class FusedMoE(torch.nn.Module):
261
262
def _load_per_channel_weight_scale(self, expert_data: torch.Tensor,
263
shard_dim: int, shard_id: str,
264
- loaded_weight: torch.tensor,
264
+ loaded_weight: torch.Tensor,
265
tp_rank: int):
266
# for per channel weight quantization
267
if shard_id == "w2":
@@ -274,7 +274,7 @@ class FusedMoE(torch.nn.Module):
274
tp_rank=tp_rank)
275
276
def _load_w13(self, expert_data: torch.Tensor, shard_dim: int,
277
- shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
277
+ shard_id: str, loaded_weight: torch.Tensor, tp_rank: int):
278
279
# Index the loaded weight for tp sharding.
280
# gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
@@ -292,7 +292,7 @@ class FusedMoE(torch.nn.Module):
292
expert_data.copy_(loaded_weight)
293
294
def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
295
- shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
295
+ shard_id: str, loaded_weight: torch.Tensor, tp_rank: int):
296
297
# Index the loaded weight for tp sharding.
298
# down_proj: "RowParallel" so tp sharding on input_dim
@@ -311,7 +311,7 @@ class FusedMoE(torch.nn.Module):
311
param_data[expert_id] = loaded_weight
312
313
def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor,
314
- shard_dim: int, loaded_weight: torch.tensor, tp_rank: int):
314
+ shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int):
315
316
if shard_id == "w2":
317
self._load_w2(shard_id=shard_id,
vllm/model_executor/models/minicpm.py
+79
-74
@@ -52,7 +52,7 @@ from vllm.model_executor.utils import set_weight_attrs
52
from vllm.sequence import IntermediateTensors
53
54
from .interfaces import SupportsLoRA, SupportsPP
55
-from .utils import (is_pp_missing_parameter,
55
+from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
56
make_empty_intermediate_tensors_factory, make_layers,
57
maybe_prefix)
58
@@ -378,6 +378,7 @@ class MiniCPMModel(nn.Module):
378
config.hidden_size,
379
org_num_embeddings=config.vocab_size,
380
)
381
+ self.num_experts = getattr(self.config, "num_experts", 0)
382
self._init_layers(prefix, config, cache_config, quant_config)
383
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
384
self.make_empty_intermediate_tensors = (
@@ -437,6 +438,73 @@ class MiniCPMModel(nn.Module):
438
hidden_states = self.norm(hidden_states)
439
return hidden_states
440
441
+ def load_weights(self, weights: Iterable[Tuple[str,
442
+ torch.Tensor]]) -> Set[str]:
443
+ stacked_params_mapping = [
444
+ # (param_name, shard_name, shard_id)
445
+ ("qkv_proj", "q_proj", "q"),
446
+ ("qkv_proj", "k_proj", "k"),
447
+ ("qkv_proj", "v_proj", "v"),
448
+ ("gate_up_proj", "gate_proj", 0),
449
+ ("gate_up_proj", "up_proj", 1),
450
+ ]
451
+ expert_params_mapping = [
452
+ # (param_name, weight_name, expert_id)
453
+ ("ws" if weight_name in ["w1", "w3"] else "w2s",
454
+ f"experts.{expert_id}.{weight_name}.weight", expert_id)
455
+ for expert_id in range(self.num_experts)
456
+ for weight_name in ["w1", "w2", "w3"]
457
+ ]
458
+ params_dict = dict(self.named_parameters())
459
+ loaded_params: Set[str] = set()
460
+ for name, loaded_weight in weights:
461
+ if "rotary_emb.inv_freq" in name:
462
+ continue
463
+ if ("rotary_emb.cos_cached" in name
464
+ or "rotary_emb.sin_cached" in name):
465
+ # Models trained using ColossalAI may include these tensors in
466
+ # the checkpoint. Skip them.
467
+ continue
468
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
469
+ if weight_name not in name:
470
+ continue
471
+ name = name.replace(weight_name, param_name)
472
+ # Skip loading extra bias for GPTQ models.
473
+ if name.endswith(".bias") and name not in params_dict:
474
+ continue
475
+ if is_pp_missing_parameter(name, self):
476
+ continue
477
+ param = params_dict[name]
478
+ weight_loader = param.weight_loader
479
+ weight_loader(param, loaded_weight, shard_id)
480
+ break
481
+ else:
482
+ for param_name, weight_name, expert_id in expert_params_mapping:
483
+ if weight_name not in name:
484
+ continue
485
+ name = name.replace(weight_name, param_name)
486
+ if is_pp_missing_parameter(name, self):
487
+ continue
488
+ param = params_dict[name]
489
+ weight_loader = param.weight_loader
490
+ weight_loader(param,
491
+ loaded_weight,
492
+ weight_name,
493
+ expert_id=expert_id)
494
+ break
495
+ else:
496
+ # Skip loading extra bias for GPTQ models.
497
+ if name.endswith(".bias") and name not in params_dict:
498
+ continue
499
+ if is_pp_missing_parameter(name, self):
500
+ continue
501
+ param = params_dict[name]
502
+ weight_loader = getattr(param, "weight_loader",
503
+ default_weight_loader)
504
+ weight_loader(param, loaded_weight)
505
+ loaded_params.add(name)
506
+ return loaded_params
507
+
508
509
class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
510
packed_modules_mapping = {
@@ -480,8 +548,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
548
self.cache_config = cache_config
549
self.quant_config = quant_config
550
483
- self.num_experts = getattr(self.config, "num_experts", 0)
484
- self._init_model(vllm_config=vllm_config, prefix=prefix)
551
+ self.model = self._init_model(vllm_config=vllm_config,
552
+ prefix=maybe_prefix(prefix, "model"))
553
+
554
unpadded_vocab_size = config.vocab_size
555
if lora_config:
556
unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -506,8 +575,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
575
self.model.make_empty_intermediate_tensors)
576
577
def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
509
- self.model = MiniCPMModel(vllm_config=vllm_config,
510
- prefix=maybe_prefix(prefix, "model"))
578
+ return MiniCPMModel(vllm_config=vllm_config, prefix=prefix)
579
580
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
581
return self.model.get_input_embeddings(input_ids)
@@ -546,72 +614,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
614
615
def load_weights(self, weights: Iterable[Tuple[str,
616
torch.Tensor]]) -> Set[str]:
549
- stacked_params_mapping = [
550
- # (param_name, shard_name, shard_id)
551
- ("qkv_proj", "q_proj", "q"),
552
- ("qkv_proj", "k_proj", "k"),
553
- ("qkv_proj", "v_proj", "v"),
554
- ("gate_up_proj", "gate_proj", 0),
555
- ("gate_up_proj", "up_proj", 1),
556
- ]
557
- expert_params_mapping = [
558
- # (param_name, weight_name, expert_id)
559
- ("ws" if weight_name in ["w1", "w3"] else "w2s",
560
- f"experts.{expert_id}.{weight_name}.weight", expert_id)
561
- for expert_id in range(self.num_experts)
562
- for weight_name in ["w1", "w2", "w3"]
563
- ]
564
- params_dict = dict(self.named_parameters())
565
- loaded_params: Set[str] = set()
566
- for name, loaded_weight in weights:
567
- if "rotary_emb.inv_freq" in name:
568
- continue
569
- if ("rotary_emb.cos_cached" in name
570
- or "rotary_emb.sin_cached" in name):
571
- # Models trained using ColossalAI may include these tensors in
572
- # the checkpoint. Skip them.
573
- continue
574
- # With tie_word_embeddings, we can skip lm_head.weight
575
- # The weight might appear unnecessarily in the files if the model is
576
- # processed with quantization, LoRA, fine-tuning, etc.
577
- if self.config.tie_word_embeddings and "lm_head.weight" in name:
578
- continue
579
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
580
- if weight_name not in name:
581
- continue
582
- name = name.replace(weight_name, param_name)
583
- # Skip loading extra bias for GPTQ models.
584
- if name.endswith(".bias") and name not in params_dict:
585
- continue
586
- if is_pp_missing_parameter(name, self):
587
- continue
588
- param = params_dict[name]
589
- weight_loader = param.weight_loader
590
- weight_loader(param, loaded_weight, shard_id)
591
- break
592
- else:
593
- for param_name, weight_name, expert_id in expert_params_mapping:
594
- if weight_name not in name:
595
- continue
596
- name = name.replace(weight_name, param_name)
597
- if is_pp_missing_parameter(name, self):
598
- continue
599
- param = params_dict[name]
600
- weight_loader = param.weight_loader
601
- weight_loader(param,
602
- loaded_weight,
603
- weight_name,
604
- expert_id=expert_id)
605
- break
606
- else:
607
- # Skip loading extra bias for GPTQ models.
608
- if name.endswith(".bias") and name not in params_dict:
609
- continue
610
- if is_pp_missing_parameter(name, self):
611
- continue
612
- param = params_dict[name]
613
- weight_loader = getattr(param, "weight_loader",
614
- default_weight_loader)
615
- weight_loader(param, loaded_weight)
616
- loaded_params.add(name)
617
- return loaded_params
617
+ loader = AutoWeightsLoader(
618
+ self,
619
+ skip_prefixes=(["lm_head."]
620
+ if self.config.tie_word_embeddings else None),
621
+ )
622
+ return loader.load_weights(weights)
vllm/model_executor/models/minicpm3.py
+2
-3
@@ -40,7 +40,7 @@ from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer,
40
MiniCPMForCausalLM,
41
MiniCPMModel)
42
43
-from .utils import make_layers, maybe_prefix
43
+from .utils import make_layers
44
45
46
class MiniCPM3Attention(nn.Module):
@@ -248,5 +248,4 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
248
}
249
250
def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
251
- self.model = MiniCPM3Model(vllm_config=vllm_config,
252
- prefix=maybe_prefix(prefix, "model"))
251
+ return MiniCPM3Model(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/minicpmv.py
+33
-103
@@ -22,7 +22,7 @@
22
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
23
import math
24
import re
25
-from functools import partial
25
+from functools import cached_property, partial
26
from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27
Set, Tuple, TypedDict, Union)
28
@@ -37,19 +37,15 @@ from vllm.attention import AttentionMetadata
37
from vllm.config import VllmConfig
38
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39
InputContext, token_inputs)
40
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
from vllm.model_executor.layers.quantization import QuantizationConfig
41
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
42
get_2d_sincos_pos_embed)
43
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
44
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
47
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48
-from vllm.model_executor.models.llama import LlamaModel
49
-from vllm.model_executor.models.minicpm import MiniCPMModel
45
+from vllm.model_executor.models.llama import LlamaForCausalLM
46
+from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
47
from vllm.model_executor.models.module_mapping import MultiModelKeys
51
-from vllm.model_executor.models.qwen2 import Qwen2Model
52
-from vllm.model_executor.models.utils import LLMWrapper
48
+from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51
from vllm.multimodal.image import cached_get_image_processor
@@ -58,11 +54,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
54
55
from .idefics2_vision_model import Idefics2VisionTransformer
56
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
61
-from .utils import is_pp_missing_parameter, maybe_prefix
62
-
63
-_KEYS_TO_MODIFY_MAPPING = {
64
- "llm.lm_head": "lm_head",
65
-}
57
+from .utils import AutoWeightsLoader, maybe_prefix
58
59
RawImageType = Union[Image.Image, torch.Tensor]
60
@@ -297,10 +289,9 @@ def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
289
290
def get_placeholder(image_size: Tuple[int, int], num_image: int):
291
if version == (2, 0) or version == (2, 5):
300
- return image_processor. \
301
- get_slice_image_placeholder(image_size)
302
- return image_processor. \
303
- get_slice_image_placeholder(image_size, num_image)
292
+ return image_processor.get_slice_image_placeholder(image_size)
293
+ return image_processor.get_slice_image_placeholder(
294
+ image_size, num_image)
295
296
prompt = inputs.get("prompt")
297
token_ids = inputs.get("prompt_token_ids")
@@ -400,37 +391,32 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
391
self.vpm = self.init_vision_module(config,
392
quant_config,
393
prefix=maybe_prefix(prefix, "vpm"))
403
- param_dtype = torch.get_default_dtype()
404
- self.vpm.to(dtype=param_dtype)
394
self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
395
self.vpm.embeddings.embed_dim)
396
self.embed_dim = self.config.hidden_size
397
+
398
self.resampler = self.init_resampler(self.embed_dim,
399
self.vision_dim,
400
quant_config=quant_config,
401
prefix=maybe_prefix(
402
prefix, "resampler"))
413
- self.resampler.to(device="cuda", dtype=param_dtype)
414
- # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
415
- self.lm_head = ParallelLMHead(config.vocab_size,
416
- config.hidden_size,
417
- quant_config=quant_config,
418
- prefix=maybe_prefix(
419
- prefix, "llm.lm_head"))
420
- self.logits_processor = LogitsProcessor(config.vocab_size)
421
- self.sampler = get_sampler()
403
404
self.make_empty_intermediate_tensors = (
405
self.llm.make_empty_intermediate_tensors)
406
407
+ @cached_property
408
+ def sampler(self):
409
+ if hasattr(self.llm, "sampler"):
410
+ return self.llm.sampler
411
+
412
+ return get_sampler()
413
+
414
def get_embedding(
415
self,
416
input_ids: torch.Tensor,
417
image_inputs: Optional[MiniCPMVImageInputs],
418
) -> Tuple[torch.Tensor, torch.Tensor]:
431
- vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
432
- if hasattr(self.config, "scale_emb"):
433
- vlm_embedding *= self.config.scale_emb
419
+ vlm_embedding: torch.Tensor = self.llm.get_input_embeddings(input_ids)
420
421
if image_inputs is None: # No image
422
vision_hidden_states = torch.tensor([], device=input_ids.device)
@@ -575,7 +561,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
561
# for `torch.compile` integration
562
input_ids = None
563
578
- output = self.llm(
564
+ output = self.llm.model(
565
input_ids=input_ids,
566
positions=positions,
567
kv_caches=kv_caches,
@@ -590,9 +576,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
576
hidden_states: torch.Tensor,
577
sampling_metadata: SamplingMetadata,
578
) -> Optional[torch.Tensor]:
593
- logits = self.logits_processor(self.lm_head, hidden_states,
594
- sampling_metadata)
595
- return logits
579
+ return self.llm.compute_logits(hidden_states, sampling_metadata)
580
581
def sample(
582
self,
@@ -604,52 +588,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
588
589
def load_weights(self, weights: Iterable[Tuple[str,
590
torch.Tensor]]) -> Set[str]:
607
- stacked_params_mapping = [
608
- # (param_name, shard_name, shard_id)
609
- ("qkv_proj", "q_proj", "q"),
610
- ("qkv_proj", "k_proj", "k"),
611
- ("qkv_proj", "v_proj", "v"),
612
- ("gate_up_proj", "gate_proj", 0),
613
- ("gate_up_proj", "up_proj", 1),
614
- ]
615
- params_dict = dict(self.named_parameters())
616
- loaded_params: Set[str] = set()
617
- for name, loaded_weight in weights:
618
- for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
619
- if key_to_modify in name:
620
- name = name.replace(key_to_modify, new_key)
621
- if "rotary_emb.inv_freq" in name:
622
- continue
623
- if ("rotary_emb.cos_cached" in name
624
- or "rotary_emb.sin_cached" in name):
625
- # Models trained using ColossalAI may include these tensors in
626
- # the checkpoint. Skip them.
627
- continue
628
- use_default_weight_loading = False
629
- if self.is_default_weight_loading(name):
630
- use_default_weight_loading = True
631
- else:
632
- for param_name, weight_name, shard_id in stacked_params_mapping:
633
- if weight_name not in name:
634
- continue
635
- name = name.replace(weight_name, param_name)
636
- if is_pp_missing_parameter(name, self):
637
- continue
638
- param = params_dict[name]
639
- weight_loader = param.weight_loader
640
- weight_loader(param, loaded_weight, shard_id)
641
- break
642
- else:
643
- use_default_weight_loading = True
644
- if use_default_weight_loading:
645
- if is_pp_missing_parameter(name, self):
646
- continue
647
- param = params_dict[name]
648
- weight_loader = getattr(param, "weight_loader",
649
- default_weight_loader)
650
- weight_loader(param, loaded_weight)
651
- loaded_params.add(name)
652
- return loaded_params
591
+ loader = AutoWeightsLoader(self)
592
+ return loader.load_weights(weights)
593
594
def get_mm_mapping(self) -> MultiModelKeys:
595
"""
@@ -693,9 +633,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
633
data: MiniCPMVImageInputs) -> torch.Tensor:
634
raise NotImplementedError
635
696
- def is_default_weight_loading(self, name: str) -> bool:
697
- raise NotImplementedError
698
-
636
637
class MiniCPMV2_0(MiniCPMVBaseModel):
638
@@ -708,8 +645,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
645
vllm_config: VllmConfig,
646
prefix: str = "",
647
) -> nn.Module:
711
- return LLMWrapper(MiniCPMModel(vllm_config=vllm_config, prefix=prefix),
712
- name="model")
648
+ return MiniCPMForCausalLM(vllm_config=vllm_config, prefix=prefix)
649
650
def init_vision_module(
651
self,
@@ -717,11 +653,12 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
653
quant_config: Optional[QuantizationConfig],
654
prefix: str = "",
655
) -> nn.Module:
720
- # TODO :refactor this vision model
656
+ # TODO: refactor this vision model
657
try:
658
import timm
659
except ImportError:
660
raise ImportError("Please install timm==0.9.10") from ImportError
661
+
662
with set_default_torch_dtype(torch.float16):
663
model = timm.create_model(
664
"vit_so400m_patch14_siglip_384.webli",
@@ -731,6 +668,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
668
dynamic_img_pad=True,
669
)
670
671
+ model = model.to(dtype=torch.get_default_dtype())
672
+
673
if (isinstance(model, timm.models.VisionTransformer)
674
and model.attn_pool is not None):
675
model.attn_pool = torch.nn.Identity()
@@ -759,7 +698,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
698
quant_config=quant_config,
699
prefix=prefix)
700
762
- return resampler
701
+ return resampler.to(device="cuda", dtype=torch.get_default_dtype())
702
703
def get_vision_embedding(
704
self,
@@ -790,9 +729,6 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
729
730
return self.get_vision_embedding(pixel_values)
731
793
- def is_default_weight_loading(self, name: str) -> bool:
794
- return "resampler" in name or "vpm" in name
795
-
732
733
class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
734
packed_modules_mapping = {
@@ -843,8 +779,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
779
vllm_config: VllmConfig,
780
prefix: str = "",
781
) -> nn.Module:
846
- return LLMWrapper(LlamaModel(vllm_config=vllm_config, prefix=prefix),
847
- name="model")
782
+ return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)
783
784
def init_vision_module(
785
self,
@@ -871,7 +806,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
806
kv_dim=vision_dim,
807
quant_config=quant_config,
808
prefix=prefix)
874
- return resampler
809
+
810
+ return resampler.to(device="cuda", dtype=torch.get_default_dtype())
811
812
def get_vision_embedding(
813
self,
@@ -913,9 +849,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
849
return self.get_vision_embedding(all_pixel_values.type(dtype),
850
patch_attn_mask, tgt_sizes)
851
916
- def is_default_weight_loading(self, name: str) -> bool:
917
- return "resampler" in name
918
-
852
853
class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
854
packed_modules_mapping = {
@@ -966,8 +899,7 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
899
vllm_config: VllmConfig,
900
prefix: str = "",
901
) -> nn.Module:
969
- return LLMWrapper(Qwen2Model(vllm_config=vllm_config, prefix=prefix),
970
- name="model")
902
+ return Qwen2ForCausalLM(vllm_config=vllm_config, prefix=prefix)
903
904
def init_vision_module(
905
self,
@@ -995,7 +927,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
927
kv_dim=vision_dim,
928
quant_config=quant_config,
929
prefix=prefix)
998
- return resampler
930
+
931
+ return resampler.to(device="cuda", dtype=torch.get_default_dtype())
932
933
def get_vision_embedding(
934
self,
@@ -1043,9 +976,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
976
977
return self.resampler(vision_embedding, tgt_sizes)
978
1046
- def is_default_weight_loading(self, name: str) -> bool:
1047
- return "resampler" in name
1048
-
979
980
_SUPPORT_VERSION = {
981
(2, 0): MiniCPMV2_0,
vllm/model_executor/models/utils.py
+2
-26
@@ -1,7 +1,7 @@
1
import itertools
2
from dataclasses import dataclass, field
3
-from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
4
- Optional, Protocol, Set, Tuple, Union, overload)
3
+from typing import (Callable, Dict, Iterable, List, Literal, Mapping, Optional,
4
+ Protocol, Set, Tuple, Union, overload)
5
6
import torch
7
import torch.nn as nn
@@ -560,30 +560,6 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
560
return make_empty_intermediate_tensors
561
562
563
-class LLMWrapper(nn.Module):
564
- """
565
- To align with the key names of LoRA trained with PEFT, we need to add an
566
- additional layer to the llm's implementation.
567
- """
568
-
569
- def __init__(self, llm: nn.Module, name: str) -> None:
570
- super().__init__()
571
- self.model_name = name
572
- setattr(self, name, llm)
573
-
574
- def __getattr__(self, key: str):
575
- llm = super().__getattr__(self.model_name)
576
- if key == self.model_name:
577
- return llm
578
-
579
- return getattr(llm, key)
580
-
581
- # We need to explicitly override this
582
- def __call__(self, *args: Any, **kwargs: Any) -> Any:
583
- llm = super().__getattr__(self.model_name)
584
- return llm(*args, **kwargs)
585
-
586
-
563
def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
564
"""
565
Get the available attention backend for Vision Transformer.