@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa6ecb9aa7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Clean up MiniCPMV (#10751)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 29, 2024 at 12:47 UTC fa6ecb9aa7a55a99f87fdec7a75011f87af2176c
7 files changed +149 -215
tests/models/decoder_only/vision_language/test_models.py
+16 -3
@@ -295,16 +295,29 @@ VLM_TEST_SETTINGS = {
295 )
296 ],
297 ),
298 - "minicpmv": VLMTestInfo(
298 + "minicpmv_25": VLMTestInfo(
299 models=["openbmb/MiniCPM-Llama3-V-2_5"],
300 - test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
300 + test_type=VLMTestType.IMAGE,
301 prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
302 img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
303 max_model_len=4096,
304 max_num_seqs=2,
305 get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
306 postprocess_inputs=model_utils.wrap_inputs_post_processor,
307 - hf_output_post_proc=model_utils.minicmpv_trunc_hf_output,
307 + hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
308 + ),
309 + "minicpmv_26": VLMTestInfo(
310 + models=["openbmb/MiniCPM-V-2_6"],
311 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
312 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
313 + img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
314 + max_model_len=4096,
315 + max_num_seqs=2,
316 + get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
317 + postprocess_inputs=model_utils.ignore_inputs_post_processor(
318 + "image_sizes"
319 + ),
320 + hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
321 ),
322 # Tests for phi3v currently live in another file because of a bug in
323 # transformers. Once this issue is fixed, we can enable them here instead.
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+12 -1
@@ -170,7 +170,7 @@ def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput,
170
171
172 ####### Post-processors for HF outputs
173 -def minicmpv_trunc_hf_output(hf_output: RunnerOutput,
173 +def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
174 model: str) -> RunnerOutput:
175 output_ids, output_str, out_logprobs = hf_output
176 if output_str.endswith("<|eot_id|>"):
@@ -197,6 +197,17 @@ def get_key_type_post_processor(
197 return process
198
199
200 +def ignore_inputs_post_processor(
201 + hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
202 + """Gets a handle to a post processor which ignores a given key."""
203 +
204 + def process(hf_inputs: BatchEncoding, dtype: str):
205 + del hf_inputs[hf_inp_key]
206 + return hf_inputs
207 +
208 + return process
209 +
210 +
211 def wrap_inputs_post_processor(hf_inputs: BatchEncoding, dtype: str):
212 return {"model_inputs": hf_inputs}
213
vllm/model_executor/layers/fused_moe/layer.py
+5 -5
@@ -242,7 +242,7 @@ class FusedMoE(torch.nn.Module):
242 def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
243 expert_data: torch.Tensor,
244 shard_id: str,
245 - loaded_weight: torch.tensor,
245 + loaded_weight: torch.Tensor,
246 tp_rank: int):
247 # Load grouped weight scales for group quantization
248 # or model weights
@@ -261,7 +261,7 @@ class FusedMoE(torch.nn.Module):
261
262 def _load_per_channel_weight_scale(self, expert_data: torch.Tensor,
263 shard_dim: int, shard_id: str,
264 - loaded_weight: torch.tensor,
264 + loaded_weight: torch.Tensor,
265 tp_rank: int):
266 # for per channel weight quantization
267 if shard_id == "w2":
@@ -274,7 +274,7 @@ class FusedMoE(torch.nn.Module):
274 tp_rank=tp_rank)
275
276 def _load_w13(self, expert_data: torch.Tensor, shard_dim: int,
277 - shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
277 + shard_id: str, loaded_weight: torch.Tensor, tp_rank: int):
278
279 # Index the loaded weight for tp sharding.
280 # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
@@ -292,7 +292,7 @@ class FusedMoE(torch.nn.Module):
292 expert_data.copy_(loaded_weight)
293
294 def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
295 - shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
295 + shard_id: str, loaded_weight: torch.Tensor, tp_rank: int):
296
297 # Index the loaded weight for tp sharding.
298 # down_proj: "RowParallel" so tp sharding on input_dim
@@ -311,7 +311,7 @@ class FusedMoE(torch.nn.Module):
311 param_data[expert_id] = loaded_weight
312
313 def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor,
314 - shard_dim: int, loaded_weight: torch.tensor, tp_rank: int):
314 + shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int):
315
316 if shard_id == "w2":
317 self._load_w2(shard_id=shard_id,
vllm/model_executor/models/minicpm.py
+79 -74
@@ -52,7 +52,7 @@ from vllm.model_executor.utils import set_weight_attrs
52 from vllm.sequence import IntermediateTensors
53
54 from .interfaces import SupportsLoRA, SupportsPP
55 -from .utils import (is_pp_missing_parameter,
55 +from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
56 make_empty_intermediate_tensors_factory, make_layers,
57 maybe_prefix)
58
@@ -378,6 +378,7 @@ class MiniCPMModel(nn.Module):
378 config.hidden_size,
379 org_num_embeddings=config.vocab_size,
380 )
381 + self.num_experts = getattr(self.config, "num_experts", 0)
382 self._init_layers(prefix, config, cache_config, quant_config)
383 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
384 self.make_empty_intermediate_tensors = (
@@ -437,6 +438,73 @@ class MiniCPMModel(nn.Module):
438 hidden_states = self.norm(hidden_states)
439 return hidden_states
440
441 + def load_weights(self, weights: Iterable[Tuple[str,
442 + torch.Tensor]]) -> Set[str]:
443 + stacked_params_mapping = [
444 + # (param_name, shard_name, shard_id)
445 + ("qkv_proj", "q_proj", "q"),
446 + ("qkv_proj", "k_proj", "k"),
447 + ("qkv_proj", "v_proj", "v"),
448 + ("gate_up_proj", "gate_proj", 0),
449 + ("gate_up_proj", "up_proj", 1),
450 + ]
451 + expert_params_mapping = [
452 + # (param_name, weight_name, expert_id)
453 + ("ws" if weight_name in ["w1", "w3"] else "w2s",
454 + f"experts.{expert_id}.{weight_name}.weight", expert_id)
455 + for expert_id in range(self.num_experts)
456 + for weight_name in ["w1", "w2", "w3"]
457 + ]
458 + params_dict = dict(self.named_parameters())
459 + loaded_params: Set[str] = set()
460 + for name, loaded_weight in weights:
461 + if "rotary_emb.inv_freq" in name:
462 + continue
463 + if ("rotary_emb.cos_cached" in name
464 + or "rotary_emb.sin_cached" in name):
465 + # Models trained using ColossalAI may include these tensors in
466 + # the checkpoint. Skip them.
467 + continue
468 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
469 + if weight_name not in name:
470 + continue
471 + name = name.replace(weight_name, param_name)
472 + # Skip loading extra bias for GPTQ models.
473 + if name.endswith(".bias") and name not in params_dict:
474 + continue
475 + if is_pp_missing_parameter(name, self):
476 + continue
477 + param = params_dict[name]
478 + weight_loader = param.weight_loader
479 + weight_loader(param, loaded_weight, shard_id)
480 + break
481 + else:
482 + for param_name, weight_name, expert_id in expert_params_mapping:
483 + if weight_name not in name:
484 + continue
485 + name = name.replace(weight_name, param_name)
486 + if is_pp_missing_parameter(name, self):
487 + continue
488 + param = params_dict[name]
489 + weight_loader = param.weight_loader
490 + weight_loader(param,
491 + loaded_weight,
492 + weight_name,
493 + expert_id=expert_id)
494 + break
495 + else:
496 + # Skip loading extra bias for GPTQ models.
497 + if name.endswith(".bias") and name not in params_dict:
498 + continue
499 + if is_pp_missing_parameter(name, self):
500 + continue
501 + param = params_dict[name]
502 + weight_loader = getattr(param, "weight_loader",
503 + default_weight_loader)
504 + weight_loader(param, loaded_weight)
505 + loaded_params.add(name)
506 + return loaded_params
507 +
508
509 class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
510 packed_modules_mapping = {
@@ -480,8 +548,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
548 self.cache_config = cache_config
549 self.quant_config = quant_config
550
483 - self.num_experts = getattr(self.config, "num_experts", 0)
484 - self._init_model(vllm_config=vllm_config, prefix=prefix)
551 + self.model = self._init_model(vllm_config=vllm_config,
552 + prefix=maybe_prefix(prefix, "model"))
553 +
554 unpadded_vocab_size = config.vocab_size
555 if lora_config:
556 unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -506,8 +575,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
575 self.model.make_empty_intermediate_tensors)
576
577 def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
509 - self.model = MiniCPMModel(vllm_config=vllm_config,
510 - prefix=maybe_prefix(prefix, "model"))
578 + return MiniCPMModel(vllm_config=vllm_config, prefix=prefix)
579
580 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
581 return self.model.get_input_embeddings(input_ids)
@@ -546,72 +614,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
614
615 def load_weights(self, weights: Iterable[Tuple[str,
616 torch.Tensor]]) -> Set[str]:
549 - stacked_params_mapping = [
550 - # (param_name, shard_name, shard_id)
551 - ("qkv_proj", "q_proj", "q"),
552 - ("qkv_proj", "k_proj", "k"),
553 - ("qkv_proj", "v_proj", "v"),
554 - ("gate_up_proj", "gate_proj", 0),
555 - ("gate_up_proj", "up_proj", 1),
556 - ]
557 - expert_params_mapping = [
558 - # (param_name, weight_name, expert_id)
559 - ("ws" if weight_name in ["w1", "w3"] else "w2s",
560 - f"experts.{expert_id}.{weight_name}.weight", expert_id)
561 - for expert_id in range(self.num_experts)
562 - for weight_name in ["w1", "w2", "w3"]
563 - ]
564 - params_dict = dict(self.named_parameters())
565 - loaded_params: Set[str] = set()
566 - for name, loaded_weight in weights:
567 - if "rotary_emb.inv_freq" in name:
568 - continue
569 - if ("rotary_emb.cos_cached" in name
570 - or "rotary_emb.sin_cached" in name):
571 - # Models trained using ColossalAI may include these tensors in
572 - # the checkpoint. Skip them.
573 - continue
574 - # With tie_word_embeddings, we can skip lm_head.weight
575 - # The weight might appear unnecessarily in the files if the model is
576 - # processed with quantization, LoRA, fine-tuning, etc.
577 - if self.config.tie_word_embeddings and "lm_head.weight" in name:
578 - continue
579 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
580 - if weight_name not in name:
581 - continue
582 - name = name.replace(weight_name, param_name)
583 - # Skip loading extra bias for GPTQ models.
584 - if name.endswith(".bias") and name not in params_dict:
585 - continue
586 - if is_pp_missing_parameter(name, self):
587 - continue
588 - param = params_dict[name]
589 - weight_loader = param.weight_loader
590 - weight_loader(param, loaded_weight, shard_id)
591 - break
592 - else:
593 - for param_name, weight_name, expert_id in expert_params_mapping:
594 - if weight_name not in name:
595 - continue
596 - name = name.replace(weight_name, param_name)
597 - if is_pp_missing_parameter(name, self):
598 - continue
599 - param = params_dict[name]
600 - weight_loader = param.weight_loader
601 - weight_loader(param,
602 - loaded_weight,
603 - weight_name,
604 - expert_id=expert_id)
605 - break
606 - else:
607 - # Skip loading extra bias for GPTQ models.
608 - if name.endswith(".bias") and name not in params_dict:
609 - continue
610 - if is_pp_missing_parameter(name, self):
611 - continue
612 - param = params_dict[name]
613 - weight_loader = getattr(param, "weight_loader",
614 - default_weight_loader)
615 - weight_loader(param, loaded_weight)
616 - loaded_params.add(name)
617 - return loaded_params
617 + loader = AutoWeightsLoader(
618 + self,
619 + skip_prefixes=(["lm_head."]
620 + if self.config.tie_word_embeddings else None),
621 + )
622 + return loader.load_weights(weights)
vllm/model_executor/models/minicpm3.py
+2 -3
@@ -40,7 +40,7 @@ from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer,
40 MiniCPMForCausalLM,
41 MiniCPMModel)
42
43 -from .utils import make_layers, maybe_prefix
43 +from .utils import make_layers
44
45
46 class MiniCPM3Attention(nn.Module):
@@ -248,5 +248,4 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
248 }
249
250 def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
251 - self.model = MiniCPM3Model(vllm_config=vllm_config,
252 - prefix=maybe_prefix(prefix, "model"))
251 + return MiniCPM3Model(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/minicpmv.py
+33 -103
@@ -22,7 +22,7 @@
22 """Inference-only MiniCPM-V model compatible with HuggingFace weights."""
23 import math
24 import re
25 -from functools import partial
25 +from functools import cached_property, partial
26 from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27 Set, Tuple, TypedDict, Union)
28
@@ -37,19 +37,15 @@ from vllm.attention import AttentionMetadata
37 from vllm.config import VllmConfig
38 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39 InputContext, token_inputs)
40 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 from vllm.model_executor.layers.quantization import QuantizationConfig
41 from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
42 get_2d_sincos_pos_embed)
43 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
44 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
47 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48 -from vllm.model_executor.models.llama import LlamaModel
49 -from vllm.model_executor.models.minicpm import MiniCPMModel
45 +from vllm.model_executor.models.llama import LlamaForCausalLM
46 +from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
47 from vllm.model_executor.models.module_mapping import MultiModelKeys
51 -from vllm.model_executor.models.qwen2 import Qwen2Model
52 -from vllm.model_executor.models.utils import LLMWrapper
48 +from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51 from vllm.multimodal.image import cached_get_image_processor
@@ -58,11 +54,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
54
55 from .idefics2_vision_model import Idefics2VisionTransformer
56 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
61 -from .utils import is_pp_missing_parameter, maybe_prefix
62 -
63 -_KEYS_TO_MODIFY_MAPPING = {
64 - "llm.lm_head": "lm_head",
65 -}
57 +from .utils import AutoWeightsLoader, maybe_prefix
58
59 RawImageType = Union[Image.Image, torch.Tensor]
60
@@ -297,10 +289,9 @@ def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
289
290 def get_placeholder(image_size: Tuple[int, int], num_image: int):
291 if version == (2, 0) or version == (2, 5):
300 - return image_processor. \
301 - get_slice_image_placeholder(image_size)
302 - return image_processor. \
303 - get_slice_image_placeholder(image_size, num_image)
292 + return image_processor.get_slice_image_placeholder(image_size)
293 + return image_processor.get_slice_image_placeholder(
294 + image_size, num_image)
295
296 prompt = inputs.get("prompt")
297 token_ids = inputs.get("prompt_token_ids")
@@ -400,37 +391,32 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
391 self.vpm = self.init_vision_module(config,
392 quant_config,
393 prefix=maybe_prefix(prefix, "vpm"))
403 - param_dtype = torch.get_default_dtype()
404 - self.vpm.to(dtype=param_dtype)
394 self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
395 self.vpm.embeddings.embed_dim)
396 self.embed_dim = self.config.hidden_size
397 +
398 self.resampler = self.init_resampler(self.embed_dim,
399 self.vision_dim,
400 quant_config=quant_config,
401 prefix=maybe_prefix(
402 prefix, "resampler"))
413 - self.resampler.to(device="cuda", dtype=param_dtype)
414 - # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
415 - self.lm_head = ParallelLMHead(config.vocab_size,
416 - config.hidden_size,
417 - quant_config=quant_config,
418 - prefix=maybe_prefix(
419 - prefix, "llm.lm_head"))
420 - self.logits_processor = LogitsProcessor(config.vocab_size)
421 - self.sampler = get_sampler()
403
404 self.make_empty_intermediate_tensors = (
405 self.llm.make_empty_intermediate_tensors)
406
407 + @cached_property
408 + def sampler(self):
409 + if hasattr(self.llm, "sampler"):
410 + return self.llm.sampler
411 +
412 + return get_sampler()
413 +
414 def get_embedding(
415 self,
416 input_ids: torch.Tensor,
417 image_inputs: Optional[MiniCPMVImageInputs],
418 ) -> Tuple[torch.Tensor, torch.Tensor]:
431 - vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
432 - if hasattr(self.config, "scale_emb"):
433 - vlm_embedding *= self.config.scale_emb
419 + vlm_embedding: torch.Tensor = self.llm.get_input_embeddings(input_ids)
420
421 if image_inputs is None: # No image
422 vision_hidden_states = torch.tensor([], device=input_ids.device)
@@ -575,7 +561,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
561 # for `torch.compile` integration
562 input_ids = None
563
578 - output = self.llm(
564 + output = self.llm.model(
565 input_ids=input_ids,
566 positions=positions,
567 kv_caches=kv_caches,
@@ -590,9 +576,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
576 hidden_states: torch.Tensor,
577 sampling_metadata: SamplingMetadata,
578 ) -> Optional[torch.Tensor]:
593 - logits = self.logits_processor(self.lm_head, hidden_states,
594 - sampling_metadata)
595 - return logits
579 + return self.llm.compute_logits(hidden_states, sampling_metadata)
580
581 def sample(
582 self,
@@ -604,52 +588,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
588
589 def load_weights(self, weights: Iterable[Tuple[str,
590 torch.Tensor]]) -> Set[str]:
607 - stacked_params_mapping = [
608 - # (param_name, shard_name, shard_id)
609 - ("qkv_proj", "q_proj", "q"),
610 - ("qkv_proj", "k_proj", "k"),
611 - ("qkv_proj", "v_proj", "v"),
612 - ("gate_up_proj", "gate_proj", 0),
613 - ("gate_up_proj", "up_proj", 1),
614 - ]
615 - params_dict = dict(self.named_parameters())
616 - loaded_params: Set[str] = set()
617 - for name, loaded_weight in weights:
618 - for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
619 - if key_to_modify in name:
620 - name = name.replace(key_to_modify, new_key)
621 - if "rotary_emb.inv_freq" in name:
622 - continue
623 - if ("rotary_emb.cos_cached" in name
624 - or "rotary_emb.sin_cached" in name):
625 - # Models trained using ColossalAI may include these tensors in
626 - # the checkpoint. Skip them.
627 - continue
628 - use_default_weight_loading = False
629 - if self.is_default_weight_loading(name):
630 - use_default_weight_loading = True
631 - else:
632 - for param_name, weight_name, shard_id in stacked_params_mapping:
633 - if weight_name not in name:
634 - continue
635 - name = name.replace(weight_name, param_name)
636 - if is_pp_missing_parameter(name, self):
637 - continue
638 - param = params_dict[name]
639 - weight_loader = param.weight_loader
640 - weight_loader(param, loaded_weight, shard_id)
641 - break
642 - else:
643 - use_default_weight_loading = True
644 - if use_default_weight_loading:
645 - if is_pp_missing_parameter(name, self):
646 - continue
647 - param = params_dict[name]
648 - weight_loader = getattr(param, "weight_loader",
649 - default_weight_loader)
650 - weight_loader(param, loaded_weight)
651 - loaded_params.add(name)
652 - return loaded_params
591 + loader = AutoWeightsLoader(self)
592 + return loader.load_weights(weights)
593
594 def get_mm_mapping(self) -> MultiModelKeys:
595 """
@@ -693,9 +633,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
633 data: MiniCPMVImageInputs) -> torch.Tensor:
634 raise NotImplementedError
635
696 - def is_default_weight_loading(self, name: str) -> bool:
697 - raise NotImplementedError
698 -
636
637 class MiniCPMV2_0(MiniCPMVBaseModel):
638
@@ -708,8 +645,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
645 vllm_config: VllmConfig,
646 prefix: str = "",
647 ) -> nn.Module:
711 - return LLMWrapper(MiniCPMModel(vllm_config=vllm_config, prefix=prefix),
712 - name="model")
648 + return MiniCPMForCausalLM(vllm_config=vllm_config, prefix=prefix)
649
650 def init_vision_module(
651 self,
@@ -717,11 +653,12 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
653 quant_config: Optional[QuantizationConfig],
654 prefix: str = "",
655 ) -> nn.Module:
720 - # TODO :refactor this vision model
656 + # TODO: refactor this vision model
657 try:
658 import timm
659 except ImportError:
660 raise ImportError("Please install timm==0.9.10") from ImportError
661 +
662 with set_default_torch_dtype(torch.float16):
663 model = timm.create_model(
664 "vit_so400m_patch14_siglip_384.webli",
@@ -731,6 +668,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
668 dynamic_img_pad=True,
669 )
670
671 + model = model.to(dtype=torch.get_default_dtype())
672 +
673 if (isinstance(model, timm.models.VisionTransformer)
674 and model.attn_pool is not None):
675 model.attn_pool = torch.nn.Identity()
@@ -759,7 +698,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
698 quant_config=quant_config,
699 prefix=prefix)
700
762 - return resampler
701 + return resampler.to(device="cuda", dtype=torch.get_default_dtype())
702
703 def get_vision_embedding(
704 self,
@@ -790,9 +729,6 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
729
730 return self.get_vision_embedding(pixel_values)
731
793 - def is_default_weight_loading(self, name: str) -> bool:
794 - return "resampler" in name or "vpm" in name
795 -
732
733 class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
734 packed_modules_mapping = {
@@ -843,8 +779,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
779 vllm_config: VllmConfig,
780 prefix: str = "",
781 ) -> nn.Module:
846 - return LLMWrapper(LlamaModel(vllm_config=vllm_config, prefix=prefix),
847 - name="model")
782 + return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)
783
784 def init_vision_module(
785 self,
@@ -871,7 +806,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
806 kv_dim=vision_dim,
807 quant_config=quant_config,
808 prefix=prefix)
874 - return resampler
809 +
810 + return resampler.to(device="cuda", dtype=torch.get_default_dtype())
811
812 def get_vision_embedding(
813 self,
@@ -913,9 +849,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
849 return self.get_vision_embedding(all_pixel_values.type(dtype),
850 patch_attn_mask, tgt_sizes)
851
916 - def is_default_weight_loading(self, name: str) -> bool:
917 - return "resampler" in name
918 -
852
853 class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
854 packed_modules_mapping = {
@@ -966,8 +899,7 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
899 vllm_config: VllmConfig,
900 prefix: str = "",
901 ) -> nn.Module:
969 - return LLMWrapper(Qwen2Model(vllm_config=vllm_config, prefix=prefix),
970 - name="model")
902 + return Qwen2ForCausalLM(vllm_config=vllm_config, prefix=prefix)
903
904 def init_vision_module(
905 self,
@@ -995,7 +927,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
927 kv_dim=vision_dim,
928 quant_config=quant_config,
929 prefix=prefix)
998 - return resampler
930 +
931 + return resampler.to(device="cuda", dtype=torch.get_default_dtype())
932
933 def get_vision_embedding(
934 self,
@@ -1043,9 +976,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
976
977 return self.resampler(vision_embedding, tgt_sizes)
978
1046 - def is_default_weight_loading(self, name: str) -> bool:
1047 - return "resampler" in name
1048 -
979
980 _SUPPORT_VERSION = {
981 (2, 0): MiniCPMV2_0,
vllm/model_executor/models/utils.py
+2 -26
@@ -1,7 +1,7 @@
1 import itertools
2 from dataclasses import dataclass, field
3 -from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
4 - Optional, Protocol, Set, Tuple, Union, overload)
3 +from typing import (Callable, Dict, Iterable, List, Literal, Mapping, Optional,
4 + Protocol, Set, Tuple, Union, overload)
5
6 import torch
7 import torch.nn as nn
@@ -560,30 +560,6 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
560 return make_empty_intermediate_tensors
561
562
563 -class LLMWrapper(nn.Module):
564 - """
565 - To align with the key names of LoRA trained with PEFT, we need to add an
566 - additional layer to the llm's implementation.
567 - """
568 -
569 - def __init__(self, llm: nn.Module, name: str) -> None:
570 - super().__init__()
571 - self.model_name = name
572 - setattr(self, name, llm)
573 -
574 - def __getattr__(self, key: str):
575 - llm = super().__getattr__(self.model_name)
576 - if key == self.model_name:
577 - return llm
578 -
579 - return getattr(llm, key)
580 -
581 - # We need to explicitly override this
582 - def __call__(self, *args: Any, **kwargs: Any) -> Any:
583 - llm = super().__getattr__(self.model_name)
584 - return llm(*args, **kwargs)
585 -
586 -
563 def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
564 """
565 Get the available attention backend for Vision Transformer.