@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8bfaa4e31e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] fix composite weight loading and EAGLE weight loading (#9160)

Cyrus Leung committed Oct 9, 2024 at 15:36 UTC 8bfaa4e31eb63d41499fec933e68969ebbedb01f
15 files changed +244 -364
vllm/model_executor/models/blip2.py
+3 -34
@@ -13,7 +13,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.quantization import QuantizationConfig
15 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.sequence import IntermediateTensors, SequenceData
@@ -21,7 +20,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
20 from .blip import (BlipVisionModel, dummy_image_for_blip,
21 get_max_blip_image_tokens)
22 from .interfaces import SupportsMultiModal, SupportsPP
24 -from .utils import (group_weights_with_prefix, init_vllm_registered_model,
23 +from .utils import (AutoWeightsLoader, init_vllm_registered_model,
24 merge_multimodal_embeddings)
25
26 # We use this internally as placeholders since there is no image token
@@ -687,35 +686,5 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
686 return self.language_model.sample(logits, sampling_metadata)
687
688 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
690 - # prepare weight iterators for components
691 - weights_group = group_weights_with_prefix(weights)
692 -
693 - # load vision encoder
694 - self.vision_model.load_weights(weights_group["vision_model"])
695 -
696 - # load query tokens
697 - for name, loaded_weight in weights_group["query_tokens"]:
698 - assert name == ""
699 - param = self.query_tokens
700 - weight_loader = getattr(param, "weight_loader",
701 - default_weight_loader)
702 - weight_loader(param, loaded_weight)
703 -
704 - # load qformer
705 - qformer_params_dict = dict(self.qformer.named_parameters())
706 - for name, loaded_weight in weights_group["qformer"]:
707 - param = qformer_params_dict[name]
708 - weight_loader = getattr(param, "weight_loader",
709 - default_weight_loader)
710 - weight_loader(param, loaded_weight)
711 -
712 - # load mlp projector
713 - mlp_params_dict = dict(self.language_projection.named_parameters())
714 - for name, loaded_weight in weights_group["language_projection"]:
715 - param = mlp_params_dict[name]
716 - weight_loader = getattr(param, "weight_loader",
717 - default_weight_loader)
718 - weight_loader(param, loaded_weight)
719 -
720 - # load llm backbone
721 - self.language_model.load_weights(weights_group["language_model"])
689 + loader = AutoWeightsLoader(self)
690 + loader.load_weights(weights)
vllm/model_executor/models/fuyu.py
+3 -16
@@ -31,7 +31,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31 from vllm.model_executor.layers.linear import ColumnParallelLinear
32 from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.layers.sampler import SamplerOutput
34 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -42,8 +41,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
41 SequenceData)
42
43 from .interfaces import SupportsMultiModal, SupportsPP
45 -from .utils import (flatten_bn, group_weights_with_prefix,
46 - merge_multimodal_embeddings)
44 +from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
45
46 # Cannot find the following 2 numbers from hf config.
47 _IMAGE_TOKEN_ID = 71011
@@ -349,16 +347,5 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
347 return next_tokens
348
349 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
352 - # prepare weight iterators for components
353 - weights_group = group_weights_with_prefix(weights)
354 -
355 - # load vision embeddings
356 - vision_params_dict = dict(self.vision_embed_tokens.named_parameters())
357 - for name, loaded_weight in weights_group["vision_embed_tokens"]:
358 - param = vision_params_dict[name]
359 - weight_loader = getattr(param, "weight_loader",
360 - default_weight_loader)
361 - weight_loader(param, loaded_weight)
362 -
363 - # load llm backbone
364 - self.language_model.load_weights(weights_group["language_model"])
350 + loader = AutoWeightsLoader(self)
351 + loader.load_weights(weights)
vllm/model_executor/models/gemma2.py
+7 -17
@@ -40,7 +40,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.sequence import IntermediateTensors
41
42 from .interfaces import SupportsLoRA, SupportsPP
43 -from .utils import (group_weights_with_prefix, is_pp_missing_parameter,
43 +from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
44 make_empty_intermediate_tensors_factory, make_layers)
45
46 logger = init_logger(__name__)
@@ -447,19 +447,9 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
447 return next_tokens
448
449 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
450 - weights_group = group_weights_with_prefix(weights)
451 -
452 - self.model.load_weights(weights_group["model"])
453 -
454 - if not self.config.tie_word_embeddings:
455 - # NOTE: For now self.lm_head is not defined because
456 - # tie_word_embeddings is assumed to the False
457 - lm_head_dict = dict(self.lm_head.named_parameters())
458 - for name, loaded_weight in weights_group["lm_head"]:
459 - if is_pp_missing_parameter(name, self.lm_head):
460 - continue
461 -
462 - param = lm_head_dict[name]
463 - weight_loader = getattr(param, "weight_loader",
464 - default_weight_loader)
465 - weight_loader(param, loaded_weight)
450 + loader = AutoWeightsLoader(
451 + self,
452 + skip_prefixes=(["lm_head."]
453 + if self.config.tie_word_embeddings else None),
454 + )
455 + loader.load_weights(weights)
vllm/model_executor/models/internvl.py
+4 -19
@@ -20,7 +20,6 @@ from vllm.config import CacheConfig, MultiModalConfig
20 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 from vllm.model_executor.models.intern_vit import InternVisionModel
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -32,8 +31,8 @@ from vllm.utils import is_list_of
31 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
32 get_clip_num_patches)
33 from .interfaces import SupportsMultiModal, SupportsPP
35 -from .utils import (flatten_bn, group_weights_with_prefix,
36 - init_vllm_registered_model, merge_multimodal_embeddings)
34 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
35 + merge_multimodal_embeddings)
36
37 IMG_START = '<img>'
38 IMG_END = '</img>'
@@ -609,19 +608,5 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
608 return self.language_model.sample(logits, sampling_metadata)
609
610 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
612 - # prepare weight iterators for components
613 - weights_group = group_weights_with_prefix(weights)
614 -
615 - # load vision encoder
616 - self.vision_model.load_weights(weights_group["vision_model"])
617 -
618 - # load mlp projector
619 - mlp_params_dict = dict(self.mlp1.named_parameters())
620 - for name, loaded_weight in weights_group["mlp1"]:
621 - param = mlp_params_dict[name]
622 - weight_loader = getattr(param, "weight_loader",
623 - default_weight_loader)
624 - weight_loader(param, loaded_weight)
625 -
626 - # load llm backbone
627 - self.language_model.load_weights(weights_group["language_model"])
611 + loader = AutoWeightsLoader(self)
612 + loader.load_weights(weights)
vllm/model_executor/models/llama.py
+8 -20
@@ -51,8 +51,7 @@ from vllm.sequence import IntermediateTensors
51 from vllm.utils import is_hip
52
53 from .interfaces import SupportsLoRA, SupportsPP
54 -from .utils import (PPMissingLayer, group_weights_with_prefix,
55 - is_pp_missing_parameter,
54 +from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
55 make_empty_intermediate_tensors_factory, make_layers)
56
57
@@ -564,25 +563,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
563 return next_tokens
564
565 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
567 - weights = [
566 + loader = AutoWeightsLoader(
567 + self,
568 + skip_prefixes=(["lm_head."]
569 + if self.config.tie_word_embeddings else None),
570 + )
571 + loader.load_weights(
572 self.maybe_remap_mistral(name, loaded_weight)
569 - for name, loaded_weight in weights
570 - ]
571 -
572 - weights_group = group_weights_with_prefix(weights)
573 -
574 - self.model.load_weights(weights_group["model"])
575 -
576 - if not self.config.tie_word_embeddings:
577 - lm_head_dict = dict(self.lm_head.named_parameters())
578 - for name, loaded_weight in weights_group["lm_head"]:
579 - if is_pp_missing_parameter(name, self.lm_head):
580 - continue
581 -
582 - param = lm_head_dict[name]
583 - weight_loader = getattr(param, "weight_loader",
584 - default_weight_loader)
585 - weight_loader(param, loaded_weight)
573 + for name, loaded_weight in weights)
574
575 def load_kv_cache_scales(self, quantization_param_path: str) -> None:
576 self.model.load_kv_cache_scales(quantization_param_path)
vllm/model_executor/models/llava.py
+4 -19
@@ -13,7 +13,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.quantization import QuantizationConfig
15 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.sequence import IntermediateTensors
@@ -26,8 +25,8 @@ from .interfaces import SupportsMultiModal, SupportsPP
25 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
26 dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
27 input_processor_for_siglip)
29 -from .utils import (flatten_bn, group_weights_with_prefix,
30 - init_vllm_registered_model, merge_multimodal_embeddings)
28 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
29 + merge_multimodal_embeddings)
30
31
32 class LlavaImagePixelInputs(TypedDict):
@@ -406,19 +405,5 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
405 return self.language_model.sample(logits, sampling_metadata)
406
407 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
409 - # prepare weight iterators for components
410 - weights_group = group_weights_with_prefix(weights)
411 -
412 - # load vision encoder
413 - self.vision_tower.load_weights(weights_group["vision_tower"])
414 -
415 - # load mlp projector
416 - mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
417 - for name, loaded_weight in weights_group["multi_modal_projector"]:
418 - param = mlp_params_dict[name]
419 - weight_loader = getattr(param, "weight_loader",
420 - default_weight_loader)
421 - weight_loader(param, loaded_weight)
422 -
423 - # load llm backbone
424 - self.language_model.load_weights(weights_group["language_model"])
408 + loader = AutoWeightsLoader(self)
409 + loader.load_weights(weights)
vllm/model_executor/models/llava_next.py
+4 -27
@@ -15,7 +15,6 @@ from vllm.config import CacheConfig, MultiModalConfig
15 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.sequence import IntermediateTensors
@@ -29,8 +28,8 @@ from .llava import LlavaMultiModalProjector
28 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
30 get_siglip_patch_grid_length, input_processor_for_siglip)
32 -from .utils import (flatten_bn, group_weights_with_prefix,
33 - init_vllm_registered_model, merge_multimodal_embeddings)
31 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
32 + merge_multimodal_embeddings)
33
34 # Result in the max possible feature size (2x2 grid of 336x336px tiles)
35 MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
@@ -642,27 +641,5 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
641 return self.language_model.sample(logits, sampling_metadata)
642
643 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
645 - # prepare weight iterators for components
646 - weights_group = group_weights_with_prefix(weights)
647 -
648 - # load vision encoder
649 - self.vision_tower.load_weights(weights_group["vision_tower"])
650 -
651 - # load mlp projector
652 - mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
653 - for name, loaded_weight in weights_group["multi_modal_projector"]:
654 - param = mlp_params_dict[name]
655 - weight_loader = getattr(param, "weight_loader",
656 - default_weight_loader)
657 - weight_loader(param, loaded_weight)
658 -
659 - # load newline
660 - for name, loaded_weight in weights_group["image_newline"]:
661 - assert name == ""
662 - param = self.image_newline
663 - weight_loader = getattr(param, "weight_loader",
664 - default_weight_loader)
665 - weight_loader(param, loaded_weight)
666 -
667 - # load llm backbone
668 - self.language_model.load_weights(weights_group["language_model"])
644 + loader = AutoWeightsLoader(self)
645 + loader.load_weights(weights)
vllm/model_executor/models/llava_next_video.py
+7 -18
@@ -15,7 +15,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 from vllm.model_executor.models.clip import CLIPVisionModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -28,7 +27,7 @@ from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
27 from .interfaces import SupportsMultiModal, SupportsPP
28 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29 dummy_seq_data_for_siglip)
31 -from .utils import (group_weights_with_prefix, init_vllm_registered_model,
30 +from .utils import (AutoWeightsLoader, init_vllm_registered_model,
31 merge_multimodal_embeddings)
32
33 # For profile run
@@ -458,19 +457,9 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
457 return self.language_model.sample(logits, sampling_metadata)
458
459 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
461 - # prepare weight iterators for components
462 - weights_group = group_weights_with_prefix(weights)
463 -
464 - # load vision encoder
465 - self.vision_tower.load_weights(weights_group["vision_tower"])
466 -
467 - # load mlp projector
468 - mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
469 - for name, loaded_weight in weights_group["multi_modal_projector"]:
470 - param = mlp_params_dict[name]
471 - weight_loader = getattr(param, "weight_loader",
472 - default_weight_loader)
473 - weight_loader(param, loaded_weight)
474 -
475 - # load llm backbone
476 - self.language_model.load_weights(weights_group["language_model"])
460 + loader = AutoWeightsLoader(
461 + self,
462 + # This model doesn't support images for now
463 + ignore_unexpected_prefixes=["image_newline"],
464 + )
465 + loader.load_weights(weights)
vllm/model_executor/models/llava_onevision.py
+4 -19
@@ -20,7 +20,6 @@ from vllm.logger import init_logger
20 from vllm.model_executor.layers.activation import get_act_fn
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.multimodal.utils import (cached_get_tokenizer,
@@ -35,8 +34,8 @@ from .interfaces import SupportsMultiModal, SupportsPP
34 from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
35 dummy_video_for_siglip, get_siglip_image_feature_size,
36 get_siglip_patch_grid_length, input_processor_for_siglip)
38 -from .utils import (flatten_bn, group_weights_with_prefix,
39 - init_vllm_registered_model, merge_multimodal_embeddings)
37 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38 + merge_multimodal_embeddings)
39
40 logger = init_logger(__name__)
41
@@ -872,19 +871,5 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
871 return self.language_model.sample(logits, sampling_metadata)
872
873 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
875 - # prepare weight iterators for components
876 - weights_group = group_weights_with_prefix(weights)
877 -
878 - # load vision encoder
879 - self.vision_tower.load_weights(weights_group["vision_tower"])
880 -
881 - # load mlp projector
882 - mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
883 - for name, loaded_weight in weights_group["multi_modal_projector"]:
884 - param = mlp_params_dict[name]
885 - weight_loader = getattr(param, "weight_loader",
886 - default_weight_loader)
887 - weight_loader(param, loaded_weight)
888 -
889 - # load llm backbone
890 - self.language_model.load_weights(weights_group["language_model"])
874 + loader = AutoWeightsLoader(self)
875 + loader.load_weights(weights)
vllm/model_executor/models/paligemma.py
+3 -18
@@ -11,7 +11,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 from vllm.logger import init_logger
12 from vllm.model_executor.layers.quantization import QuantizationConfig
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
14 from vllm.model_executor.models.gemma import GemmaForCausalLM
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
16 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -21,7 +20,7 @@ from vllm.sequence import IntermediateTensors
20 from .interfaces import SupportsMultiModal, SupportsPP
21 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
22 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
24 -from .utils import group_weights_with_prefix, merge_multimodal_embeddings
23 +from .utils import AutoWeightsLoader, merge_multimodal_embeddings
24
25 logger = init_logger(__name__)
26
@@ -292,19 +291,5 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
291 return self.language_model.sample(logits, sampling_metadata)
292
293 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
295 - # prepare weight iterators for components
296 - weights_group = group_weights_with_prefix(weights)
297 -
298 - # load vision tower
299 - self.vision_tower.load_weights(weights_group["vision_tower"])
300 -
301 - # load mlp projector
302 - mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
303 - for name, loaded_weight in weights_group["multi_modal_projector"]:
304 - param = mlp_params_dict[name]
305 - weight_loader = getattr(param, "weight_loader",
306 - default_weight_loader)
307 - weight_loader(param, loaded_weight)
308 -
309 - # load llm backbone
310 - self.language_model.load_weights(weights_group["language_model"])
294 + loader = AutoWeightsLoader(self)
295 + loader.load_weights(weights)
vllm/model_executor/models/phi3v.py
+12 -59
@@ -31,7 +31,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31 from vllm.logger import init_logger
32 from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
34 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34 from vllm.model_executor.models.clip import CLIPVisionModel
35 from vllm.model_executor.models.llama import LlamaForCausalLM
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -42,15 +41,11 @@ from vllm.utils import is_list_of
41
42 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
43 from .interfaces import SupportsMultiModal, SupportsPP
45 -from .utils import (flatten_bn, group_weights_with_prefix,
44 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
45 merge_multimodal_embeddings)
46
47 logger = init_logger(__name__)
48
50 -_KEYS_TO_MODIFY_MAPPING = {
51 - "model.vision_embed_tokens": "vision_embed_tokens",
52 -}
53 -
49 # Cannot find the following 2 numbers from hf config.
50 _IMAGE_TOKEN_ID = 32044
51
@@ -295,35 +290,8 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
290 return image_features_hd_newline
291
292 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
298 - # prepare weight iterators for components
299 - weights_group = group_weights_with_prefix(weights)
300 -
301 - # load vision encoder
302 - self.img_processor.load_weights(weights_group["img_processor"])
303 -
304 - # load glb_GN
305 - for name, loaded_weight in weights_group["glb_GN"]:
306 - assert name == ""
307 - param = self.glb_GN
308 - weight_loader = getattr(param, "weight_loader",
309 - default_weight_loader)
310 - weight_loader(param, loaded_weight)
311 -
312 - # load sub_GN
313 - for name, loaded_weight in weights_group["sub_GN"]:
314 - assert name == ""
315 - param = self.sub_GN
316 - weight_loader = getattr(param, "weight_loader",
317 - default_weight_loader)
318 - weight_loader(param, loaded_weight)
319 -
320 - # load mlp projector
321 - mlp_params_dict = dict(self.img_projection.named_parameters())
322 - for name, loaded_weight in weights_group["img_projection"]:
323 - param = mlp_params_dict[name]
324 - weight_loader = getattr(param, "weight_loader",
325 - default_weight_loader)
326 - weight_loader(param, loaded_weight)
293 + loader = AutoWeightsLoader(self)
294 + loader.load_weights(weights)
295
296
297 # Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57
@@ -715,27 +683,12 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
683 return self.language_model.sample(logits, sampling_metadata)
684
685 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
718 - hf_to_vllm_mapping = {
719 - "model.vision_embed_tokens.": "vision_embed_tokens.",
720 - "lm_head.": "language_model.lm_head.",
721 - "model.": "language_model.model.",
722 - }
723 -
724 - def hf_to_vllm_name(key: str) -> str:
725 - for hf_name, vllm_name in hf_to_vllm_mapping.items():
726 - if key.startswith(hf_name):
727 - return key.replace(hf_name, vllm_name, 1)
728 -
729 - return key
730 -
731 - vllm_weights = {hf_to_vllm_name(k): v for k, v in weights}
732 -
733 - # prepare weight iterators for components
734 - weights_group = group_weights_with_prefix(vllm_weights.items())
735 -
736 - # load vision embeddings and encoder
737 - self.vision_embed_tokens.load_weights(
738 - weights_group["vision_embed_tokens"])
739 -
740 - # load llm backbone
741 - self.language_model.load_weights(weights_group["language_model"])
686 + hf_to_vllm_mapper = WeightsMapper(
687 + orig_to_new_prefix={
688 + "model.vision_embed_tokens.": "vision_embed_tokens.",
689 + "lm_head.": "language_model.lm_head.",
690 + "model.": "language_model.model.",
691 + })
692 +
693 + loader = AutoWeightsLoader(self)
694 + loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/qwen2.py
+7 -16
@@ -48,8 +48,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA, SupportsPP
51 -from .utils import (PPMissingLayer, group_weights_with_prefix,
52 - is_pp_missing_parameter,
51 +from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
52 make_empty_intermediate_tensors_factory, make_layers)
53
54
@@ -435,17 +434,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
434 return next_tokens
435
436 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
438 - weights_group = group_weights_with_prefix(weights)
439 -
440 - self.model.load_weights(weights_group["model"])
441 -
442 - if not self.config.tie_word_embeddings:
443 - lm_head_dict = dict(self.lm_head.named_parameters())
444 - for name, loaded_weight in weights_group["lm_head"]:
445 - if is_pp_missing_parameter(name, self.lm_head):
446 - continue
447 -
448 - param = lm_head_dict[name]
449 - weight_loader = getattr(param, "weight_loader",
450 - default_weight_loader)
451 - weight_loader(param, loaded_weight)
437 + loader = AutoWeightsLoader(
438 + self,
439 + skip_prefixes=(["lm_head."]
440 + if self.config.tie_word_embeddings else None),
441 + )
442 + loader.load_weights(weights)
vllm/model_executor/models/qwen2_rm.py
+3 -12
@@ -16,13 +16,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 RowParallelLinear)
17 from vllm.model_executor.layers.pooler import Pooler, PoolingType
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19 from vllm.model_executor.pooling_metadata import PoolingMetadata
20 from vllm.sequence import IntermediateTensors, PoolerOutput
21
22 from .interfaces import SupportsPP
23 from .qwen2 import Qwen2Model
25 -from .utils import group_weights_with_prefix
24 +from .utils import AutoWeightsLoader
25
26
27 class ReLU(nn.Module):
@@ -120,13 +119,5 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
119 return self._pooler(hidden_states, pooling_metadata)
120
121 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
123 - weights_group = group_weights_with_prefix(weights)
124 -
125 - self.model.load_weights(weights_group["model"])
126 -
127 - score_dict = dict(self.score.named_parameters())
128 - for name, loaded_weight in weights_group["score"]:
129 - param = score_dict[name]
130 - weight_loader = getattr(param, "weight_loader",
131 - default_weight_loader)
132 - weight_loader(param, loaded_weight)
122 + loader = AutoWeightsLoader(self)
123 + loader.load_weights(weights)
vllm/model_executor/models/ultravox.py
+8 -32
@@ -25,11 +25,6 @@ from vllm.model_executor.layers.layernorm import RMSNorm
25 from vllm.model_executor.layers.quantization import QuantizationConfig
26 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
27 from vllm.model_executor.model_loader.loader import DefaultModelLoader
28 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 -from vllm.model_executor.models.utils import (flatten_bn,
30 - group_weights_with_prefix,
31 - init_vllm_registered_model,
32 - merge_multimodal_embeddings)
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 from vllm.multimodal import MULTIMODAL_REGISTRY
30 from vllm.multimodal.base import MultiModalInputs, NestedTensors
@@ -41,6 +36,8 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
36 from vllm.utils import is_list_of
37
38 from .interfaces import SupportsMultiModal, SupportsPP
39 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
40 + init_vllm_registered_model, merge_multimodal_embeddings)
41
42 _AUDIO_PLACEHOLDER_TOKEN = 128002
43 _AUDIO_TOKENS_PER_SECOND = 6.25
@@ -498,30 +495,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
495 return self.language_model.sample(logits, sampling_metadata)
496
497 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
501 - # prepare weight iterators for components
502 - weights_group = group_weights_with_prefix(weights)
503 -
504 - # load audio tower weights
505 - audio_tower_weights = weights_group["audio_tower"]
506 - audio_tower_params_dict = dict(
507 - self.audio_tower.named_parameters(
508 - prefix=self.audio_tower.base_model_prefix))
509 - for name, loaded_weight in audio_tower_weights:
510 - if name in audio_tower_params_dict:
511 - param = audio_tower_params_dict[name]
512 - weight_loader = getattr(param, "weight_loader",
513 - default_weight_loader)
514 - weight_loader(param, loaded_weight)
515 -
516 - # load projector weights
517 - projector_weights = weights_group["multi_modal_projector"]
518 - projector_params_dict = dict(
519 - self.multi_modal_projector.named_parameters())
520 - for name, loaded_weight in projector_weights:
521 - param = projector_params_dict[name]
522 - weight_loader = getattr(param, "weight_loader",
523 - default_weight_loader)
524 - weight_loader(param, loaded_weight)
525 -
526 - # load llm backbone
527 - self.language_model.load_weights(weights_group["language_model"])
498 + hf_to_vllm_mapper = WeightsMapper(
499 + orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
500 +
501 + loader = AutoWeightsLoader(self,
502 + ignore_unexpected_prefixes=["audio_tower."])
503 + loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/utils.py
+167 -38
@@ -1,7 +1,7 @@
1 import itertools
2 -from collections import UserDict
3 -from typing import (Any, Dict, Iterable, List, Literal, Optional, Protocol,
4 - Tuple, Union, overload)
2 +from dataclasses import dataclass, field
3 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
4 + Protocol, Tuple, Union, overload)
5
6 import torch
7 import torch.nn as nn
@@ -12,55 +12,184 @@ from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
12 SchedulerConfig)
13 from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.model_loader.loader import build_model
15 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.models import ModelRegistry
17 from vllm.multimodal.base import NestedTensors
18 from vllm.sequence import IntermediateTensors
19 from vllm.utils import is_pin_memory_available
20
21 +WeightsMapping = Mapping[str, Optional[str]]
22 +"""If a key maps to a value of `None`, the corresponding weight is ignored."""
23
21 -class WeightsGroup(UserDict):
22 - """
23 - Wraps grouped weights dictionary for a more informative error message
24 - when attempting to access a weight component that does not exist.
25 - """
24
27 - def __getitem__(self, key: str) -> Iterable[Tuple[str, torch.Tensor]]:
28 - try:
29 - return super().__getitem__(key)
30 - except KeyError as exc:
31 - msg = (f"There is no weights named with the prefix: {key}. "
32 - f"Available prefix: {set(self.keys())}")
33 - raise KeyError(msg) from exc
25 +@dataclass
26 +class WeightsMapper:
27 + """Maps the name of each weight if they match the following patterns."""
28
29 + orig_to_new_substr: WeightsMapping = field(default_factory=dict)
30 + orig_to_new_prefix: WeightsMapping = field(default_factory=dict)
31 + orig_to_new_suffix: WeightsMapping = field(default_factory=dict)
32
36 -def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]],
37 - prefix: str) -> Iterable[Tuple[str, torch.Tensor]]:
38 - """
39 - Helper function to load weights for inner vLLM models.
33 + def _map_name(self, key: str) -> Optional[str]:
34 + for substr, new_key in self.orig_to_new_substr.items():
35 + if substr in key:
36 + if new_key is None:
37 + return None
38
41 - See also:
42 - :ref:`init_vllm_registered_model`
43 - """
44 - for name, loaded_weight in weights:
45 - name = name.split(".")
46 - if prefix == name.pop(0):
47 - name = ".".join(name)
48 - yield name, loaded_weight
39 + key = key.replace(substr, new_key, 1)
40 +
41 + for prefix, new_key in self.orig_to_new_prefix.items():
42 + if key.startswith(prefix):
43 + if new_key is None:
44 + return None
45 +
46 + key = key.replace(prefix, new_key, 1)
47 +
48 + for suffix, new_key in self.orig_to_new_suffix.items():
49 + if key.endswith(suffix):
50 + if new_key is None:
51 + return None
52 +
53 + key = new_key.join(key.rsplit(suffix, 1))
54
55 + return key
56
51 -def group_weights_with_prefix(
52 - weights: Iterable[Tuple[str, torch.Tensor]], ) -> WeightsGroup:
57 + def apply(
58 + self, weights: Iterable[Tuple[str, torch.Tensor]]
59 + ) -> Iterable[Tuple[str, torch.Tensor]]:
60 + return ((out_name, data) for name, data in weights
61 + if (out_name := self._map_name(name)) is not None)
62 +
63 +
64 +class AutoWeightsLoader:
65 """
54 - Helper function to group weights with prefix
66 + Helper class to load weights into a :class:`torch.nn.Module`. It is able
67 + to automatically detect child modules and parameters while iterating over
68 + the weights only once.
69 +
70 + The weight loading logic for individual modules can be overridden
71 + by defining a ``load_weights`` method.
72 +
73 + Similarly, the weight loading logic for individual parameters can be
74 + overridden by defining a ``weight_loader`` method.
75 """
56 - init_weights, repeated_weights = itertools.tee(weights, 2)
57 - weights_prefix = {name.split(".")[0] for name, _ in init_weights}
58 - repeated_weights = itertools.tee(repeated_weights, len(weights_prefix))
59 -
60 - return WeightsGroup({
61 - prefix: filter_weights(component, prefix)
62 - for component, prefix in zip(repeated_weights, weights_prefix)
63 - })
76 +
77 + def __init__(
78 + self,
79 + module: nn.Module,
80 + *,
81 + skip_prefixes: Optional[List[str]] = None,
82 + ignore_unexpected_prefixes: Optional[List[str]] = None,
83 + ) -> None:
84 + super().__init__()
85 +
86 + self.module = module
87 + self.skip_prefixes = skip_prefixes or []
88 + self.ignore_unexpected_prefixes = ignore_unexpected_prefixes or []
89 +
90 + def _groupby_prefix(
91 + self,
92 + weights: Iterable[Tuple[str, torch.Tensor]],
93 + ) -> Iterable[Tuple[str, Iterable[Tuple[str, torch.Tensor]]]]:
94 + weights_by_parts = ((weight_name.split(".", 1), weight_data)
95 + for weight_name, weight_data in weights)
96 +
97 + for prefix, group in itertools.groupby(weights_by_parts,
98 + key=lambda x: x[0][0]):
99 + yield (
100 + prefix,
101 + # Because maxsplit=1 in weight_name.split(...),
102 + # the length of `parts` must either be 1 or 2
103 + (("" if len(parts) == 1 else parts[1], weights_data)
104 + for parts, weights_data in group),
105 + )
106 +
107 + def _get_qualname(self, prefix: str, rest: str) -> str:
108 + if prefix == "":
109 + return rest
110 + if rest == "":
111 + return prefix
112 +
113 + return ".".join((prefix, rest))
114 +
115 + def _can_skip(self, qualname: str) -> bool:
116 + return any(qualname.startswith(p) for p in self.skip_prefixes)
117 +
118 + def _can_ignore_unexpected(self, qualname: str) -> bool:
119 + return any(
120 + qualname.startswith(p) for p in self.ignore_unexpected_prefixes)
121 +
122 + def _load_param(
123 + self,
124 + base_prefix: str,
125 + param: nn.Parameter,
126 + weights: Iterable[Tuple[str, torch.Tensor]],
127 + ) -> None:
128 + for weight_name, weight_data in weights:
129 + weight_qualname = self._get_qualname(base_prefix, weight_name)
130 +
131 + if self._can_skip(weight_qualname):
132 + continue
133 +
134 + if weight_name != "":
135 + if not self._can_ignore_unexpected(weight_qualname):
136 + raise ValueError(
137 + f"Attempted to load nested weight '{weight_qualname}' "
138 + f"into a single parameter '{base_prefix}'")
139 +
140 + continue
141 +
142 + weight_loader = getattr(param, "weight_loader",
143 + default_weight_loader)
144 + weight_loader(param, weight_data)
145 +
146 + def _load_module(
147 + self,
148 + base_prefix: str,
149 + module: nn.Module,
150 + weights: Iterable[Tuple[str, torch.Tensor]],
151 + ) -> None:
152 + if isinstance(module, PPMissingLayer):
153 + return
154 +
155 + # Avoid infinite recursion since this function is typically
156 + # called inside load_weights of the module itself
157 + if module != self.module:
158 + module_load_weights = getattr(module, "load_weights", None)
159 + if callable(module_load_weights):
160 + module_load_weights(weights)
161 + return
162 +
163 + child_modules = dict(module.named_children())
164 + child_params = dict(module.named_parameters(recurse=False))
165 +
166 + for child_prefix, child_weights in self._groupby_prefix(weights):
167 + prefix = self._get_qualname(base_prefix, child_prefix)
168 +
169 + if self._can_skip(prefix):
170 + continue
171 +
172 + if child_prefix in child_modules:
173 + self._load_module(prefix, child_modules[child_prefix],
174 + child_weights)
175 + elif child_prefix in child_params:
176 + self._load_param(prefix, child_params[child_prefix],
177 + child_weights)
178 + else:
179 + if not self._can_ignore_unexpected(prefix):
180 + msg = f"There is no module or parameter named '{prefix}'"
181 + raise ValueError(msg)
182 +
183 + def load_weights(
184 + self,
185 + weights: Iterable[Tuple[str, torch.Tensor]],
186 + *,
187 + mapper: Optional[WeightsMapper] = None,
188 + ) -> None:
189 + if mapper is not None:
190 + weights = mapper.apply(weights)
191 +
192 + self._load_module("", self.module, weights)
193
194
195 def init_vllm_registered_model(