A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] fix composite weight loading and EAGLE weight loading (#9160)
Cyrus Leung committed
Oct 9, 2024 at 15:36 UTC
8bfaa4e31eb63d41499fec933e68969ebbedb01f
15 files changed
+244
-364
vllm/model_executor/models/blip2.py
+3
-34
@@ -13,7 +13,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.sequence import IntermediateTensors, SequenceData
@@ -21,7 +20,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
20
from .blip import (BlipVisionModel, dummy_image_for_blip,
21
get_max_blip_image_tokens)
22
from .interfaces import SupportsMultiModal, SupportsPP
24
-from .utils import (group_weights_with_prefix, init_vllm_registered_model,
23
+from .utils import (AutoWeightsLoader, init_vllm_registered_model,
24
merge_multimodal_embeddings)
25
26
# We use this internally as placeholders since there is no image token
@@ -687,35 +686,5 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
686
return self.language_model.sample(logits, sampling_metadata)
687
688
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
690
- # prepare weight iterators for components
691
- weights_group = group_weights_with_prefix(weights)
692
-
693
- # load vision encoder
694
- self.vision_model.load_weights(weights_group["vision_model"])
695
-
696
- # load query tokens
697
- for name, loaded_weight in weights_group["query_tokens"]:
698
- assert name == ""
699
- param = self.query_tokens
700
- weight_loader = getattr(param, "weight_loader",
701
- default_weight_loader)
702
- weight_loader(param, loaded_weight)
703
-
704
- # load qformer
705
- qformer_params_dict = dict(self.qformer.named_parameters())
706
- for name, loaded_weight in weights_group["qformer"]:
707
- param = qformer_params_dict[name]
708
- weight_loader = getattr(param, "weight_loader",
709
- default_weight_loader)
710
- weight_loader(param, loaded_weight)
711
-
712
- # load mlp projector
713
- mlp_params_dict = dict(self.language_projection.named_parameters())
714
- for name, loaded_weight in weights_group["language_projection"]:
715
- param = mlp_params_dict[name]
716
- weight_loader = getattr(param, "weight_loader",
717
- default_weight_loader)
718
- weight_loader(param, loaded_weight)
719
-
720
- # load llm backbone
721
- self.language_model.load_weights(weights_group["language_model"])
689
+ loader = AutoWeightsLoader(self)
690
+ loader.load_weights(weights)
vllm/model_executor/models/fuyu.py
+3
-16
@@ -31,7 +31,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31
from vllm.model_executor.layers.linear import ColumnParallelLinear
32
from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.model_executor.layers.sampler import SamplerOutput
34
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -42,8 +41,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
41
SequenceData)
42
43
from .interfaces import SupportsMultiModal, SupportsPP
45
-from .utils import (flatten_bn, group_weights_with_prefix,
46
- merge_multimodal_embeddings)
44
+from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
45
46
# Cannot find the following 2 numbers from hf config.
47
_IMAGE_TOKEN_ID = 71011
@@ -349,16 +347,5 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
347
return next_tokens
348
349
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
352
- # prepare weight iterators for components
353
- weights_group = group_weights_with_prefix(weights)
354
-
355
- # load vision embeddings
356
- vision_params_dict = dict(self.vision_embed_tokens.named_parameters())
357
- for name, loaded_weight in weights_group["vision_embed_tokens"]:
358
- param = vision_params_dict[name]
359
- weight_loader = getattr(param, "weight_loader",
360
- default_weight_loader)
361
- weight_loader(param, loaded_weight)
362
-
363
- # load llm backbone
364
- self.language_model.load_weights(weights_group["language_model"])
350
+ loader = AutoWeightsLoader(self)
351
+ loader.load_weights(weights)
vllm/model_executor/models/gemma2.py
+7
-17
@@ -40,7 +40,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.sequence import IntermediateTensors
41
42
from .interfaces import SupportsLoRA, SupportsPP
43
-from .utils import (group_weights_with_prefix, is_pp_missing_parameter,
43
+from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
44
make_empty_intermediate_tensors_factory, make_layers)
45
46
logger = init_logger(__name__)
@@ -447,19 +447,9 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
447
return next_tokens
448
449
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
450
- weights_group = group_weights_with_prefix(weights)
451
-
452
- self.model.load_weights(weights_group["model"])
453
-
454
- if not self.config.tie_word_embeddings:
455
- # NOTE: For now self.lm_head is not defined because
456
- # tie_word_embeddings is assumed to the False
457
- lm_head_dict = dict(self.lm_head.named_parameters())
458
- for name, loaded_weight in weights_group["lm_head"]:
459
- if is_pp_missing_parameter(name, self.lm_head):
460
- continue
461
-
462
- param = lm_head_dict[name]
463
- weight_loader = getattr(param, "weight_loader",
464
- default_weight_loader)
465
- weight_loader(param, loaded_weight)
450
+ loader = AutoWeightsLoader(
451
+ self,
452
+ skip_prefixes=(["lm_head."]
453
+ if self.config.tie_word_embeddings else None),
454
+ )
455
+ loader.load_weights(weights)
vllm/model_executor/models/internvl.py
+4
-19
@@ -20,7 +20,6 @@ from vllm.config import CacheConfig, MultiModalConfig
20
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.models.intern_vit import InternVisionModel
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -32,8 +31,8 @@ from vllm.utils import is_list_of
31
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
32
get_clip_num_patches)
33
from .interfaces import SupportsMultiModal, SupportsPP
35
-from .utils import (flatten_bn, group_weights_with_prefix,
36
- init_vllm_registered_model, merge_multimodal_embeddings)
34
+from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
35
+ merge_multimodal_embeddings)
36
37
IMG_START = '<img>'
38
IMG_END = '</img>'
@@ -609,19 +608,5 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
608
return self.language_model.sample(logits, sampling_metadata)
609
610
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
612
- # prepare weight iterators for components
613
- weights_group = group_weights_with_prefix(weights)
614
-
615
- # load vision encoder
616
- self.vision_model.load_weights(weights_group["vision_model"])
617
-
618
- # load mlp projector
619
- mlp_params_dict = dict(self.mlp1.named_parameters())
620
- for name, loaded_weight in weights_group["mlp1"]:
621
- param = mlp_params_dict[name]
622
- weight_loader = getattr(param, "weight_loader",
623
- default_weight_loader)
624
- weight_loader(param, loaded_weight)
625
-
626
- # load llm backbone
627
- self.language_model.load_weights(weights_group["language_model"])
611
+ loader = AutoWeightsLoader(self)
612
+ loader.load_weights(weights)
vllm/model_executor/models/llama.py
+8
-20
@@ -51,8 +51,7 @@ from vllm.sequence import IntermediateTensors
51
from vllm.utils import is_hip
52
53
from .interfaces import SupportsLoRA, SupportsPP
54
-from .utils import (PPMissingLayer, group_weights_with_prefix,
55
- is_pp_missing_parameter,
54
+from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
55
make_empty_intermediate_tensors_factory, make_layers)
56
57
@@ -564,25 +563,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
563
return next_tokens
564
565
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
567
- weights = [
566
+ loader = AutoWeightsLoader(
567
+ self,
568
+ skip_prefixes=(["lm_head."]
569
+ if self.config.tie_word_embeddings else None),
570
+ )
571
+ loader.load_weights(
572
self.maybe_remap_mistral(name, loaded_weight)
569
- for name, loaded_weight in weights
570
- ]
571
-
572
- weights_group = group_weights_with_prefix(weights)
573
-
574
- self.model.load_weights(weights_group["model"])
575
-
576
- if not self.config.tie_word_embeddings:
577
- lm_head_dict = dict(self.lm_head.named_parameters())
578
- for name, loaded_weight in weights_group["lm_head"]:
579
- if is_pp_missing_parameter(name, self.lm_head):
580
- continue
581
-
582
- param = lm_head_dict[name]
583
- weight_loader = getattr(param, "weight_loader",
584
- default_weight_loader)
585
- weight_loader(param, loaded_weight)
573
+ for name, loaded_weight in weights)
574
575
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
576
self.model.load_kv_cache_scales(quantization_param_path)
vllm/model_executor/models/llava.py
+4
-19
@@ -13,7 +13,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.sequence import IntermediateTensors
@@ -26,8 +25,8 @@ from .interfaces import SupportsMultiModal, SupportsPP
25
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
26
dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
27
input_processor_for_siglip)
29
-from .utils import (flatten_bn, group_weights_with_prefix,
30
- init_vllm_registered_model, merge_multimodal_embeddings)
28
+from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
29
+ merge_multimodal_embeddings)
30
31
32
class LlavaImagePixelInputs(TypedDict):
@@ -406,19 +405,5 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
405
return self.language_model.sample(logits, sampling_metadata)
406
407
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
409
- # prepare weight iterators for components
410
- weights_group = group_weights_with_prefix(weights)
411
-
412
- # load vision encoder
413
- self.vision_tower.load_weights(weights_group["vision_tower"])
414
-
415
- # load mlp projector
416
- mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
417
- for name, loaded_weight in weights_group["multi_modal_projector"]:
418
- param = mlp_params_dict[name]
419
- weight_loader = getattr(param, "weight_loader",
420
- default_weight_loader)
421
- weight_loader(param, loaded_weight)
422
-
423
- # load llm backbone
424
- self.language_model.load_weights(weights_group["language_model"])
408
+ loader = AutoWeightsLoader(self)
409
+ loader.load_weights(weights)
vllm/model_executor/models/llava_next.py
+4
-27
@@ -15,7 +15,6 @@ from vllm.config import CacheConfig, MultiModalConfig
15
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.sequence import IntermediateTensors
@@ -29,8 +28,8 @@ from .llava import LlavaMultiModalProjector
28
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
30
get_siglip_patch_grid_length, input_processor_for_siglip)
32
-from .utils import (flatten_bn, group_weights_with_prefix,
33
- init_vllm_registered_model, merge_multimodal_embeddings)
31
+from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
32
+ merge_multimodal_embeddings)
33
34
# Result in the max possible feature size (2x2 grid of 336x336px tiles)
35
MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
@@ -642,27 +641,5 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
641
return self.language_model.sample(logits, sampling_metadata)
642
643
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
645
- # prepare weight iterators for components
646
- weights_group = group_weights_with_prefix(weights)
647
-
648
- # load vision encoder
649
- self.vision_tower.load_weights(weights_group["vision_tower"])
650
-
651
- # load mlp projector
652
- mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
653
- for name, loaded_weight in weights_group["multi_modal_projector"]:
654
- param = mlp_params_dict[name]
655
- weight_loader = getattr(param, "weight_loader",
656
- default_weight_loader)
657
- weight_loader(param, loaded_weight)
658
-
659
- # load newline
660
- for name, loaded_weight in weights_group["image_newline"]:
661
- assert name == ""
662
- param = self.image_newline
663
- weight_loader = getattr(param, "weight_loader",
664
- default_weight_loader)
665
- weight_loader(param, loaded_weight)
666
-
667
- # load llm backbone
668
- self.language_model.load_weights(weights_group["language_model"])
644
+ loader = AutoWeightsLoader(self)
645
+ loader.load_weights(weights)
vllm/model_executor/models/llava_next_video.py
+7
-18
@@ -15,7 +15,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15
from vllm.model_executor.layers.activation import get_act_fn
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.model_executor.models.clip import CLIPVisionModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -28,7 +27,7 @@ from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
27
from .interfaces import SupportsMultiModal, SupportsPP
28
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29
dummy_seq_data_for_siglip)
31
-from .utils import (group_weights_with_prefix, init_vllm_registered_model,
30
+from .utils import (AutoWeightsLoader, init_vllm_registered_model,
31
merge_multimodal_embeddings)
32
33
# For profile run
@@ -458,19 +457,9 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
457
return self.language_model.sample(logits, sampling_metadata)
458
459
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
461
- # prepare weight iterators for components
462
- weights_group = group_weights_with_prefix(weights)
463
-
464
- # load vision encoder
465
- self.vision_tower.load_weights(weights_group["vision_tower"])
466
-
467
- # load mlp projector
468
- mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
469
- for name, loaded_weight in weights_group["multi_modal_projector"]:
470
- param = mlp_params_dict[name]
471
- weight_loader = getattr(param, "weight_loader",
472
- default_weight_loader)
473
- weight_loader(param, loaded_weight)
474
-
475
- # load llm backbone
476
- self.language_model.load_weights(weights_group["language_model"])
460
+ loader = AutoWeightsLoader(
461
+ self,
462
+ # This model doesn't support images for now
463
+ ignore_unexpected_prefixes=["image_newline"],
464
+ )
465
+ loader.load_weights(weights)
vllm/model_executor/models/llava_onevision.py
+4
-19
@@ -20,7 +20,6 @@ from vllm.logger import init_logger
20
from vllm.model_executor.layers.activation import get_act_fn
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.utils import (cached_get_tokenizer,
@@ -35,8 +34,8 @@ from .interfaces import SupportsMultiModal, SupportsPP
34
from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
35
dummy_video_for_siglip, get_siglip_image_feature_size,
36
get_siglip_patch_grid_length, input_processor_for_siglip)
38
-from .utils import (flatten_bn, group_weights_with_prefix,
39
- init_vllm_registered_model, merge_multimodal_embeddings)
37
+from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38
+ merge_multimodal_embeddings)
39
40
logger = init_logger(__name__)
41
@@ -872,19 +871,5 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
871
return self.language_model.sample(logits, sampling_metadata)
872
873
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
875
- # prepare weight iterators for components
876
- weights_group = group_weights_with_prefix(weights)
877
-
878
- # load vision encoder
879
- self.vision_tower.load_weights(weights_group["vision_tower"])
880
-
881
- # load mlp projector
882
- mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
883
- for name, loaded_weight in weights_group["multi_modal_projector"]:
884
- param = mlp_params_dict[name]
885
- weight_loader = getattr(param, "weight_loader",
886
- default_weight_loader)
887
- weight_loader(param, loaded_weight)
888
-
889
- # load llm backbone
890
- self.language_model.load_weights(weights_group["language_model"])
874
+ loader = AutoWeightsLoader(self)
875
+ loader.load_weights(weights)
vllm/model_executor/models/paligemma.py
+3
-18
@@ -11,7 +11,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11
from vllm.logger import init_logger
12
from vllm.model_executor.layers.quantization import QuantizationConfig
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
14
from vllm.model_executor.models.gemma import GemmaForCausalLM
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -21,7 +20,7 @@ from vllm.sequence import IntermediateTensors
20
from .interfaces import SupportsMultiModal, SupportsPP
21
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
22
dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
24
-from .utils import group_weights_with_prefix, merge_multimodal_embeddings
23
+from .utils import AutoWeightsLoader, merge_multimodal_embeddings
24
25
logger = init_logger(__name__)
26
@@ -292,19 +291,5 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
291
return self.language_model.sample(logits, sampling_metadata)
292
293
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
295
- # prepare weight iterators for components
296
- weights_group = group_weights_with_prefix(weights)
297
-
298
- # load vision tower
299
- self.vision_tower.load_weights(weights_group["vision_tower"])
300
-
301
- # load mlp projector
302
- mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
303
- for name, loaded_weight in weights_group["multi_modal_projector"]:
304
- param = mlp_params_dict[name]
305
- weight_loader = getattr(param, "weight_loader",
306
- default_weight_loader)
307
- weight_loader(param, loaded_weight)
308
-
309
- # load llm backbone
310
- self.language_model.load_weights(weights_group["language_model"])
294
+ loader = AutoWeightsLoader(self)
295
+ loader.load_weights(weights)
vllm/model_executor/models/phi3v.py
+12
-59
@@ -31,7 +31,6 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31
from vllm.logger import init_logger
32
from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
34
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.model_executor.models.clip import CLIPVisionModel
35
from vllm.model_executor.models.llama import LlamaForCausalLM
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -42,15 +41,11 @@ from vllm.utils import is_list_of
41
42
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
43
from .interfaces import SupportsMultiModal, SupportsPP
45
-from .utils import (flatten_bn, group_weights_with_prefix,
44
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
45
merge_multimodal_embeddings)
46
47
logger = init_logger(__name__)
48
50
-_KEYS_TO_MODIFY_MAPPING = {
51
- "model.vision_embed_tokens": "vision_embed_tokens",
52
-}
53
-
49
# Cannot find the following 2 numbers from hf config.
50
_IMAGE_TOKEN_ID = 32044
51
@@ -295,35 +290,8 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
290
return image_features_hd_newline
291
292
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
298
- # prepare weight iterators for components
299
- weights_group = group_weights_with_prefix(weights)
300
-
301
- # load vision encoder
302
- self.img_processor.load_weights(weights_group["img_processor"])
303
-
304
- # load glb_GN
305
- for name, loaded_weight in weights_group["glb_GN"]:
306
- assert name == ""
307
- param = self.glb_GN
308
- weight_loader = getattr(param, "weight_loader",
309
- default_weight_loader)
310
- weight_loader(param, loaded_weight)
311
-
312
- # load sub_GN
313
- for name, loaded_weight in weights_group["sub_GN"]:
314
- assert name == ""
315
- param = self.sub_GN
316
- weight_loader = getattr(param, "weight_loader",
317
- default_weight_loader)
318
- weight_loader(param, loaded_weight)
319
-
320
- # load mlp projector
321
- mlp_params_dict = dict(self.img_projection.named_parameters())
322
- for name, loaded_weight in weights_group["img_projection"]:
323
- param = mlp_params_dict[name]
324
- weight_loader = getattr(param, "weight_loader",
325
- default_weight_loader)
326
- weight_loader(param, loaded_weight)
293
+ loader = AutoWeightsLoader(self)
294
+ loader.load_weights(weights)
295
296
297
# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57
@@ -715,27 +683,12 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
683
return self.language_model.sample(logits, sampling_metadata)
684
685
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
718
- hf_to_vllm_mapping = {
719
- "model.vision_embed_tokens.": "vision_embed_tokens.",
720
- "lm_head.": "language_model.lm_head.",
721
- "model.": "language_model.model.",
722
- }
723
-
724
- def hf_to_vllm_name(key: str) -> str:
725
- for hf_name, vllm_name in hf_to_vllm_mapping.items():
726
- if key.startswith(hf_name):
727
- return key.replace(hf_name, vllm_name, 1)
728
-
729
- return key
730
-
731
- vllm_weights = {hf_to_vllm_name(k): v for k, v in weights}
732
-
733
- # prepare weight iterators for components
734
- weights_group = group_weights_with_prefix(vllm_weights.items())
735
-
736
- # load vision embeddings and encoder
737
- self.vision_embed_tokens.load_weights(
738
- weights_group["vision_embed_tokens"])
739
-
740
- # load llm backbone
741
- self.language_model.load_weights(weights_group["language_model"])
686
+ hf_to_vllm_mapper = WeightsMapper(
687
+ orig_to_new_prefix={
688
+ "model.vision_embed_tokens.": "vision_embed_tokens.",
689
+ "lm_head.": "language_model.lm_head.",
690
+ "model.": "language_model.model.",
691
+ })
692
+
693
+ loader = AutoWeightsLoader(self)
694
+ loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/qwen2.py
+7
-16
@@ -48,8 +48,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA, SupportsPP
51
-from .utils import (PPMissingLayer, group_weights_with_prefix,
52
- is_pp_missing_parameter,
51
+from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
52
make_empty_intermediate_tensors_factory, make_layers)
53
54
@@ -435,17 +434,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
434
return next_tokens
435
436
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
438
- weights_group = group_weights_with_prefix(weights)
439
-
440
- self.model.load_weights(weights_group["model"])
441
-
442
- if not self.config.tie_word_embeddings:
443
- lm_head_dict = dict(self.lm_head.named_parameters())
444
- for name, loaded_weight in weights_group["lm_head"]:
445
- if is_pp_missing_parameter(name, self.lm_head):
446
- continue
447
-
448
- param = lm_head_dict[name]
449
- weight_loader = getattr(param, "weight_loader",
450
- default_weight_loader)
451
- weight_loader(param, loaded_weight)
437
+ loader = AutoWeightsLoader(
438
+ self,
439
+ skip_prefixes=(["lm_head."]
440
+ if self.config.tie_word_embeddings else None),
441
+ )
442
+ loader.load_weights(weights)
vllm/model_executor/models/qwen2_rm.py
+3
-12
@@ -16,13 +16,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
RowParallelLinear)
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19
from vllm.model_executor.pooling_metadata import PoolingMetadata
20
from vllm.sequence import IntermediateTensors, PoolerOutput
21
22
from .interfaces import SupportsPP
23
from .qwen2 import Qwen2Model
25
-from .utils import group_weights_with_prefix
24
+from .utils import AutoWeightsLoader
25
26
27
class ReLU(nn.Module):
@@ -120,13 +119,5 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
119
return self._pooler(hidden_states, pooling_metadata)
120
121
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
123
- weights_group = group_weights_with_prefix(weights)
124
-
125
- self.model.load_weights(weights_group["model"])
126
-
127
- score_dict = dict(self.score.named_parameters())
128
- for name, loaded_weight in weights_group["score"]:
129
- param = score_dict[name]
130
- weight_loader = getattr(param, "weight_loader",
131
- default_weight_loader)
132
- weight_loader(param, loaded_weight)
122
+ loader = AutoWeightsLoader(self)
123
+ loader.load_weights(weights)
vllm/model_executor/models/ultravox.py
+8
-32
@@ -25,11 +25,6 @@ from vllm.model_executor.layers.layernorm import RMSNorm
25
from vllm.model_executor.layers.quantization import QuantizationConfig
26
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
27
from vllm.model_executor.model_loader.loader import DefaultModelLoader
28
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29
-from vllm.model_executor.models.utils import (flatten_bn,
30
- group_weights_with_prefix,
31
- init_vllm_registered_model,
32
- merge_multimodal_embeddings)
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.base import MultiModalInputs, NestedTensors
@@ -41,6 +36,8 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
36
from vllm.utils import is_list_of
37
38
from .interfaces import SupportsMultiModal, SupportsPP
39
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
40
+ init_vllm_registered_model, merge_multimodal_embeddings)
41
42
_AUDIO_PLACEHOLDER_TOKEN = 128002
43
_AUDIO_TOKENS_PER_SECOND = 6.25
@@ -498,30 +495,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
495
return self.language_model.sample(logits, sampling_metadata)
496
497
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
501
- # prepare weight iterators for components
502
- weights_group = group_weights_with_prefix(weights)
503
-
504
- # load audio tower weights
505
- audio_tower_weights = weights_group["audio_tower"]
506
- audio_tower_params_dict = dict(
507
- self.audio_tower.named_parameters(
508
- prefix=self.audio_tower.base_model_prefix))
509
- for name, loaded_weight in audio_tower_weights:
510
- if name in audio_tower_params_dict:
511
- param = audio_tower_params_dict[name]
512
- weight_loader = getattr(param, "weight_loader",
513
- default_weight_loader)
514
- weight_loader(param, loaded_weight)
515
-
516
- # load projector weights
517
- projector_weights = weights_group["multi_modal_projector"]
518
- projector_params_dict = dict(
519
- self.multi_modal_projector.named_parameters())
520
- for name, loaded_weight in projector_weights:
521
- param = projector_params_dict[name]
522
- weight_loader = getattr(param, "weight_loader",
523
- default_weight_loader)
524
- weight_loader(param, loaded_weight)
525
-
526
- # load llm backbone
527
- self.language_model.load_weights(weights_group["language_model"])
498
+ hf_to_vllm_mapper = WeightsMapper(
499
+ orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
500
+
501
+ loader = AutoWeightsLoader(self,
502
+ ignore_unexpected_prefixes=["audio_tower."])
503
+ loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/utils.py
+167
-38
@@ -1,7 +1,7 @@
1
import itertools
2
-from collections import UserDict
3
-from typing import (Any, Dict, Iterable, List, Literal, Optional, Protocol,
4
- Tuple, Union, overload)
2
+from dataclasses import dataclass, field
3
+from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
4
+ Protocol, Tuple, Union, overload)
5
6
import torch
7
import torch.nn as nn
@@ -12,55 +12,184 @@ from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
12
SchedulerConfig)
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.model_loader.loader import build_model
15
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.models import ModelRegistry
17
from vllm.multimodal.base import NestedTensors
18
from vllm.sequence import IntermediateTensors
19
from vllm.utils import is_pin_memory_available
20
21
+WeightsMapping = Mapping[str, Optional[str]]
22
+"""If a key maps to a value of `None`, the corresponding weight is ignored."""
23
21
-class WeightsGroup(UserDict):
22
- """
23
- Wraps grouped weights dictionary for a more informative error message
24
- when attempting to access a weight component that does not exist.
25
- """
24
27
- def __getitem__(self, key: str) -> Iterable[Tuple[str, torch.Tensor]]:
28
- try:
29
- return super().__getitem__(key)
30
- except KeyError as exc:
31
- msg = (f"There is no weights named with the prefix: {key}. "
32
- f"Available prefix: {set(self.keys())}")
33
- raise KeyError(msg) from exc
25
+@dataclass
26
+class WeightsMapper:
27
+ """Maps the name of each weight if they match the following patterns."""
28
29
+ orig_to_new_substr: WeightsMapping = field(default_factory=dict)
30
+ orig_to_new_prefix: WeightsMapping = field(default_factory=dict)
31
+ orig_to_new_suffix: WeightsMapping = field(default_factory=dict)
32
36
-def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]],
37
- prefix: str) -> Iterable[Tuple[str, torch.Tensor]]:
38
- """
39
- Helper function to load weights for inner vLLM models.
33
+ def _map_name(self, key: str) -> Optional[str]:
34
+ for substr, new_key in self.orig_to_new_substr.items():
35
+ if substr in key:
36
+ if new_key is None:
37
+ return None
38
41
- See also:
42
- :ref:`init_vllm_registered_model`
43
- """
44
- for name, loaded_weight in weights:
45
- name = name.split(".")
46
- if prefix == name.pop(0):
47
- name = ".".join(name)
48
- yield name, loaded_weight
39
+ key = key.replace(substr, new_key, 1)
40
+
41
+ for prefix, new_key in self.orig_to_new_prefix.items():
42
+ if key.startswith(prefix):
43
+ if new_key is None:
44
+ return None
45
+
46
+ key = key.replace(prefix, new_key, 1)
47
+
48
+ for suffix, new_key in self.orig_to_new_suffix.items():
49
+ if key.endswith(suffix):
50
+ if new_key is None:
51
+ return None
52
+
53
+ key = new_key.join(key.rsplit(suffix, 1))
54
55
+ return key
56
51
-def group_weights_with_prefix(
52
- weights: Iterable[Tuple[str, torch.Tensor]], ) -> WeightsGroup:
57
+ def apply(
58
+ self, weights: Iterable[Tuple[str, torch.Tensor]]
59
+ ) -> Iterable[Tuple[str, torch.Tensor]]:
60
+ return ((out_name, data) for name, data in weights
61
+ if (out_name := self._map_name(name)) is not None)
62
+
63
+
64
+class AutoWeightsLoader:
65
"""
54
- Helper function to group weights with prefix
66
+ Helper class to load weights into a :class:`torch.nn.Module`. It is able
67
+ to automatically detect child modules and parameters while iterating over
68
+ the weights only once.
69
+
70
+ The weight loading logic for individual modules can be overridden
71
+ by defining a ``load_weights`` method.
72
+
73
+ Similarly, the weight loading logic for individual parameters can be
74
+ overridden by defining a ``weight_loader`` method.
75
"""
56
- init_weights, repeated_weights = itertools.tee(weights, 2)
57
- weights_prefix = {name.split(".")[0] for name, _ in init_weights}
58
- repeated_weights = itertools.tee(repeated_weights, len(weights_prefix))
59
-
60
- return WeightsGroup({
61
- prefix: filter_weights(component, prefix)
62
- for component, prefix in zip(repeated_weights, weights_prefix)
63
- })
76
+
77
+ def __init__(
78
+ self,
79
+ module: nn.Module,
80
+ *,
81
+ skip_prefixes: Optional[List[str]] = None,
82
+ ignore_unexpected_prefixes: Optional[List[str]] = None,
83
+ ) -> None:
84
+ super().__init__()
85
+
86
+ self.module = module
87
+ self.skip_prefixes = skip_prefixes or []
88
+ self.ignore_unexpected_prefixes = ignore_unexpected_prefixes or []
89
+
90
+ def _groupby_prefix(
91
+ self,
92
+ weights: Iterable[Tuple[str, torch.Tensor]],
93
+ ) -> Iterable[Tuple[str, Iterable[Tuple[str, torch.Tensor]]]]:
94
+ weights_by_parts = ((weight_name.split(".", 1), weight_data)
95
+ for weight_name, weight_data in weights)
96
+
97
+ for prefix, group in itertools.groupby(weights_by_parts,
98
+ key=lambda x: x[0][0]):
99
+ yield (
100
+ prefix,
101
+ # Because maxsplit=1 in weight_name.split(...),
102
+ # the length of `parts` must either be 1 or 2
103
+ (("" if len(parts) == 1 else parts[1], weights_data)
104
+ for parts, weights_data in group),
105
+ )
106
+
107
+ def _get_qualname(self, prefix: str, rest: str) -> str:
108
+ if prefix == "":
109
+ return rest
110
+ if rest == "":
111
+ return prefix
112
+
113
+ return ".".join((prefix, rest))
114
+
115
+ def _can_skip(self, qualname: str) -> bool:
116
+ return any(qualname.startswith(p) for p in self.skip_prefixes)
117
+
118
+ def _can_ignore_unexpected(self, qualname: str) -> bool:
119
+ return any(
120
+ qualname.startswith(p) for p in self.ignore_unexpected_prefixes)
121
+
122
+ def _load_param(
123
+ self,
124
+ base_prefix: str,
125
+ param: nn.Parameter,
126
+ weights: Iterable[Tuple[str, torch.Tensor]],
127
+ ) -> None:
128
+ for weight_name, weight_data in weights:
129
+ weight_qualname = self._get_qualname(base_prefix, weight_name)
130
+
131
+ if self._can_skip(weight_qualname):
132
+ continue
133
+
134
+ if weight_name != "":
135
+ if not self._can_ignore_unexpected(weight_qualname):
136
+ raise ValueError(
137
+ f"Attempted to load nested weight '{weight_qualname}' "
138
+ f"into a single parameter '{base_prefix}'")
139
+
140
+ continue
141
+
142
+ weight_loader = getattr(param, "weight_loader",
143
+ default_weight_loader)
144
+ weight_loader(param, weight_data)
145
+
146
+ def _load_module(
147
+ self,
148
+ base_prefix: str,
149
+ module: nn.Module,
150
+ weights: Iterable[Tuple[str, torch.Tensor]],
151
+ ) -> None:
152
+ if isinstance(module, PPMissingLayer):
153
+ return
154
+
155
+ # Avoid infinite recursion since this function is typically
156
+ # called inside load_weights of the module itself
157
+ if module != self.module:
158
+ module_load_weights = getattr(module, "load_weights", None)
159
+ if callable(module_load_weights):
160
+ module_load_weights(weights)
161
+ return
162
+
163
+ child_modules = dict(module.named_children())
164
+ child_params = dict(module.named_parameters(recurse=False))
165
+
166
+ for child_prefix, child_weights in self._groupby_prefix(weights):
167
+ prefix = self._get_qualname(base_prefix, child_prefix)
168
+
169
+ if self._can_skip(prefix):
170
+ continue
171
+
172
+ if child_prefix in child_modules:
173
+ self._load_module(prefix, child_modules[child_prefix],
174
+ child_weights)
175
+ elif child_prefix in child_params:
176
+ self._load_param(prefix, child_params[child_prefix],
177
+ child_weights)
178
+ else:
179
+ if not self._can_ignore_unexpected(prefix):
180
+ msg = f"There is no module or parameter named '{prefix}'"
181
+ raise ValueError(msg)
182
+
183
+ def load_weights(
184
+ self,
185
+ weights: Iterable[Tuple[str, torch.Tensor]],
186
+ *,
187
+ mapper: Optional[WeightsMapper] = None,
188
+ ) -> None:
189
+ if mapper is not None:
190
+ weights = mapper.apply(weights)
191
+
192
+ self._load_module("", self.module, weights)
193
194
195
def init_vllm_registered_model(