@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 13d88d4137
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Refactor composite weight loading logic (#8656)

Isotr0py committed Sep 22, 2024 at 12:33 UTC 13d88d4137f97b8cf3c79f39d7df5e4c8348603a
7 files changed +70 -61
vllm/model_executor/models/internvl.py
+6 -10
@@ -4,7 +4,6 @@
4 # Copyright (c) 2023 OpenGVLab
5 # Licensed under The MIT License [see LICENSE for details]
6 # --------------------------------------------------------
7 -import itertools
7 import re
8 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9 TypedDict, Union)
@@ -33,8 +32,8 @@ from vllm.utils import is_list_of
32 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
33 get_clip_num_patches)
34 from .interfaces import SupportsMultiModal
36 -from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
37 - merge_multimodal_embeddings)
35 +from .utils import (flatten_bn, group_weights_with_prefix,
36 + init_vllm_registered_model, merge_multimodal_embeddings)
37
38 IMG_START = '<img>'
39 IMG_END = '</img>'
@@ -518,21 +517,18 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
517
518 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
519 # prepare weight iterators for components
521 - vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
520 + weights_group = group_weights_with_prefix(weights)
521
522 # load vision encoder
524 - vit_weights = filter_weights(vit_weights, "vision_model")
525 - self.vision_model.load_weights(vit_weights)
523 + self.vision_model.load_weights(weights_group["vision_model"])
524
525 # load mlp projector
528 - mlp_weights = filter_weights(mlp_weights, "mlp1")
526 mlp_params_dict = dict(self.mlp1.named_parameters())
530 - for name, loaded_weight in mlp_weights:
527 + for name, loaded_weight in weights_group["mlp1"]:
528 param = mlp_params_dict[name]
529 weight_loader = getattr(param, "weight_loader",
530 default_weight_loader)
531 weight_loader(param, loaded_weight)
532
533 # load llm backbone
537 - llm_weights = filter_weights(llm_weights, "language_model")
538 - self.language_model.load_weights(llm_weights)
534 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/llava.py
+6 -10
@@ -1,4 +1,3 @@
1 -import itertools
1 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 TypedDict, Union)
3
@@ -26,8 +25,8 @@ from .interfaces import SupportsMultiModal
25 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
26 dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
27 input_processor_for_siglip)
29 -from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
30 - merge_multimodal_embeddings)
28 +from .utils import (flatten_bn, group_weights_with_prefix,
29 + init_vllm_registered_model, merge_multimodal_embeddings)
30
31
32 class LlavaImagePixelInputs(TypedDict):
@@ -393,21 +392,18 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
392
393 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
394 # prepare weight iterators for components
396 - vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
395 + weights_group = group_weights_with_prefix(weights)
396
397 # load vision encoder
399 - vit_weights = filter_weights(vit_weights, "vision_tower")
400 - self.vision_tower.load_weights(vit_weights)
398 + self.vision_tower.load_weights(weights_group["vision_tower"])
399
400 # load mlp projector
403 - mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
401 mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
405 - for name, loaded_weight in mlp_weights:
402 + for name, loaded_weight in weights_group["multi_modal_projector"]:
403 param = mlp_params_dict[name]
404 weight_loader = getattr(param, "weight_loader",
405 default_weight_loader)
406 weight_loader(param, loaded_weight)
407
408 # load llm backbone
412 - llm_weights = filter_weights(llm_weights, "language_model")
413 - self.language_model.load_weights(llm_weights)
409 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/llava_next.py
+7 -13
@@ -1,4 +1,3 @@
1 -import itertools
1 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 TypedDict, Union)
3
@@ -30,8 +29,8 @@ from .llava import LlavaMultiModalProjector
29 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
31 get_siglip_patch_grid_length, input_processor_for_siglip)
33 -from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
34 - merge_multimodal_embeddings)
32 +from .utils import (flatten_bn, group_weights_with_prefix,
33 + init_vllm_registered_model, merge_multimodal_embeddings)
34
35 logger = init_logger(__name__)
36
@@ -637,25 +636,21 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
636
637 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
638 # prepare weight iterators for components
640 - vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
641 - weights, 4)
639 + weights_group = group_weights_with_prefix(weights)
640
641 # load vision encoder
644 - vit_weights = filter_weights(vit_weights, "vision_tower")
645 - self.vision_tower.load_weights(vit_weights)
642 + self.vision_tower.load_weights(weights_group["vision_tower"])
643
644 # load mlp projector
648 - mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
645 mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
650 - for name, loaded_weight in mlp_weights:
646 + for name, loaded_weight in weights_group["multi_modal_projector"]:
647 param = mlp_params_dict[name]
648 weight_loader = getattr(param, "weight_loader",
649 default_weight_loader)
650 weight_loader(param, loaded_weight)
651
652 # load newline
657 - newline_weights = filter_weights(newline_weights, "image_newline")
658 - for name, loaded_weight in newline_weights:
653 + for name, loaded_weight in weights_group["image_newline"]:
654 assert name == ""
655 param = self.image_newline
656 weight_loader = getattr(param, "weight_loader",
@@ -663,5 +658,4 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
658 weight_loader(param, loaded_weight)
659
660 # load llm backbone
666 - llm_weights = filter_weights(llm_weights, "language_model")
667 - self.language_model.load_weights(llm_weights)
661 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/llava_next_video.py
+6 -11
@@ -1,4 +1,3 @@
1 -import itertools
1 import math
2 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 TypedDict, Union)
@@ -30,7 +29,7 @@ from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
29 from .interfaces import SupportsMultiModal
30 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 dummy_seq_data_for_siglip)
33 -from .utils import (filter_weights, init_vllm_registered_model,
32 +from .utils import (group_weights_with_prefix, init_vllm_registered_model,
33 merge_multimodal_embeddings)
34
35 logger = init_logger(__name__)
@@ -449,23 +448,19 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal):
448 return self.language_model.sample(logits, sampling_metadata)
449
450 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
452 - # prepare weight iterators
453 - vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
454 - weights, 4)
451 + # prepare weight iterators for components
452 + weights_group = group_weights_with_prefix(weights)
453
454 # load vision encoder
457 - vit_weights = filter_weights(vit_weights, "vision_tower")
458 - self.vision_tower.load_weights(vit_weights)
455 + self.vision_tower.load_weights(weights_group["vision_tower"])
456
457 # load mlp projector
461 - mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
458 mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
463 - for name, loaded_weight in mlp_weights:
459 + for name, loaded_weight in weights_group["multi_modal_projector"]:
460 param = mlp_params_dict[name]
461 weight_loader = getattr(param, "weight_loader",
462 default_weight_loader)
463 weight_loader(param, loaded_weight)
464
465 # load llm backbone
470 - llm_weights = filter_weights(llm_weights, "language_model")
471 - self.language_model.load_weights(llm_weights)
466 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/paligemma.py
+5 -9
@@ -1,4 +1,3 @@
1 -import itertools
1 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 TypedDict, Union)
3
@@ -23,7 +22,7 @@ from vllm.sequence import IntermediateTensors
22 from .interfaces import SupportsMultiModal
23 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
26 -from .utils import filter_weights, merge_multimodal_embeddings
25 +from .utils import group_weights_with_prefix, merge_multimodal_embeddings
26
27 logger = init_logger(__name__)
28
@@ -286,21 +285,18 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
285
286 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
287 # prepare weight iterators for components
289 - vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
288 + weights_group = group_weights_with_prefix(weights)
289
290 # load vision tower
292 - vit_weights = filter_weights(vit_weights, "vision_tower")
293 - self.vision_tower.load_weights(vit_weights)
291 + self.vision_tower.load_weights(weights_group["vision_tower"])
292
293 # load mlp projector
296 - mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
294 mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
298 - for name, loaded_weight in mlp_weights:
295 + for name, loaded_weight in weights_group["multi_modal_projector"]:
296 param = mlp_params_dict[name]
297 weight_loader = getattr(param, "weight_loader",
298 default_weight_loader)
299 weight_loader(param, loaded_weight)
300
301 # load llm backbone
305 - llm_weights = filter_weights(llm_weights, "language_model")
306 - self.language_model.load_weights(llm_weights)
302 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/ultravox.py
+5 -7
@@ -1,7 +1,6 @@
1 # Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
2 """PyTorch Ultravox model."""
3
4 -import itertools
4 import math
5 from array import array
6 from functools import lru_cache
@@ -29,7 +28,8 @@ from vllm.model_executor.layers.quantization.base_config import (
28 from vllm.model_executor.layers.sampler import SamplerOutput
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.models.interfaces import SupportsMultiModal
32 -from vllm.model_executor.models.utils import (filter_weights, flatten_bn,
31 +from vllm.model_executor.models.utils import (flatten_bn,
32 + group_weights_with_prefix,
33 init_vllm_registered_model,
34 merge_multimodal_embeddings)
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -467,11 +467,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
467
468 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
469 # prepare weight iterators for components
470 - projector_weights, llm_weights = itertools.tee(weights, 2)
470 + weights_group = group_weights_with_prefix(weights)
471
472 # load projector weights
473 - projector_weights = filter_weights(projector_weights,
474 - "multi_modal_projector")
473 + projector_weights = weights_group["multi_modal_projector"]
474 projector_params_dict = dict(
475 self.multi_modal_projector.named_parameters())
476 for name, loaded_weight in projector_weights:
@@ -481,5 +480,4 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
480 weight_loader(param, loaded_weight)
481
482 # load llm backbone
484 - llm_weights = filter_weights(llm_weights, "language_model")
485 - self.language_model.load_weights(llm_weights)
483 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/utils.py
+35 -1
@@ -1,3 +1,5 @@
1 +import itertools
2 +from collections import UserDict
3 from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
4 Union, overload)
5
@@ -16,7 +18,23 @@ from vllm.sequence import IntermediateTensors
18 from vllm.utils import is_pin_memory_available
19
20
19 -def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]], prefix: str):
21 +class WeightsGroup(UserDict):
22 + """
23 + Wraps grouped weights dictionary for a more informative error message
24 + when attempting to access a weight component that does not exist.
25 + """
26 +
27 + def __getitem__(self, key: str) -> int:
28 + try:
29 + return super().__getitem__(key)
30 + except KeyError as exc:
31 + msg = (f"There is no weights named with the prefix: {key}. "
32 + f"Available prefix: {set(self.keys())}")
33 + raise KeyError(msg) from exc
34 +
35 +
36 +def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]],
37 + prefix: str) -> Iterable[Tuple[str, torch.Tensor]]:
38 """
39 Helper function to load weights for inner vLLM models.
40
@@ -30,6 +48,22 @@ def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]], prefix: str):
48 yield name, loaded_weight
49
50
51 +def group_weights_with_prefix(
52 + weights: Iterable[Tuple[str, torch.Tensor]]
53 +) -> Dict[str, Iterable[Tuple[str, torch.Tensor]]]:
54 + """
55 + Helper function to group weights with prefix
56 + """
57 + init_weights, repeated_weights = itertools.tee(weights, 2)
58 + weights_prefix = {name.split(".")[0] for name, _ in init_weights}
59 + repeated_weights = itertools.tee(repeated_weights, len(weights_prefix))
60 +
61 + return WeightsGroup({
62 + prefix: filter_weights(component, prefix)
63 + for component, prefix in zip(repeated_weights, weights_prefix)
64 + })
65 +
66 +
67 def init_vllm_registered_model(
68 hf_config: PretrainedConfig,
69 cache_config: Optional[CacheConfig],