A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Refactor composite weight loading logic (#8656)
Isotr0py committed
Sep 22, 2024 at 12:33 UTC
13d88d4137f97b8cf3c79f39d7df5e4c8348603a
7 files changed
+70
-61
vllm/model_executor/models/internvl.py
+6
-10
@@ -4,7 +4,6 @@
4
# Copyright (c) 2023 OpenGVLab
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
-import itertools
7
import re
8
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9
TypedDict, Union)
@@ -33,8 +32,8 @@ from vllm.utils import is_list_of
32
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
33
get_clip_num_patches)
34
from .interfaces import SupportsMultiModal
36
-from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
37
- merge_multimodal_embeddings)
35
+from .utils import (flatten_bn, group_weights_with_prefix,
36
+ init_vllm_registered_model, merge_multimodal_embeddings)
37
38
IMG_START = '<img>'
39
IMG_END = '</img>'
@@ -518,21 +517,18 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
517
518
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
519
# prepare weight iterators for components
521
- vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
520
+ weights_group = group_weights_with_prefix(weights)
521
522
# load vision encoder
524
- vit_weights = filter_weights(vit_weights, "vision_model")
525
- self.vision_model.load_weights(vit_weights)
523
+ self.vision_model.load_weights(weights_group["vision_model"])
524
525
# load mlp projector
528
- mlp_weights = filter_weights(mlp_weights, "mlp1")
526
mlp_params_dict = dict(self.mlp1.named_parameters())
530
- for name, loaded_weight in mlp_weights:
527
+ for name, loaded_weight in weights_group["mlp1"]:
528
param = mlp_params_dict[name]
529
weight_loader = getattr(param, "weight_loader",
530
default_weight_loader)
531
weight_loader(param, loaded_weight)
532
533
# load llm backbone
537
- llm_weights = filter_weights(llm_weights, "language_model")
538
- self.language_model.load_weights(llm_weights)
534
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/llava.py
+6
-10
@@ -1,4 +1,3 @@
1
-import itertools
1
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
TypedDict, Union)
3
@@ -26,8 +25,8 @@ from .interfaces import SupportsMultiModal
25
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
26
dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
27
input_processor_for_siglip)
29
-from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
30
- merge_multimodal_embeddings)
28
+from .utils import (flatten_bn, group_weights_with_prefix,
29
+ init_vllm_registered_model, merge_multimodal_embeddings)
30
31
32
class LlavaImagePixelInputs(TypedDict):
@@ -393,21 +392,18 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
392
393
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
394
# prepare weight iterators for components
396
- vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
395
+ weights_group = group_weights_with_prefix(weights)
396
397
# load vision encoder
399
- vit_weights = filter_weights(vit_weights, "vision_tower")
400
- self.vision_tower.load_weights(vit_weights)
398
+ self.vision_tower.load_weights(weights_group["vision_tower"])
399
400
# load mlp projector
403
- mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
401
mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
405
- for name, loaded_weight in mlp_weights:
402
+ for name, loaded_weight in weights_group["multi_modal_projector"]:
403
param = mlp_params_dict[name]
404
weight_loader = getattr(param, "weight_loader",
405
default_weight_loader)
406
weight_loader(param, loaded_weight)
407
408
# load llm backbone
412
- llm_weights = filter_weights(llm_weights, "language_model")
413
- self.language_model.load_weights(llm_weights)
409
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/llava_next.py
+7
-13
@@ -1,4 +1,3 @@
1
-import itertools
1
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
TypedDict, Union)
3
@@ -30,8 +29,8 @@ from .llava import LlavaMultiModalProjector
29
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
31
get_siglip_patch_grid_length, input_processor_for_siglip)
33
-from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
34
- merge_multimodal_embeddings)
32
+from .utils import (flatten_bn, group_weights_with_prefix,
33
+ init_vllm_registered_model, merge_multimodal_embeddings)
34
35
logger = init_logger(__name__)
36
@@ -637,25 +636,21 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
636
637
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
638
# prepare weight iterators for components
640
- vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
641
- weights, 4)
639
+ weights_group = group_weights_with_prefix(weights)
640
641
# load vision encoder
644
- vit_weights = filter_weights(vit_weights, "vision_tower")
645
- self.vision_tower.load_weights(vit_weights)
642
+ self.vision_tower.load_weights(weights_group["vision_tower"])
643
644
# load mlp projector
648
- mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
645
mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
650
- for name, loaded_weight in mlp_weights:
646
+ for name, loaded_weight in weights_group["multi_modal_projector"]:
647
param = mlp_params_dict[name]
648
weight_loader = getattr(param, "weight_loader",
649
default_weight_loader)
650
weight_loader(param, loaded_weight)
651
652
# load newline
657
- newline_weights = filter_weights(newline_weights, "image_newline")
658
- for name, loaded_weight in newline_weights:
653
+ for name, loaded_weight in weights_group["image_newline"]:
654
assert name == ""
655
param = self.image_newline
656
weight_loader = getattr(param, "weight_loader",
@@ -663,5 +658,4 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
658
weight_loader(param, loaded_weight)
659
660
# load llm backbone
666
- llm_weights = filter_weights(llm_weights, "language_model")
667
- self.language_model.load_weights(llm_weights)
661
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/llava_next_video.py
+6
-11
@@ -1,4 +1,3 @@
1
-import itertools
1
import math
2
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
TypedDict, Union)
@@ -30,7 +29,7 @@ from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
29
from .interfaces import SupportsMultiModal
30
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
dummy_seq_data_for_siglip)
33
-from .utils import (filter_weights, init_vllm_registered_model,
32
+from .utils import (group_weights_with_prefix, init_vllm_registered_model,
33
merge_multimodal_embeddings)
34
35
logger = init_logger(__name__)
@@ -449,23 +448,19 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal):
448
return self.language_model.sample(logits, sampling_metadata)
449
450
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
452
- # prepare weight iterators
453
- vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
454
- weights, 4)
451
+ # prepare weight iterators for components
452
+ weights_group = group_weights_with_prefix(weights)
453
454
# load vision encoder
457
- vit_weights = filter_weights(vit_weights, "vision_tower")
458
- self.vision_tower.load_weights(vit_weights)
455
+ self.vision_tower.load_weights(weights_group["vision_tower"])
456
457
# load mlp projector
461
- mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
458
mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
463
- for name, loaded_weight in mlp_weights:
459
+ for name, loaded_weight in weights_group["multi_modal_projector"]:
460
param = mlp_params_dict[name]
461
weight_loader = getattr(param, "weight_loader",
462
default_weight_loader)
463
weight_loader(param, loaded_weight)
464
465
# load llm backbone
470
- llm_weights = filter_weights(llm_weights, "language_model")
471
- self.language_model.load_weights(llm_weights)
466
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/paligemma.py
+5
-9
@@ -1,4 +1,3 @@
1
-import itertools
1
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
TypedDict, Union)
3
@@ -23,7 +22,7 @@ from vllm.sequence import IntermediateTensors
22
from .interfaces import SupportsMultiModal
23
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24
dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
26
-from .utils import filter_weights, merge_multimodal_embeddings
25
+from .utils import group_weights_with_prefix, merge_multimodal_embeddings
26
27
logger = init_logger(__name__)
28
@@ -286,21 +285,18 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
285
286
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
287
# prepare weight iterators for components
289
- vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
288
+ weights_group = group_weights_with_prefix(weights)
289
290
# load vision tower
292
- vit_weights = filter_weights(vit_weights, "vision_tower")
293
- self.vision_tower.load_weights(vit_weights)
291
+ self.vision_tower.load_weights(weights_group["vision_tower"])
292
293
# load mlp projector
296
- mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
294
mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
298
- for name, loaded_weight in mlp_weights:
295
+ for name, loaded_weight in weights_group["multi_modal_projector"]:
296
param = mlp_params_dict[name]
297
weight_loader = getattr(param, "weight_loader",
298
default_weight_loader)
299
weight_loader(param, loaded_weight)
300
301
# load llm backbone
305
- llm_weights = filter_weights(llm_weights, "language_model")
306
- self.language_model.load_weights(llm_weights)
302
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/ultravox.py
+5
-7
@@ -1,7 +1,6 @@
1
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
2
"""PyTorch Ultravox model."""
3
4
-import itertools
4
import math
5
from array import array
6
from functools import lru_cache
@@ -29,7 +28,8 @@ from vllm.model_executor.layers.quantization.base_config import (
28
from vllm.model_executor.layers.sampler import SamplerOutput
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.models.interfaces import SupportsMultiModal
32
-from vllm.model_executor.models.utils import (filter_weights, flatten_bn,
31
+from vllm.model_executor.models.utils import (flatten_bn,
32
+ group_weights_with_prefix,
33
init_vllm_registered_model,
34
merge_multimodal_embeddings)
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -467,11 +467,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
467
468
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
469
# prepare weight iterators for components
470
- projector_weights, llm_weights = itertools.tee(weights, 2)
470
+ weights_group = group_weights_with_prefix(weights)
471
472
# load projector weights
473
- projector_weights = filter_weights(projector_weights,
474
- "multi_modal_projector")
473
+ projector_weights = weights_group["multi_modal_projector"]
474
projector_params_dict = dict(
475
self.multi_modal_projector.named_parameters())
476
for name, loaded_weight in projector_weights:
@@ -481,5 +480,4 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
480
weight_loader(param, loaded_weight)
481
482
# load llm backbone
484
- llm_weights = filter_weights(llm_weights, "language_model")
485
- self.language_model.load_weights(llm_weights)
483
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/utils.py
+35
-1
@@ -1,3 +1,5 @@
1
+import itertools
2
+from collections import UserDict
3
from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
4
Union, overload)
5
@@ -16,7 +18,23 @@ from vllm.sequence import IntermediateTensors
18
from vllm.utils import is_pin_memory_available
19
20
19
-def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]], prefix: str):
21
+class WeightsGroup(UserDict):
22
+ """
23
+ Wraps grouped weights dictionary for a more informative error message
24
+ when attempting to access a weight component that does not exist.
25
+ """
26
+
27
+ def __getitem__(self, key: str) -> int:
28
+ try:
29
+ return super().__getitem__(key)
30
+ except KeyError as exc:
31
+ msg = (f"There is no weights named with the prefix: {key}. "
32
+ f"Available prefix: {set(self.keys())}")
33
+ raise KeyError(msg) from exc
34
+
35
+
36
+def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]],
37
+ prefix: str) -> Iterable[Tuple[str, torch.Tensor]]:
38
"""
39
Helper function to load weights for inner vLLM models.
40
@@ -30,6 +48,22 @@ def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]], prefix: str):
48
yield name, loaded_weight
49
50
51
+def group_weights_with_prefix(
52
+ weights: Iterable[Tuple[str, torch.Tensor]]
53
+) -> Dict[str, Iterable[Tuple[str, torch.Tensor]]]:
54
+ """
55
+ Helper function to group weights with prefix
56
+ """
57
+ init_weights, repeated_weights = itertools.tee(weights, 2)
58
+ weights_prefix = {name.split(".")[0] for name, _ in init_weights}
59
+ repeated_weights = itertools.tee(repeated_weights, len(weights_prefix))
60
+
61
+ return WeightsGroup({
62
+ prefix: filter_weights(component, prefix)
63
+ for component, prefix in zip(repeated_weights, weights_prefix)
64
+ })
65
+
66
+
67
def init_vllm_registered_model(
68
hf_config: PretrainedConfig,
69
cache_config: Optional[CacheConfig],