A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Misc] Add uninitialized params tracking for `AutoWeightsLoader` (#10327)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 18, 2024 at 09:07 UTC
c4e464333eac5a46e1cc2701e095a44057c82927
74 files changed
+454
-185
vllm/model_executor/model_loader/loader.py
+11
-1
@@ -334,7 +334,17 @@ class DefaultModelLoader(BaseModelLoader):
334
with target_device:
335
model = _initialize_model(vllm_config=vllm_config)
336
337
- model.load_weights(self._get_all_weights(model_config, model))
337
+ weights_to_load = {name for name, _ in model.named_parameters()}
338
+ loaded_weights = model.load_weights(
339
+ self._get_all_weights(model_config, model))
340
+ # We only enable strict check for non-quantiized models
341
+ # that have loaded weights tracking currently.
342
+ if model_config.quantization is None and loaded_weights is not None:
343
+ weights_not_loaded = weights_to_load - loaded_weights
344
+ if weights_not_loaded:
345
+ raise ValueError(
346
+ "Following weights were not initialized from "
347
+ f"checkpoint: {weights_not_loaded}")
348
349
for _, module in model.named_modules():
350
quant_method = getattr(module, "quant_method", None)
vllm/model_executor/models/arctic.py
+6
-2
@@ -1,5 +1,5 @@
1
"""Inference-only Snowflake Arctic model."""
2
-from typing import Iterable, List, Optional, Tuple, Union
2
+from typing import Iterable, List, Optional, Set, Tuple, Union
3
4
import torch
5
from torch import nn
@@ -480,7 +480,8 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
480
next_tokens = self.sampler(logits, sampling_metadata)
481
return next_tokens
482
483
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
483
+ def load_weights(self, weights: Iterable[Tuple[str,
484
+ torch.Tensor]]) -> Set[str]:
485
stacked_params_mapping = [
486
# (param_name, shard_name, shard_id)
487
("qkv_proj", "q_proj", "q"),
@@ -518,6 +519,7 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
519
("ws", f"experts.{expert_id}.w3.weight", expert_id))
520
521
params_dict = dict(self.named_parameters())
522
+ loaded_params: Set[str] = set()
523
524
logger.info(
525
"It will take ~10 minutes loading from the 16-bit weights. "
@@ -573,3 +575,5 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
575
weight_loader = getattr(param, "weight_loader",
576
default_weight_loader)
577
weight_loader(param, loaded_weight)
578
+ loaded_params.add(name)
579
+ return loaded_params
vllm/model_executor/models/baichuan.py
+6
-2
@@ -18,7 +18,7 @@
18
# limitations under the License.
19
"""Inference-only BaiChuan model compatible with HuggingFace weights."""
20
import math
21
-from typing import Iterable, List, Optional, Tuple, Union
21
+from typing import Iterable, List, Optional, Set, Tuple, Union
22
23
import torch
24
from torch import nn
@@ -404,13 +404,15 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
404
next_tokens = self.sampler(logits, sampling_metadata)
405
return next_tokens
406
407
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
407
+ def load_weights(self, weights: Iterable[Tuple[str,
408
+ torch.Tensor]]) -> Set[str]:
409
stacked_params_mapping = [
410
# (param_name, shard_name, shard_id)
411
("gate_up_proj", "gate_proj", 0),
412
("gate_up_proj", "up_proj", 1),
413
]
414
params_dict = dict(self.named_parameters())
415
+ loaded_params: Set[str] = set()
416
for name, loaded_weight in weights:
417
if "rotary_emb.inv_freq" in name:
418
continue
@@ -449,6 +451,8 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
451
weight_loader = getattr(param, "weight_loader",
452
default_weight_loader)
453
weight_loader(param, loaded_weight)
454
+ loaded_params.add(name)
455
+ return loaded_params
456
457
458
class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
vllm/model_executor/models/bert.py
+6
-2
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Optional, Tuple
1
+from typing import Iterable, List, Optional, Set, Tuple
2
3
import torch
4
from torch import nn
@@ -337,7 +337,8 @@ class BertModel(nn.Module):
337
338
return self.encoder(hidden_states, kv_caches, attn_metadata)
339
340
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
340
+ def load_weights(self, weights: Iterable[Tuple[str,
341
+ torch.Tensor]]) -> Set[str]:
342
stacked_params_mapping = [
343
# (param_name, shard_name, shard_id)
344
("qkv_proj", "query", "q"),
@@ -346,6 +347,7 @@ class BertModel(nn.Module):
347
]
348
349
params_dict = dict(self.named_parameters())
350
+ loaded_params: Set[str] = set()
351
for name, loaded_weight in weights:
352
if "pooler" in name:
353
continue
@@ -368,6 +370,8 @@ class BertModel(nn.Module):
370
weight_loader = getattr(param, "weight_loader",
371
default_weight_loader)
372
weight_loader(param, loaded_weight)
373
+ loaded_params.add(name)
374
+ return loaded_params
375
376
377
class BertEmbeddingModel(nn.Module):
vllm/model_executor/models/blip.py
+8
-4
@@ -1,6 +1,6 @@
1
"""Minimal implementation of BlipVisionModel intended to be only used
2
within a vision language model."""
3
-from typing import Iterable, Optional, Tuple, Union
3
+from typing import Iterable, Optional, Set, Tuple, Union
4
5
import torch
6
import torch.nn as nn
@@ -415,7 +415,8 @@ class BlipVisionModel(nn.Module):
415
416
return self.post_layernorm(hidden_states)
417
418
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
418
+ def load_weights(self, weights: Iterable[Tuple[str,
419
+ torch.Tensor]]) -> Set[str]:
420
stacked_params_mapping = [
421
# (param_name, shard_name, shard_id)
422
("qkv_proj", "q_proj", "q"),
@@ -423,6 +424,7 @@ class BlipVisionModel(nn.Module):
424
("qkv_proj", "v_proj", "v"),
425
] if self.shard_weight else []
426
params_dict = dict(self.named_parameters())
427
+ loaded_params: Set[str] = set()
428
layer_count = len(self.encoder.layers)
429
430
for name, loaded_weight in weights:
@@ -440,8 +442,8 @@ class BlipVisionModel(nn.Module):
442
for (param_name, weight_name, shard_id) in stacked_params_mapping:
443
if weight_name not in name:
444
continue
443
-
444
- param = params_dict[name.replace(weight_name, param_name)]
445
+ name = name.replace(weight_name, param_name)
446
+ param = params_dict[name]
447
weight_loader = param.weight_loader
448
weight_loader(param, loaded_weight, shard_id)
449
break
@@ -450,3 +452,5 @@ class BlipVisionModel(nn.Module):
452
weight_loader = getattr(param, "weight_loader",
453
default_weight_loader)
454
weight_loader(param, loaded_weight)
455
+ loaded_params.add(name)
456
+ return loaded_params
vllm/model_executor/models/blip2.py
+4
-3
@@ -1,5 +1,5 @@
1
from functools import cached_property
2
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3
TypedDict, Union)
4
5
import torch
@@ -692,6 +692,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
692
) -> Optional[SamplerOutput]:
693
return self.language_model.sample(logits, sampling_metadata)
694
695
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
695
+ def load_weights(self, weights: Iterable[Tuple[str,
696
+ torch.Tensor]]) -> Set[str]:
697
loader = AutoWeightsLoader(self)
697
- loader.load_weights(weights)
698
+ return loader.load_weights(weights)
vllm/model_executor/models/bloom.py
+6
-2
@@ -16,7 +16,7 @@
16
# limitations under the License.
17
"""Inference-only BLOOM model compatible with HuggingFace weights."""
18
import math
19
-from typing import Iterable, List, Optional, Tuple, Union
19
+from typing import Iterable, List, Optional, Set, Tuple, Union
20
21
import torch
22
from torch import nn
@@ -341,8 +341,10 @@ class BloomForCausalLM(nn.Module, SupportsPP):
341
next_tokens = self.sampler(logits, sampling_metadata)
342
return next_tokens
343
344
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
344
+ def load_weights(self, weights: Iterable[Tuple[str,
345
+ torch.Tensor]]) -> Set[str]:
346
params_dict = dict(self.named_parameters(remove_duplicate=False))
347
+ loaded_params: Set[str] = set()
348
for name, loaded_weight in weights:
349
if name == "lm_head.weight":
350
continue
@@ -371,3 +373,5 @@ class BloomForCausalLM(nn.Module, SupportsPP):
373
weight_loader = getattr(param, "weight_loader",
374
default_weight_loader)
375
weight_loader(param, loaded_weight)
376
+ loaded_params.add(name)
377
+ return loaded_params
vllm/model_executor/models/chameleon.py
+6
-2
@@ -1,5 +1,5 @@
1
from functools import cached_property
2
-from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
2
+from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
3
Tuple, TypedDict, Union)
4
5
import torch
@@ -1034,7 +1034,8 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1034
next_tokens = self.sampler(logits, sampling_metadata)
1035
return next_tokens
1036
1037
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1037
+ def load_weights(self, weights: Iterable[Tuple[str,
1038
+ torch.Tensor]]) -> Set[str]:
1039
stacked_params_mapping = [
1040
# (param_name, shard_name, shard_id)
1041
(".qkv_proj", ".q_proj", "q"),
@@ -1044,6 +1045,7 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1045
(".gate_up_proj", ".up_proj", 1),
1046
]
1047
params_dict = dict(self.named_parameters())
1048
+ loaded_params: Set[str] = set()
1049
for name, loaded_weight in weights:
1050
if "rotary_emb.inv_freq" in name:
1051
continue
@@ -1111,3 +1113,5 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1113
weight_loader = getattr(param, "weight_loader",
1114
default_weight_loader)
1115
weight_loader(param, loaded_weight)
1116
+ loaded_params.add(name)
1117
+ return loaded_params
vllm/model_executor/models/chatglm.py
+8
-2
@@ -3,7 +3,8 @@
3
"""Inference-only ChatGLM model compatible with THUDM weights."""
4
from argparse import Namespace
5
from array import array
6
-from typing import Dict, Iterable, List, Mapping, Optional, Tuple, TypedDict
6
+from typing import (Dict, Iterable, List, Mapping, Optional, Set, Tuple,
7
+ TypedDict)
8
9
import torch
10
from PIL import Image
@@ -645,7 +646,8 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
646
next_tokens = self.sampler(logits, sampling_metadata)
647
return next_tokens
648
648
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
649
+ def load_weights(self, weights: Iterable[Tuple[str,
650
+ torch.Tensor]]) -> Set[str]:
651
# Merge two ColumnParallelLinear into one MergedColumnParallelLinear
652
merged_weights_dict: Dict[str, Dict[str, Optional[torch.Tensor]]] = {
653
"transformer.vision.linear_proj.merged_proj.weight": {
@@ -655,6 +657,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
657
}
658
659
params_dict = dict(self.named_parameters(remove_duplicate=False))
660
+ loaded_params: Set[str] = set()
661
for name, loaded_weight in weights:
662
is_weight_to_be_merge = False
663
for _, merged_weight_dict in merged_weights_dict.items():
@@ -677,6 +680,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
680
weight_loader = getattr(param, "weight_loader",
681
default_weight_loader)
682
weight_loader(param, loaded_weight)
683
+ loaded_params.add(name)
684
685
for combined_name, merged_weight_dict in merged_weights_dict.items():
686
if combined_name in params_dict:
@@ -686,3 +690,5 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
690
weight_loader = getattr(param, "weight_loader",
691
default_weight_loader)
692
weight_loader(param, combined_weight)
693
+ loaded_params.add(combined_name)
694
+ return loaded_params
vllm/model_executor/models/clip.py
+8
-3
@@ -1,6 +1,6 @@
1
"""Minimal implementation of CLIPVisionModel intended to be only used
2
within a vision language model."""
3
-from typing import Iterable, List, Optional, Tuple, Union
3
+from typing import Iterable, List, Optional, Set, Tuple, Union
4
5
import numpy as np
6
import torch
@@ -483,7 +483,8 @@ class CLIPVisionModel(nn.Module):
483
484
# (TODO) Add prefix argument for filtering out weights to be loaded
485
# ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
486
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
486
+ def load_weights(self, weights: Iterable[Tuple[str,
487
+ torch.Tensor]]) -> Set[str]:
488
stacked_params_mapping = [
489
# (param_name, shard_name, shard_id)
490
("qkv_proj", "q_proj", "q"),
@@ -491,6 +492,7 @@ class CLIPVisionModel(nn.Module):
492
("qkv_proj", "v_proj", "v"),
493
] if self.shard_weight else []
494
params_dict = dict(self.named_parameters())
495
+ loaded_params: Set[str] = set()
496
layer_count = len(self.vision_model.encoder.layers)
497
498
for name, loaded_weight in weights:
@@ -508,8 +510,9 @@ class CLIPVisionModel(nn.Module):
510
for (param_name, weight_name, shard_id) in stacked_params_mapping:
511
if weight_name not in name:
512
continue
513
+ name = name.replace(weight_name, param_name)
514
512
- param = params_dict[name.replace(weight_name, param_name)]
515
+ param = params_dict[name]
516
weight_loader = param.weight_loader
517
weight_loader(param, loaded_weight, shard_id)
518
break
@@ -518,3 +521,5 @@ class CLIPVisionModel(nn.Module):
521
weight_loader = getattr(param, "weight_loader",
522
default_weight_loader)
523
weight_loader(param, loaded_weight)
524
+ loaded_params.add(name)
525
+ return loaded_params
vllm/model_executor/models/commandr.py
+3
-1
@@ -402,7 +402,8 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
402
next_tokens = self.sampler(logits, sampling_metadata)
403
return next_tokens
404
405
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
405
+ def load_weights(self, weights: Iterable[Tuple[str,
406
+ torch.Tensor]]) -> Set[str]:
407
stacked_params_mapping = [
408
# (param_name, shard_name, shard_id)
409
("qkv_proj", "q_proj", "q"),
@@ -447,3 +448,4 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
448
default_weight_loader)
449
weight_loader(param, loaded_weight)
450
loaded_params.add(name)
451
+ return loaded_params
vllm/model_executor/models/dbrx.py
+6
-2
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Optional, Tuple, Union
1
+from typing import Iterable, List, Optional, Set, Tuple, Union
2
3
import torch
4
import torch.nn as nn
@@ -417,13 +417,15 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
417
next_tokens = self.sampler(logits, sampling_metadata)
418
return next_tokens
419
420
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
420
+ def load_weights(self, weights: Iterable[Tuple[str,
421
+ torch.Tensor]]) -> Set[str]:
422
423
expert_params_mapping = [(
424
"w13_weight" if weight_name in ["w1", "v1"] else "w2_weight",
425
f"mlp.{weight_name}",
426
) for weight_name in ["w1", "v1", "w2"]]
427
params_dict = dict(self.named_parameters(remove_duplicate=False))
428
+ loaded_params: Set[str] = set()
429
for name, loaded_weight in weights:
430
for param_name, weight_name in expert_params_mapping:
431
if weight_name not in name:
@@ -447,3 +449,5 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
449
weight_loader = getattr(param, "weight_loader",
450
default_weight_loader)
451
weight_loader(param, loaded_weight)
452
+ loaded_params.add(name)
453
+ return loaded_params
vllm/model_executor/models/decilm.py
+6
-2
@@ -22,7 +22,7 @@
22
# limitations under the License.
23
"""Inference-only DeciLM model compatible with HuggingFace weights."""
24
25
-from typing import Iterable, Tuple
25
+from typing import Iterable, Set, Tuple
26
27
import torch
28
@@ -57,7 +57,8 @@ class DeciLMForCausalLM(LlamaForCausalLM):
57
delattr(config, "num_key_value_heads_per_layer")
58
super().__init__(vllm_config=vllm_config)
59
60
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
60
+ def load_weights(self, weights: Iterable[Tuple[str,
61
+ torch.Tensor]]) -> Set[str]:
62
stacked_params_mapping = [
63
# (param_name, shard_name, shard_id)
64
("qkv_proj", "q_proj", "q"),
@@ -67,6 +68,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
68
("gate_up_proj", "up_proj", 1),
69
]
70
params_dict = dict(self.named_parameters())
71
+ loaded_params: Set[str] = set()
72
for name, loaded_weight in weights:
73
if "rotary_emb.inv_freq" in name:
74
continue
@@ -97,6 +99,8 @@ class DeciLMForCausalLM(LlamaForCausalLM):
99
weight_loader = getattr(param, "weight_loader",
100
default_weight_loader)
101
weight_loader(param, loaded_weight)
102
+ loaded_params.add(name)
103
+ return loaded_params
104
105
def _degroup_weight(self, loaded_weight: torch.Tensor) -> torch.Tensor:
106
hidden_size = self.config.hidden_size
vllm/model_executor/models/deepseek.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only Deepseek model."""
23
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -442,7 +442,8 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
442
next_tokens = self.sampler(logits, sampling_metadata)
443
return next_tokens
444
445
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
445
+ def load_weights(self, weights: Iterable[Tuple[str,
446
+ torch.Tensor]]) -> Set[str]:
447
stacked_params_mapping = [
448
# (param_name, shard_name, shard_id)
449
("qkv_proj", "q_proj", "q"),
@@ -453,6 +454,7 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
454
]
455
456
params_dict = dict(self.named_parameters())
457
+ loaded_params: Set[str] = set()
458
for name, loaded_weight in weights:
459
if "rotary_emb.inv_freq" in name:
460
continue
@@ -487,3 +489,5 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
489
weight_loader = getattr(param, "weight_loader",
490
default_weight_loader)
491
weight_loader(param, loaded_weight)
492
+ loaded_params.add(name)
493
+ return loaded_params
vllm/model_executor/models/deepseek_v2.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only DeepseekV2 model."""
23
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -550,7 +550,8 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
550
device=device),
551
})
552
553
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
553
+ def load_weights(self, weights: Iterable[Tuple[str,
554
+ torch.Tensor]]) -> Set[str]:
555
stacked_params_mapping = [
556
# (param_name, shard_name, shard_id)
557
("gate_up_proj", "gate_proj", 0),
@@ -566,6 +567,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
567
num_experts=self.config.n_routed_experts)
568
569
params_dict = dict(self.named_parameters())
570
+ loaded_params: Set[str] = set()
571
for name, loaded_weight in weights:
572
if "rotary_emb.inv_freq" in name:
573
continue
@@ -623,3 +625,5 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
625
weight_loader = getattr(param, "weight_loader",
626
default_weight_loader)
627
weight_loader(param, loaded_weight)
628
+ loaded_params.add(name)
629
+ return loaded_params
vllm/model_executor/models/exaone.py
+7
-2
@@ -22,7 +22,7 @@
22
# limitations under the License.
23
"""Inference-only Exaone model compatible with HuggingFace weights."""
24
25
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
26
27
import torch
28
from torch import nn
@@ -513,7 +513,8 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
513
next_tokens = self.sampler(logits, sampling_metadata)
514
return next_tokens
515
516
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
516
+ def load_weights(self, weights: Iterable[Tuple[str,
517
+ torch.Tensor]]) -> Set[str]:
518
stacked_params_mapping = [
519
# (param_name, shard_name, shard_id)
520
(".qkv_proj", ".q_proj", "q"),
@@ -523,6 +524,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
524
(".gate_up_proj", ".c_fc_1", 1),
525
]
526
params_dict = dict(self.named_parameters())
527
+ loaded_params: Set[str] = set()
528
for name, loaded_weight in weights:
529
if "rotary_emb.inv_freq" in name:
530
continue
@@ -543,6 +545,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
545
default_weight_loader)
546
loaded_weight = loaded_weight[0]
547
weight_loader(param, loaded_weight)
548
+ loaded_params.add(scale_name)
549
continue
550
for param_name, weight_name, shard_id in stacked_params_mapping:
551
if weight_name not in name:
@@ -576,6 +579,8 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
579
weight_loader = getattr(param, "weight_loader",
580
default_weight_loader)
581
weight_loader(param, loaded_weight)
582
+ loaded_params.add(name)
583
+ return loaded_params
584
585
# If this function is called, it should always initialize KV cache scale
586
# factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/falcon.py
+6
-2
@@ -18,7 +18,7 @@
18
"""PyTorch Falcon model."""
19
20
import math
21
-from typing import Iterable, List, Optional, Tuple, Union
21
+from typing import Iterable, List, Optional, Set, Tuple, Union
22
23
import torch
24
from torch import nn
@@ -473,7 +473,8 @@ class FalconForCausalLM(nn.Module, SupportsPP):
473
next_tokens = self.sampler(logits, sampling_metadata)
474
return next_tokens
475
476
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
476
+ def load_weights(self, weights: Iterable[Tuple[str,
477
+ torch.Tensor]]) -> Set[str]:
478
total_num_heads = self.config.num_attention_heads
479
if self.config.new_decoder_architecture:
480
total_num_kv_heads = self.config.num_kv_heads
@@ -483,6 +484,7 @@ class FalconForCausalLM(nn.Module, SupportsPP):
484
total_num_kv_heads = total_num_heads
485
num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
486
params_dict = dict(self.named_parameters(remove_duplicate=False))
487
+ loaded_params: Set[str] = set()
488
for name, loaded_weight in weights:
489
if name == "lm_head.weight" and self.tie_word_embeddings:
490
# Falcon uses tied embeddings except Falcon-11b.
@@ -519,3 +521,5 @@ class FalconForCausalLM(nn.Module, SupportsPP):
521
weight_loader = getattr(param, "weight_loader",
522
default_weight_loader)
523
weight_loader(param, loaded_weight)
524
+ loaded_params.add(name)
525
+ return loaded_params
vllm/model_executor/models/florence2.py
+11
-6
@@ -1,5 +1,5 @@
1
import math
2
-from typing import Iterable, List, Optional, Tuple
2
+from typing import Iterable, List, Optional, Set, Tuple
3
4
import torch
5
import torch.nn as nn
@@ -156,7 +156,8 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
156
next_tokens = self.sampler(logits, sampling_metadata)
157
return next_tokens
158
159
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
159
+ def load_weights(self, weights: Iterable[Tuple[str,
160
+ torch.Tensor]]) -> Set[str]:
161
stacked_params_mapping = [
162
# (param_name, shard_name, shard_id)
163
("qkv_proj", "q_proj", "q"),
@@ -165,12 +166,13 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
166
]
167
168
params_dict = dict(self.named_parameters())
169
+ loaded_params: Set[str] = set()
170
for name, loaded_weight in weights:
171
for (param_name, weight_name, shard_id) in stacked_params_mapping:
172
if weight_name not in name:
173
continue
172
-
173
- param = params_dict[name.replace(weight_name, param_name)]
174
+ name = name.replace(weight_name, param_name)
175
+ param = params_dict[name]
176
weight_loader = param.weight_loader
177
weight_loader(param, loaded_weight, shard_id)
178
break
@@ -183,6 +185,8 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
185
weight_loader = getattr(param, "weight_loader",
186
default_weight_loader)
187
weight_loader(param, loaded_weight)
188
+ loaded_params.add(name)
189
+ return loaded_params
190
191
192
class Florence2ForConditionalGeneration(nn.Module):
@@ -248,10 +252,11 @@ class Florence2ForConditionalGeneration(nn.Module):
252
) -> SamplerOutput:
253
return self.language_model.sample(logits, sampling_metadata)
254
251
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
255
+ def load_weights(self, weights: Iterable[Tuple[str,
256
+ torch.Tensor]]) -> Set[str]:
257
skip_prefixes = [
258
'image_projection', "vision_tower", "image_proj_norm",
259
"image_pos_embed", "visual_temporal_embed"
260
]
261
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
257
- loader.load_weights(weights)
262
+ return loader.load_weights(weights)
vllm/model_executor/models/fuyu.py
+5
-3
@@ -16,7 +16,8 @@
16
""" PyTorch Fuyu model."""
17
import math
18
from array import array
19
-from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict
19
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
20
+ TypedDict)
21
22
import torch
23
import torch.nn as nn
@@ -354,6 +355,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
355
next_tokens = self.language_model.sampler(logits, sampling_metadata)
356
return next_tokens
357
357
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
358
+ def load_weights(self, weights: Iterable[Tuple[str,
359
+ torch.Tensor]]) -> Set[str]:
360
loader = AutoWeightsLoader(self)
359
- loader.load_weights(weights)
361
+ return loader.load_weights(weights)
vllm/model_executor/models/gemma.py
+3
-1
@@ -424,7 +424,8 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
424
next_tokens = self.sampler(logits, sampling_metadata)
425
return next_tokens
426
427
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
427
+ def load_weights(self, weights: Iterable[Tuple[str,
428
+ torch.Tensor]]) -> Set[str]:
429
stacked_params_mapping = [
430
# (param_name, shard_name, shard_id)
431
("qkv_proj", "q_proj", "q"),
@@ -469,3 +470,4 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
470
logger.warning(
471
"Some weights are not initialized from checkpoints: %s",
472
unloaded_params)
473
+ return loaded_params
vllm/model_executor/models/gemma2.py
+6
-3
@@ -312,7 +312,8 @@ class Gemma2Model(nn.Module):
312
hidden_states, _ = self.norm(hidden_states, residual)
313
return hidden_states
314
315
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
315
+ def load_weights(self, weights: Iterable[Tuple[str,
316
+ torch.Tensor]]) -> Set[str]:
317
stacked_params_mapping = [
318
# (param_name, shard_name, shard_id)
319
("qkv_proj", "q_proj", "q"),
@@ -354,6 +355,7 @@ class Gemma2Model(nn.Module):
355
logger.warning(
356
"Some weights are not initialized from checkpoints: %s",
357
unloaded_params)
358
+ return loaded_params
359
360
361
class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
@@ -451,13 +453,14 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
453
next_tokens = self.sampler(logits, sampling_metadata)
454
return next_tokens
455
454
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
456
+ def load_weights(self, weights: Iterable[Tuple[str,
457
+ torch.Tensor]]) -> Set[str]:
458
loader = AutoWeightsLoader(
459
self,
460
skip_prefixes=(["lm_head."]
461
if self.config.tie_word_embeddings else None),
462
)
460
- loader.load_weights(weights)
463
+ return loader.load_weights(weights)
464
465
466
class Gemma2EmbeddingModel(nn.Module, SupportsPP):
vllm/model_executor/models/gpt2.py
+6
-2
@@ -16,7 +16,7 @@
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
"""Inference-only GPT-2 model compatible with HuggingFace weights."""
19
-from typing import Iterable, List, Optional, Tuple, Union
19
+from typing import Iterable, List, Optional, Set, Tuple, Union
20
21
import torch
22
from torch import nn
@@ -298,8 +298,10 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
298
next_tokens = self.sampler(logits, sampling_metadata)
299
return next_tokens
300
301
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
301
+ def load_weights(self, weights: Iterable[Tuple[str,
302
+ torch.Tensor]]) -> Set[str]:
303
params_dict = dict(self.named_parameters(remove_duplicate=False))
304
+ loaded_params: Set[str] = set()
305
for name, loaded_weight in weights:
306
if "lm_head.weight" in name:
307
# GPT-2 ties the weights of the embedding layer and the final
@@ -328,3 +330,5 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
330
weight_loader = getattr(param, "weight_loader",
331
default_weight_loader)
332
weight_loader(param, loaded_weight)
333
+ loaded_params.add(name)
334
+ return loaded_params
vllm/model_executor/models/gpt_bigcode.py
+6
-2
@@ -17,7 +17,7 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
"""Inference-only GPTBigCode model compatible with HuggingFace weights."""
20
-from typing import Iterable, List, Optional, Tuple, Union
20
+from typing import Iterable, List, Optional, Set, Tuple, Union
21
22
import torch
23
from torch import nn
@@ -323,8 +323,10 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
323
next_tokens = self.sampler(logits, sampling_metadata)
324
return next_tokens
325
326
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
326
+ def load_weights(self, weights: Iterable[Tuple[str,
327
+ torch.Tensor]]) -> Set[str]:
328
params_dict = dict(self.named_parameters(remove_duplicate=False))
329
+ loaded_params: Set[str] = set()
330
for name, loaded_weight in weights:
331
if "lm_head.weight" in name:
332
continue
@@ -344,3 +346,5 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
346
weight_loader(param, loaded_weight, 'v')
347
else:
348
weight_loader(param, loaded_weight)
349
+ loaded_params.add(name)
350
+ return loaded_params
vllm/model_executor/models/gpt_j.py
+6
-2
@@ -15,7 +15,7 @@
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
"""Inference-only GPT-J model compatible with HuggingFace weights."""
18
-from typing import Iterable, List, Optional, Tuple, Union
18
+from typing import Iterable, List, Optional, Set, Tuple, Union
19
20
import torch
21
from torch import nn
@@ -291,7 +291,8 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
291
next_tokens = self.sampler(logits, sampling_metadata)
292
return next_tokens
293
294
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
294
+ def load_weights(self, weights: Iterable[Tuple[str,
295
+ torch.Tensor]]) -> Set[str]:
296
stacked_params_mapping = [
297
# (param_name, shard_name, shard_id)
298
("qkv_proj", "q_proj", "q"),
@@ -301,6 +302,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
302
("gate_up_proj", "up_proj", 1),
303
]
304
params_dict = dict(self.named_parameters())
305
+ loaded_params: Set[str] = set()
306
for name, loaded_weight in weights:
307
if "attn.bias" in name or "attn.masked_bias" in name:
308
continue
@@ -330,3 +332,5 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
332
weight_loader = getattr(param, "weight_loader",
333
default_weight_loader)
334
weight_loader(param, loaded_weight)
335
+ loaded_params.add(name)
336
+ return loaded_params
vllm/model_executor/models/gpt_neox.py
+6
-2
@@ -15,7 +15,7 @@
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
"""Inference-only GPT-NeoX model compatible with HuggingFace weights."""
18
-from typing import Iterable, List, Optional, Tuple, Union
18
+from typing import Iterable, List, Optional, Set, Tuple, Union
19
20
import torch
21
from torch import nn
@@ -303,8 +303,10 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
303
next_tokens = self.sampler(logits, sampling_metadata)
304
return next_tokens
305
306
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
306
+ def load_weights(self, weights: Iterable[Tuple[str,
307
+ torch.Tensor]]) -> Set[str]:
308
params_dict = dict(self.named_parameters())
309
+ loaded_params: Set[str] = set()
310
for name, loaded_weight in weights:
311
if ("attention.bias" in name or "attention.masked_bias" in name
312
or "rotary_emb.inv_freq" in name):
@@ -337,3 +339,5 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
339
weight_loader = getattr(param, "weight_loader",
340
default_weight_loader)
341
weight_loader(param, loaded_weight)
342
+ loaded_params.add(name)
343
+ return loaded_params
vllm/model_executor/models/granite.py
+7
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only IBM Granite model compatible with HuggingFace weights."""
23
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -455,7 +455,8 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
455
device=device),
456
})
457
458
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
458
+ def load_weights(self, weights: Iterable[Tuple[str,
459
+ torch.Tensor]]) -> Set[str]:
460
stacked_params_mapping = [
461
# (param_name, shard_name, shard_id)
462
(".qkv_proj", ".q_proj", "q"),
@@ -465,6 +466,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
466
(".gate_up_proj", ".up_proj", 1),
467
]
468
params_dict = dict(self.named_parameters())
469
+ loaded_params: Set[str] = set()
470
for name, loaded_weight in weights:
471
if "rotary_emb.inv_freq" in name:
472
continue
@@ -485,6 +487,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
487
default_weight_loader)
488
loaded_weight = loaded_weight[0]
489
weight_loader(param, loaded_weight)
490
+ loaded_params.add(scale_name)
491
continue
492
for (param_name, weight_name, shard_id) in stacked_params_mapping:
493
if weight_name not in name:
@@ -518,6 +521,8 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
521
weight_loader = getattr(param, "weight_loader",
522
default_weight_loader)
523
weight_loader(param, loaded_weight)
524
+ loaded_params.add(name)
525
+ return loaded_params
526
527
# If this function is called, it should always initialize KV cache scale
528
# factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/granitemoe.py
+5
-3
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only GraniteMoe model."""
23
-from typing import Iterable, List, Optional, Tuple
23
+from typing import Iterable, List, Optional, Set, Tuple
24
25
import torch
26
from torch import nn
@@ -419,7 +419,8 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419
next_tokens = self.sampler(logits, sampling_metadata)
420
return next_tokens
421
422
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
422
+ def load_weights(self, weights: Iterable[Tuple[str,
423
+ torch.Tensor]]) -> Set[str]:
424
new_weights = {}
425
for n, p in weights:
426
if n.endswith('.block_sparse_moe.input_linear.weight'):
@@ -452,4 +453,5 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
453
pass
454
else:
455
new_weights[n] = p
455
- mixtral.MixtralForCausalLM.load_weights(self, new_weights.items())
456
+ return mixtral.MixtralForCausalLM.load_weights(self,
457
+ new_weights.items())
vllm/model_executor/models/idefics2_vision_model.py
+8
-3
@@ -15,7 +15,7 @@
15
# limitations under the License.
16
"""PyTorch Idefics2 model."""
17
18
-from typing import Iterable, Optional, Tuple
18
+from typing import Iterable, Optional, Set, Tuple
19
20
import torch
21
from torch import nn
@@ -331,7 +331,8 @@ class Idefics2VisionTransformer(nn.Module):
331
last_hidden_state = self.post_layernorm(encoder_outputs)
332
return last_hidden_state
333
334
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
334
+ def load_weights(self, weights: Iterable[Tuple[str,
335
+ torch.Tensor]]) -> Set[str]:
336
stacked_params_mapping = [
337
# (param_name, shard_name, shard_id)
338
("qkv_proj", "q_proj", "q"),
@@ -339,11 +340,13 @@ class Idefics2VisionTransformer(nn.Module):
340
("qkv_proj", "v_proj", "v"),
341
]
342
params_dict = dict(self.named_parameters())
343
+ loaded_params: Set[str] = set()
344
for name, loaded_weight in weights:
345
for param_name, weight_name, shard_id in stacked_params_mapping:
346
if weight_name not in name:
347
continue
346
- param = params_dict[name.replace(weight_name, param_name)]
348
+ name = name.replace(weight_name, param_name)
349
+ param = params_dict[name]
350
weight_loader = param.weight_loader
351
weight_loader(param, loaded_weight, shard_id)
352
break
@@ -352,3 +355,5 @@ class Idefics2VisionTransformer(nn.Module):
355
weight_loader = getattr(param, "weight_loader",
356
default_weight_loader)
357
weight_loader(param, loaded_weight)
358
+ loaded_params.add(name)
359
+ return loaded_params
vllm/model_executor/models/idefics3.py
+4
-3
@@ -15,7 +15,7 @@
15
16
import math
17
from typing import (Dict, Iterable, List, Literal, Mapping, NamedTuple,
18
- Optional, Tuple, TypedDict, Union)
18
+ Optional, Set, Tuple, TypedDict, Union)
19
20
import torch
21
import torch.utils.checkpoint
@@ -751,9 +751,10 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
751
next_tokens = self.sampler(logits, sampling_metadata)
752
return next_tokens
753
754
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
754
+ def load_weights(self, weights: Iterable[Tuple[str,
755
+ torch.Tensor]]) -> Set[str]:
756
loader = AutoWeightsLoader(self)
756
- loader.load_weights(weights)
757
+ return loader.load_weights(weights)
758
759
def get_mm_mapping(self) -> MultiModelKeys:
760
"""
vllm/model_executor/models/intern_vit.py
+6
-2
@@ -5,7 +5,7 @@
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
from functools import partial
8
-from typing import Iterable, Optional, Tuple
8
+from typing import Iterable, Optional, Set, Tuple
9
10
import torch
11
import torch.nn as nn
@@ -469,10 +469,14 @@ class InternVisionModel(nn.Module):
469
470
return encoder_outputs
471
472
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
472
+ def load_weights(self, weights: Iterable[Tuple[str,
473
+ torch.Tensor]]) -> Set[str]:
474
params_dict = dict(self.named_parameters())
475
+ loaded_params: Set[str] = set()
476
for name, loaded_weight in weights:
477
param = params_dict[name]
478
weight_loader = getattr(param, "weight_loader",
479
default_weight_loader)
480
weight_loader(param, loaded_weight)
481
+ loaded_params.add(name)
482
+ return loaded_params
vllm/model_executor/models/internlm2.py
+6
-2
@@ -1,5 +1,5 @@
1
from functools import partial
2
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
2
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
3
4
import torch
5
from torch import nn
@@ -369,13 +369,15 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
369
next_tokens = self.sampler(logits, sampling_metadata)
370
return next_tokens
371
372
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
372
+ def load_weights(self, weights: Iterable[Tuple[str,
373
+ torch.Tensor]]) -> Set[str]:
374
stacked_params_mapping = [
375
# (param_name, shard_name, shard_id)
376
("gate_up_proj", "w1", 0),
377
("gate_up_proj", "w3", 1),
378
]
379
params_dict = dict(self.named_parameters())
380
+ loaded_params: Set[str] = set()
381
for name, loaded_weight in weights:
382
if "rotary_emb.inv_freq" in name:
383
continue
@@ -402,3 +404,5 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
404
weight_loader = getattr(param, "weight_loader",
405
default_weight_loader)
406
weight_loader(param, loaded_weight)
407
+ loaded_params.add(name)
408
+ return loaded_params
vllm/model_executor/models/internvl.py
+4
-3
@@ -6,7 +6,7 @@
6
# --------------------------------------------------------
7
import re
8
from functools import cached_property, partial
9
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
10
TypedDict, Union)
11
12
import torch
@@ -663,6 +663,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
663
) -> Optional[SamplerOutput]:
664
return self.language_model.sample(logits, sampling_metadata)
665
666
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
666
+ def load_weights(self, weights: Iterable[Tuple[str,
667
+ torch.Tensor]]) -> Set[str]:
668
loader = AutoWeightsLoader(self)
668
- loader.load_weights(weights)
669
+ return loader.load_weights(weights)
vllm/model_executor/models/jais.py
+6
-2
@@ -19,7 +19,7 @@
19
"""Inference-only Jais model compatible with HuggingFace weights."""
20
21
import math
22
-from typing import Iterable, List, Optional, Tuple, Union
22
+from typing import Iterable, List, Optional, Set, Tuple, Union
23
24
import torch
25
from torch import nn
@@ -350,8 +350,10 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
350
next_tokens = self.sampler(logits, sampling_metadata)
351
return next_tokens
352
353
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
353
+ def load_weights(self, weights: Iterable[Tuple[str,
354
+ torch.Tensor]]) -> Set[str]:
355
params_dict = dict(self.named_parameters(remove_duplicate=False))
356
+ loaded_params: Set[str] = set()
357
for name, loaded_weight in weights:
358
if "lm_head.weight" in name:
359
# GPT-2 ties the weights of the embedding layer and the final
@@ -382,3 +384,5 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
384
weight_loader = getattr(param, "weight_loader",
385
default_weight_loader)
386
weight_loader(param, loaded_weight)
387
+ loaded_params.add(name)
388
+ return loaded_params
vllm/model_executor/models/jamba.py
+6
-2
@@ -1,5 +1,5 @@
1
"""Inference-only Jamba model."""
2
-from typing import Iterable, List, Optional, Tuple
2
+from typing import Iterable, List, Optional, Set, Tuple
3
4
import torch
5
from torch import nn
@@ -462,7 +462,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
462
next_tokens = self.sampler(logits, sampling_metadata)
463
return next_tokens
464
465
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
465
+ def load_weights(self, weights: Iterable[Tuple[str,
466
+ torch.Tensor]]) -> Set[str]:
467
stacked_params_mapping = [
468
# (param_name, shard_name, shard_id)
469
("qkv_proj", "q_proj", "q"),
@@ -479,6 +480,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
480
num_experts=self.config.num_experts)
481
482
params_dict = dict(self.named_parameters())
483
+ loaded_params: Set[str] = set()
484
for name, loaded_weight in weights:
485
if "rotary_emb.inv_freq" in name:
486
continue
@@ -534,6 +536,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
536
weight_loader = getattr(param, "weight_loader",
537
default_weight_loader)
538
weight_loader(param, loaded_weight)
539
+ loaded_params.add(name)
540
+ return loaded_params
541
542
543
def _is_moe_layer(name: str):
vllm/model_executor/models/llama.py
+10
-5
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only LLaMA model compatible with HuggingFace weights."""
23
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -350,7 +350,8 @@ class LlamaModel(nn.Module):
350
hidden_states, _ = self.norm(hidden_states, residual)
351
return hidden_states
352
353
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
353
+ def load_weights(self, weights: Iterable[Tuple[str,
354
+ torch.Tensor]]) -> Set[str]:
355
stacked_params_mapping = [
356
# (param_name, shard_name, shard_id)
357
(".qkv_proj", ".q_proj", "q"),
@@ -360,6 +361,7 @@ class LlamaModel(nn.Module):
361
(".gate_up_proj", ".up_proj", 1),
362
]
363
params_dict = dict(self.named_parameters())
364
+ loaded_params: Set[str] = set()
365
for name, loaded_weight in weights:
366
if "rotary_emb.inv_freq" in name:
367
continue
@@ -375,6 +377,7 @@ class LlamaModel(nn.Module):
377
default_weight_loader)
378
loaded_weight = loaded_weight[0]
379
weight_loader(param, loaded_weight)
380
+ loaded_params.add(scale_name)
381
continue
382
for param_name, weight_name, shard_id in stacked_params_mapping:
383
if weight_name not in name:
@@ -390,7 +393,6 @@ class LlamaModel(nn.Module):
393
param = params_dict[name]
394
weight_loader = param.weight_loader
395
weight_loader(param, loaded_weight, shard_id)
393
-
396
break
397
else:
398
# Skip loading extra bias for GPTQ models.
@@ -408,6 +410,8 @@ class LlamaModel(nn.Module):
410
weight_loader = getattr(param, "weight_loader",
411
default_weight_loader)
412
weight_loader(param, loaded_weight)
413
+ loaded_params.add(name)
414
+ return loaded_params
415
416
# If this function is called, it should always initialize KV cache scale
417
# factors (or else raise an exception). Thus, handled exceptions should
@@ -577,13 +581,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
581
next_tokens = self.sampler(logits, sampling_metadata)
582
return next_tokens
583
580
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
584
+ def load_weights(self, weights: Iterable[Tuple[str,
585
+ torch.Tensor]]) -> Set[str]:
586
loader = AutoWeightsLoader(
587
self,
588
skip_prefixes=(["lm_head."]
589
if self.config.tie_word_embeddings else None),
590
)
586
- loader.load_weights(
591
+ return loader.load_weights(
592
self.maybe_remap_mistral(name, loaded_weight)
593
for name, loaded_weight in weights)
594
vllm/model_executor/models/llava.py
+4
-3
@@ -1,5 +1,5 @@
1
from functools import cached_property
2
-from typing import (Iterable, List, Literal, Mapping, Optional, Protocol,
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
3
Tuple, TypedDict, Union)
4
5
import torch
@@ -547,6 +547,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
547
) -> Optional[SamplerOutput]:
548
return self.language_model.sample(logits, sampling_metadata)
549
550
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
550
+ def load_weights(self, weights: Iterable[Tuple[str,
551
+ torch.Tensor]]) -> Set[str]:
552
loader = AutoWeightsLoader(self)
552
- loader.load_weights(weights)
553
+ return loader.load_weights(weights)
vllm/model_executor/models/llava_next.py
+4
-3
@@ -1,5 +1,5 @@
1
from functools import cached_property
2
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3
TypedDict, Union)
4
5
import torch
@@ -654,6 +654,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
654
) -> Optional[PoolerOutput]:
655
return self._pooler(hidden_states, pooling_metadata)
656
657
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
657
+ def load_weights(self, weights: Iterable[Tuple[str,
658
+ torch.Tensor]]) -> Set[str]:
659
loader = AutoWeightsLoader(self)
659
- loader.load_weights(weights)
660
+ return loader.load_weights(weights)
vllm/model_executor/models/llava_next_video.py
+4
-3
@@ -1,6 +1,6 @@
1
import math
2
from functools import cached_property
3
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
TypedDict, Union)
5
6
import numpy as np
@@ -445,10 +445,11 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
445
) -> Optional[SamplerOutput]:
446
return self.language_model.sample(logits, sampling_metadata)
447
448
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
448
+ def load_weights(self, weights: Iterable[Tuple[str,
449
+ torch.Tensor]]) -> Set[str]:
450
loader = AutoWeightsLoader(
451
self,
452
# This model doesn't support images for now
453
ignore_unexpected_prefixes=["image_newline"],
454
)
454
- loader.load_weights(weights)
455
+ return loader.load_weights(weights)
vllm/model_executor/models/llava_onevision.py
+4
-3
@@ -1,6 +1,6 @@
1
import math
2
from functools import cached_property
3
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
TypedDict, Union)
5
6
import numpy as np
@@ -887,6 +887,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
887
) -> Optional[SamplerOutput]:
888
return self.language_model.sample(logits, sampling_metadata)
889
890
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
890
+ def load_weights(self, weights: Iterable[Tuple[str,
891
+ torch.Tensor]]) -> Set[str]:
892
loader = AutoWeightsLoader(self)
892
- loader.load_weights(weights)
893
+ return loader.load_weights(weights)
vllm/model_executor/models/mamba.py
+6
-2
@@ -1,5 +1,5 @@
1
"""PyTorch MAMBA model."""
2
-from typing import Iterable, List, Optional, Tuple
2
+from typing import Iterable, List, Optional, Set, Tuple
3
4
import torch
5
from torch import nn
@@ -243,8 +243,10 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
243
next_tokens = self.sampler(logits, sampling_metadata)
244
return next_tokens
245
246
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
246
+ def load_weights(self, weights: Iterable[Tuple[str,
247
+ torch.Tensor]]) -> Set[str]:
248
params_dict = dict(self.named_parameters())
249
+ loaded_params: Set[str] = set()
250
for name, loaded_weight in weights:
251
if "A_log" in name:
252
name = name.replace("A_log", "A")
@@ -256,3 +258,5 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
258
weight_loader = getattr(param, "weight_loader",
259
default_weight_loader)
260
weight_loader(param, loaded_weight)
261
+ loaded_params.add(name)
262
+ return loaded_params
vllm/model_executor/models/medusa.py
+7
-2
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Optional, Tuple
1
+from typing import Iterable, List, Optional, Set, Tuple
2
3
import torch
4
import torch.nn as nn
@@ -156,8 +156,10 @@ class Medusa(nn.Module):
156
sampling_metadata=sampling_metadata,
157
)
158
159
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
159
+ def load_weights(self, weights: Iterable[Tuple[str,
160
+ torch.Tensor]]) -> Set[str]:
161
params_dict = dict(self.named_parameters())
162
+ loaded_params: Set[str] = set()
163
164
weights_map = {}
165
@@ -181,9 +183,12 @@ class Medusa(nn.Module):
183
weight_loader = getattr(param, "weight_loader",
184
default_weight_loader)
185
weight_loader(param, loaded_weight)
186
+ loaded_params.add(name)
187
188
if self.token_map is not None:
189
self.token_map.to(device=self.lm_heads[0].weight.device)
190
191
assert (self.truncated_vocab_size
192
== self.orig_vocab_size) or (self.token_map is not None)
193
+
194
+ return loaded_params
vllm/model_executor/models/minicpm.py
+6
-2
@@ -21,7 +21,7 @@
21
# limitations under the License.
22
"""Inference-only MiniCPM model compatible with HuggingFace weights."""
23
import math
24
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
24
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
25
26
import torch
27
from torch import nn
@@ -539,7 +539,8 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
539
next_tokens = self.sampler(logits, sampling_metadata)
540
return next_tokens
541
542
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
542
+ def load_weights(self, weights: Iterable[Tuple[str,
543
+ torch.Tensor]]) -> Set[str]:
544
stacked_params_mapping = [
545
# (param_name, shard_name, shard_id)
546
("qkv_proj", "q_proj", "q"),
@@ -556,6 +557,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
557
for weight_name in ["w1", "w2", "w3"]
558
]
559
params_dict = dict(self.named_parameters())
560
+ loaded_params: Set[str] = set()
561
for name, loaded_weight in weights:
562
if "rotary_emb.inv_freq" in name:
563
continue
@@ -606,3 +608,5 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
608
weight_loader = getattr(param, "weight_loader",
609
default_weight_loader)
610
weight_loader(param, loaded_weight)
611
+ loaded_params.add(name)
612
+ return loaded_params
vllm/model_executor/models/minicpmv.py
+9
-5
@@ -24,7 +24,7 @@ import math
24
import re
25
from functools import partial
26
from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27
- Tuple, TypedDict, Union)
27
+ Set, Tuple, TypedDict, Union)
28
29
import torch
30
import torch.types
@@ -602,7 +602,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
602
next_tokens = self.sampler(logits, sampling_metadata)
603
return next_tokens
604
605
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
605
+ def load_weights(self, weights: Iterable[Tuple[str,
606
+ torch.Tensor]]) -> Set[str]:
607
stacked_params_mapping = [
608
# (param_name, shard_name, shard_id)
609
("qkv_proj", "q_proj", "q"),
@@ -612,6 +613,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
613
("gate_up_proj", "up_proj", 1),
614
]
615
params_dict = dict(self.named_parameters())
616
+ loaded_params: Set[str] = set()
617
for name, loaded_weight in weights:
618
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
619
if key_to_modify in name:
@@ -630,10 +632,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
632
for param_name, weight_name, shard_id in stacked_params_mapping:
633
if weight_name not in name:
634
continue
633
- if is_pp_missing_parameter(
634
- name.replace(weight_name, param_name), self):
635
+ name = name.replace(weight_name, param_name)
636
+ if is_pp_missing_parameter(name, self):
637
continue
636
- param = params_dict[name.replace(weight_name, param_name)]
638
+ param = params_dict[name]
639
weight_loader = param.weight_loader
640
weight_loader(param, loaded_weight, shard_id)
641
break
@@ -646,6 +648,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
648
weight_loader = getattr(param, "weight_loader",
649
default_weight_loader)
650
weight_loader(param, loaded_weight)
651
+ loaded_params.add(name)
652
+ return loaded_params
653
654
def get_mm_mapping(self) -> MultiModelKeys:
655
"""
vllm/model_executor/models/mixtral.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only Mixtral model."""
23
-from typing import Iterable, List, Optional, Tuple, Union
23
+from typing import Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -404,7 +404,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
404
next_tokens = self.sampler(logits, sampling_metadata)
405
return next_tokens
406
407
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
407
+ def load_weights(self, weights: Iterable[Tuple[str,
408
+ torch.Tensor]]) -> Set[str]:
409
stacked_params_mapping = [
410
# (param_name, shard_name, shard_id)
411
("qkv_proj", "q_proj", "q"),
@@ -421,6 +422,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
422
num_experts=self.config.num_local_experts)
423
424
params_dict = dict(self.named_parameters())
425
+ loaded_params: Set[str] = set()
426
for name, loaded_weight in weights:
427
if "rotary_emb.inv_freq" in name:
428
continue
@@ -478,3 +480,5 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
480
weight_loader = getattr(param, "weight_loader",
481
default_weight_loader)
482
weight_loader(param, loaded_weight)
483
+ loaded_params.add(name)
484
+ return loaded_params
vllm/model_executor/models/mixtral_quant.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only Mixtral model."""
23
-from typing import Iterable, List, Optional, Tuple, Union
23
+from typing import Iterable, List, Optional, Set, Tuple, Union
24
25
import numpy as np
26
import torch
@@ -409,7 +409,8 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
409
next_tokens = self.sampler(logits, sampling_metadata)
410
return next_tokens
411
412
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
412
+ def load_weights(self, weights: Iterable[Tuple[str,
413
+ torch.Tensor]]) -> Set[str]:
414
stacked_params_mapping = [
415
# (param_name, shard_name, shard_id)
416
("qkv_proj", "q_proj", "q"),
@@ -418,6 +419,7 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
419
]
420
421
params_dict = dict(self.named_parameters())
422
+ loaded_params: Set[str] = set()
423
for name, loaded_weight in weights:
424
if "rotary_emb.inv_freq" in name:
425
continue
@@ -448,3 +450,5 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
450
weight_loader = getattr(param, "weight_loader",
451
default_weight_loader)
452
weight_loader(param, loaded_weight)
453
+ loaded_params.add(name)
454
+ return loaded_params
vllm/model_executor/models/mllama.py
+6
-3
@@ -13,7 +13,7 @@
13
# limitations under the License.
14
"""PyTorch Mllama model."""
15
import math
16
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
16
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
17
TypedDict, Union)
18
19
import numpy as np
@@ -1427,7 +1427,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1427
1428
return outputs
1429
1430
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1430
+ def load_weights(self, weights: Iterable[Tuple[str,
1431
+ torch.Tensor]]) -> Set[str]:
1432
stacked_params_mapping = [
1433
# (param_name, shard_name, shard_id)
1434
(".qkv_proj", ".q_proj", "q"),
@@ -1437,7 +1438,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1438
(".gate_up_proj", ".up_proj", 1),
1439
]
1440
params_dict = dict(self.named_parameters())
1440
- updated_params = set()
1441
+ updated_params: Set[str] = set()
1442
for name, loaded_weight in weights:
1443
if 'patch_embedding.weight' in name:
1444
name = name.replace('patch_embedding.weight',
@@ -1457,6 +1458,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1458
weight_loader = getattr(param, "weight_loader",
1459
default_weight_loader)
1460
weight_loader(param, loaded_weight)
1461
+ updated_params.add(name)
1462
+ return updated_params
1463
1464
1465
def skip_attention_mask(sparse_mask: List[List[int]]) -> bool:
vllm/model_executor/models/mlp_speculator.py
+6
-2
@@ -1,5 +1,5 @@
1
import math
2
-from typing import Iterable, List, Tuple
2
+from typing import Iterable, List, Set, Tuple
3
4
import torch
5
import torch.nn as nn
@@ -188,11 +188,15 @@ class MLPSpeculator(nn.Module):
188
189
return next_tokens
190
191
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
191
+ def load_weights(self, weights: Iterable[Tuple[str,
192
+ torch.Tensor]]) -> Set[str]:
193
params_dict = dict(self.named_parameters())
194
+ loaded_params: Set[str] = set()
195
for name, loaded_weight in weights:
196
param = params_dict.get(name.replace("speculator.", ""))
197
if param is not None:
198
weight_loader = getattr(param, "weight_loader",
199
default_weight_loader)
200
weight_loader(param, loaded_weight)
201
+ loaded_params.add(name)
202
+ return loaded_params
vllm/model_executor/models/mpt.py
+6
-2
@@ -1,6 +1,6 @@
1
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
2
import math
3
-from typing import Iterable, List, Optional, Tuple, Union
3
+from typing import Iterable, List, Optional, Set, Tuple, Union
4
5
import torch
6
import torch.nn as nn
@@ -324,8 +324,10 @@ class MPTForCausalLM(nn.Module, SupportsPP):
324
next_tokens = self.sampler(logits, sampling_metadata)
325
return next_tokens
326
327
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
327
+ def load_weights(self, weights: Iterable[Tuple[str,
328
+ torch.Tensor]]) -> Set[str]:
329
params_dict = dict(self.named_parameters(remove_duplicate=False))
330
+ loaded_params: Set[str] = set()
331
for name, loaded_weight in weights:
332
# Skip loading extra bias for GPTQ models.
333
if name.endswith(".bias") and name not in params_dict:
@@ -336,3 +338,5 @@ class MPTForCausalLM(nn.Module, SupportsPP):
338
weight_loader = getattr(param, "weight_loader",
339
default_weight_loader)
340
weight_loader(param, loaded_weight)
341
+ loaded_params.add(name)
342
+ return loaded_params
vllm/model_executor/models/nemotron.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only Nemotron model compatible with HuggingFace weights."""
23
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -474,7 +474,8 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
474
next_tokens = self.sampler(logits, sampling_metadata)
475
return next_tokens
476
477
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
477
+ def load_weights(self, weights: Iterable[Tuple[str,
478
+ torch.Tensor]]) -> Set[str]:
479
stacked_params_mapping = [
480
# (param_name, shard_name, shard_id)
481
(".qkv_proj", ".q_proj", "q"),
@@ -482,6 +483,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
483
(".qkv_proj", ".v_proj", "v"),
484
]
485
params_dict = dict(self.named_parameters())
486
+ loaded_params: Set[str] = set()
487
for name, loaded_weight in weights:
488
if "rotary_emb.inv_freq" in name:
489
continue
@@ -522,3 +524,5 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
524
weight_loader = getattr(param, "weight_loader",
525
default_weight_loader)
526
weight_loader(param, loaded_weight)
527
+ loaded_params.add(name)
528
+ return loaded_params
vllm/model_executor/models/olmo.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only OLMo model compatible with HuggingFace weights."""
23
-from typing import Iterable, List, Optional, Tuple, Union
23
+from typing import Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -356,7 +356,8 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
356
next_tokens = self.sampler(logits, sampling_metadata)
357
return next_tokens
358
359
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
359
+ def load_weights(self, weights: Iterable[Tuple[str,
360
+ torch.Tensor]]) -> Set[str]:
361
stacked_params_mapping = [
362
# (param_name, shard_name, shard_id)
363
("qkv_proj", "q_proj", "q"),
@@ -366,6 +367,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
367
("gate_up_proj", "up_proj", 1),
368
]
369
params_dict = dict(self.named_parameters(remove_duplicate=False))
370
+ loaded_params: Set[str] = set()
371
for name, loaded_weight in weights:
372
if "rotary_emb.inv_freq" in name:
373
continue
@@ -402,3 +404,5 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
404
weight_loader = getattr(param, "weight_loader",
405
default_weight_loader)
406
weight_loader(param, loaded_weight)
407
+ loaded_params.add(name)
408
+ return loaded_params
vllm/model_executor/models/olmoe.py
+6
-2
@@ -10,7 +10,7 @@
10
# See the License for the specific language governing permissions and
11
# limitations under the License.
12
"""Inference-only OLMoE model compatible with HuggingFace weights."""
13
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
13
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
14
15
import torch
16
from torch import nn
@@ -364,7 +364,8 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
364
next_tokens = self.sampler(logits, sampling_metadata)
365
return next_tokens
366
367
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
367
+ def load_weights(self, weights: Iterable[Tuple[str,
368
+ torch.Tensor]]) -> Set[str]:
369
stacked_params_mapping = [
370
# (param_name, shard_name, shard_id)
371
("qkv_proj", "q_proj", "q"),
@@ -383,6 +384,7 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
384
num_experts=self.config.num_experts)
385
386
params_dict = dict(self.named_parameters())
387
+ loaded_params: Set[str] = set()
388
for name, loaded_weight in weights:
389
if "rotary_emb.inv_freq" in name:
390
continue
@@ -455,3 +457,5 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
457
weight_loader = getattr(param, "weight_loader",
458
default_weight_loader)
459
weight_loader(param, loaded_weight)
460
+ loaded_params.add(name)
461
+ return loaded_params
vllm/model_executor/models/opt.py
+6
-2
@@ -16,7 +16,7 @@
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
"""Inference-only OPT model compatible with HuggingFace weights."""
19
-from typing import Iterable, List, Optional, Tuple, Union
19
+from typing import Iterable, List, Optional, Set, Tuple, Union
20
21
import torch
22
from torch import nn
@@ -394,7 +394,8 @@ class OPTForCausalLM(nn.Module, SupportsPP):
394
next_tokens = self.sampler(logits, sampling_metadata)
395
return next_tokens
396
397
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
397
+ def load_weights(self, weights: Iterable[Tuple[str,
398
+ torch.Tensor]]) -> Set[str]:
399
stacked_params_mapping = [
400
# (param_name, shard_name, shard_id)
401
("qkv_proj", "q_proj", "q"),
@@ -402,6 +403,7 @@ class OPTForCausalLM(nn.Module, SupportsPP):
403
("qkv_proj", "v_proj", "v"),
404
]
405
params_dict = dict(self.named_parameters(remove_duplicate=False))
406
+ loaded_params: Set[str] = set()
407
for name, loaded_weight in weights:
408
if "lm_head.weight" in name and self.config.tie_word_embeddings:
409
continue
@@ -431,3 +433,5 @@ class OPTForCausalLM(nn.Module, SupportsPP):
433
weight_loader = getattr(param, "weight_loader",
434
default_weight_loader)
435
weight_loader(param, loaded_weight)
436
+ loaded_params.add(name)
437
+ return loaded_params
vllm/model_executor/models/orion.py
+6
-2
@@ -3,7 +3,7 @@
3
# Copyright (c) OrionStar Inc.
4
# LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE
5
"""Inference-only Orion-14B model compatible with HuggingFace weights."""
6
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
6
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
7
8
import torch
9
from torch import nn
@@ -327,7 +327,8 @@ class OrionForCausalLM(nn.Module, SupportsPP):
327
next_tokens = self.sampler(logits, sampling_metadata)
328
return next_tokens
329
330
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
330
+ def load_weights(self, weights: Iterable[Tuple[str,
331
+ torch.Tensor]]) -> Set[str]:
332
stacked_params_mapping = [
333
# (param_name, shard_name, shard_id)
334
("qkv_proj", "q_proj", "q"),
@@ -337,6 +338,7 @@ class OrionForCausalLM(nn.Module, SupportsPP):
338
("gate_up_proj", "up_proj", 1),
339
]
340
params_dict = dict(self.named_parameters())
341
+ loaded_params: Set[str] = set()
342
for name, loaded_weight in weights:
343
if "rotary_emb.inv_freq" in name:
344
continue
@@ -368,3 +370,5 @@ class OrionForCausalLM(nn.Module, SupportsPP):
370
weight_loader = getattr(param, "weight_loader",
371
default_weight_loader)
372
weight_loader(param, loaded_weight)
373
+ loaded_params.add(name)
374
+ return loaded_params
vllm/model_executor/models/paligemma.py
+4
-3
@@ -1,4 +1,4 @@
1
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
1
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
2
TypedDict, Union)
3
4
import torch
@@ -295,6 +295,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
295
) -> Optional[SamplerOutput]:
296
return self.language_model.sample(logits, sampling_metadata)
297
298
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
298
+ def load_weights(self, weights: Iterable[Tuple[str,
299
+ torch.Tensor]]) -> Set[str]:
300
loader = AutoWeightsLoader(self)
300
- loader.load_weights(weights)
301
+ return loader.load_weights(weights)
vllm/model_executor/models/persimmon.py
+6
-2
@@ -19,7 +19,7 @@
19
# See the License for the specific language governing permissions and
20
# limitations under the License.
21
"""Inference-only persimmon model compatible with HuggingFace weights."""
22
-from typing import Iterable, List, Optional, Tuple, Union
22
+from typing import Iterable, List, Optional, Set, Tuple, Union
23
24
import torch
25
from torch import nn
@@ -324,8 +324,10 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
324
next_tokens = self.sampler(logits, sampling_metadata)
325
return next_tokens
326
327
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
327
+ def load_weights(self, weights: Iterable[Tuple[str,
328
+ torch.Tensor]]) -> Set[str]:
329
params_dict = dict(self.named_parameters(remove_duplicate=False))
330
+ loaded_params: Set[str] = set()
331
for name, loaded_weight in weights:
332
if "rotary_emb.inv_freq" in name:
333
continue
@@ -358,3 +360,5 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
360
weight_loader = getattr(param, "weight_loader",
361
default_weight_loader)
362
weight_loader(param, loaded_weight)
363
+ loaded_params.add(name)
364
+ return loaded_params
vllm/model_executor/models/phi.py
+6
-2
@@ -34,7 +34,7 @@
34
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36
"""Inference-only Phi-1.5 model compatible with HuggingFace weights."""
37
-from typing import Iterable, List, Optional, Tuple, Union
37
+from typing import Iterable, List, Optional, Set, Tuple, Union
38
39
import torch
40
from torch import nn
@@ -345,7 +345,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
345
next_tokens = self.sampler(logits, sampling_metadata)
346
return next_tokens
347
348
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
348
+ def load_weights(self, weights: Iterable[Tuple[str,
349
+ torch.Tensor]]) -> Set[str]:
350
stacked_params_mapping = [
351
# (param_name, shard_name, shard_id)
352
("qkv_proj", "q_proj", "q"),
@@ -353,6 +354,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
354
("qkv_proj", "v_proj", "v")
355
]
356
params_dict = dict(self.named_parameters())
357
+ loaded_params: Set[str] = set()
358
359
for name, loaded_weight in weights:
360
if "rotary_emb.inv_freq" in name:
@@ -383,3 +385,5 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
385
weight_loader = getattr(param, "weight_loader",
386
default_weight_loader)
387
weight_loader(param, loaded_weight)
388
+ loaded_params.add(name)
389
+ return loaded_params
vllm/model_executor/models/phi3_small.py
+6
-2
@@ -1,5 +1,5 @@
1
import math
2
-from typing import Iterable, List, Optional, Tuple, Union
2
+from typing import Iterable, List, Optional, Set, Tuple, Union
3
4
import torch
5
from torch import nn
@@ -457,9 +457,11 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
457
sampling_metadata)
458
return next_tokens
459
460
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
460
+ def load_weights(self, weights: Iterable[Tuple[str,
461
+ torch.Tensor]]) -> Set[str]:
462
463
params_dict = dict(self.named_parameters())
464
+ loaded_params: Set[str] = set()
465
for name, loaded_weight in weights:
466
if "rotary_emb.inv_freq" in name:
467
continue
@@ -471,3 +473,5 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
473
weight_loader = getattr(param, "weight_loader",
474
default_weight_loader)
475
weight_loader(param, loaded_weight)
476
+ loaded_params.add(name)
477
+ return loaded_params
vllm/model_executor/models/phi3v.py
+6
-3
@@ -15,7 +15,7 @@
15
import itertools
16
import re
17
from functools import cached_property, lru_cache
18
-from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
18
+from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
19
Tuple, TypedDict, Union)
20
21
import numpy as np
@@ -744,7 +744,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
744
) -> Optional[PoolerOutput]:
745
return self._pooler(hidden_states, pooling_metadata)
746
747
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
747
+ def load_weights(self, weights: Iterable[Tuple[str,
748
+ torch.Tensor]]) -> Set[str]:
749
hf_to_vllm_mapper = WeightsMapper(
750
orig_to_new_prefix={
751
"model.vision_embed_tokens.wte": "embed_tokens",
@@ -759,5 +760,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
760
761
# The HF config doesn't specify whether these are tied,
762
# so we detect it this way
762
- if "embed_tokens" not in autoloaded_weights:
763
+ if "embed_tokens.weight" not in autoloaded_weights:
764
self.embed_tokens = self.language_model.model.embed_tokens
765
+ autoloaded_weights.add("embed_tokens.weight")
766
+ return autoloaded_weights
vllm/model_executor/models/phimoe.py
+6
-2
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only PhiMoE model."""
23
-from typing import Iterable, List, Optional, Tuple, Union
23
+from typing import Iterable, List, Optional, Set, Tuple, Union
24
25
import torch
26
from torch import nn
@@ -598,7 +598,8 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
598
next_tokens = self.sampler(logits, sampling_metadata)
599
return next_tokens
600
601
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
601
+ def load_weights(self, weights: Iterable[Tuple[str,
602
+ torch.Tensor]]) -> Set[str]:
603
stacked_params_mapping = [
604
# (param_name, shard_name, shard_id)
605
("qkv_proj", "q_proj", "q"),
@@ -613,6 +614,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
614
num_experts=self.config.num_local_experts)
615
616
params_dict = dict(self.named_parameters())
617
+ loaded_params: Set[str] = set()
618
for name, loaded_weight in weights:
619
if "rotary_emb.inv_freq" in name:
620
continue
@@ -666,3 +668,5 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
668
weight_loader = getattr(param, "weight_loader",
669
default_weight_loader)
670
weight_loader(param, loaded_weight)
671
+ loaded_params.add(name)
672
+ return loaded_params
vllm/model_executor/models/pixtral.py
+8
-4
@@ -1,7 +1,7 @@
1
from dataclasses import dataclass, fields
2
from functools import cached_property
3
from itertools import tee
4
-from typing import Iterable, List, Mapping, Optional, Tuple, Union
4
+from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union
5
6
import numpy
7
import torch
@@ -1053,7 +1053,8 @@ class PixtralHFVisionModel(nn.Module):
1053
1054
# (TODO) Add prefix argument for filtering out weights to be loaded
1055
# ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
1056
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1056
+ def load_weights(self, weights: Iterable[Tuple[str,
1057
+ torch.Tensor]]) -> Set[str]:
1058
stacked_params_mapping = [
1059
# (param_name, shard_name, shard_id)
1060
(".qkv_proj", ".q_proj", "q"),
@@ -1063,6 +1064,7 @@ class PixtralHFVisionModel(nn.Module):
1064
(".gate_up_proj", ".up_proj", 1),
1065
]
1066
params_dict = dict(self.named_parameters())
1067
+ loaded_params: Set[str] = set()
1068
layer_count = len(self.transformer.layers)
1069
1070
for name, loaded_weight in weights:
@@ -1075,8 +1077,8 @@ class PixtralHFVisionModel(nn.Module):
1077
for (param_name, weight_name, shard_id) in stacked_params_mapping:
1078
if weight_name not in name:
1079
continue
1078
-
1079
- param = params_dict[name.replace(weight_name, param_name)]
1080
+ name = name.replace(weight_name, param_name)
1081
+ param = params_dict[name]
1082
weight_loader = param.weight_loader
1083
weight_loader(param, loaded_weight, shard_id)
1084
break
@@ -1085,3 +1087,5 @@ class PixtralHFVisionModel(nn.Module):
1087
weight_loader = getattr(param, "weight_loader",
1088
default_weight_loader)
1089
weight_loader(param, loaded_weight)
1090
+ loaded_params.add(name)
1091
+ return loaded_params
vllm/model_executor/models/qwen.py
+6
-2
@@ -8,7 +8,7 @@ import math
8
import re
9
from functools import partial
10
from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
11
- Optional, Tuple, TypedDict, Union)
11
+ Optional, Set, Tuple, TypedDict, Union)
12
13
import numpy as np
14
import torch
@@ -964,13 +964,15 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
964
next_tokens = self.sampler(logits, sampling_metadata)
965
return next_tokens
966
967
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
967
+ def load_weights(self, weights: Iterable[Tuple[str,
968
+ torch.Tensor]]) -> Set[str]:
969
stacked_params_mapping = [
970
# (param_name, shard_name, shard_id)
971
("gate_up_proj", "w2", 0),
972
("gate_up_proj", "w1", 1),
973
]
974
params_dict = dict(self.named_parameters())
975
+ loaded_params: Set[str] = set()
976
for name, loaded_weight in weights:
977
if "rotary_emb.inv_freq" in name:
978
continue
@@ -999,6 +1001,8 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
1001
weight_loader = getattr(param, "weight_loader",
1002
default_weight_loader)
1003
weight_loader(param, loaded_weight)
1004
+ loaded_params.add(name)
1005
+ return loaded_params
1006
1007
1008
class QWenLLM(QWenBaseModel):
vllm/model_executor/models/qwen2.py
+12
-6
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Qwen2 model compatible with HuggingFace weights."""
24
-from typing import Iterable, List, Optional, Tuple, Union
24
+from typing import Iterable, List, Optional, Set, Tuple, Union
25
26
import torch
27
from torch import nn
@@ -332,7 +332,8 @@ class Qwen2Model(nn.Module):
332
hidden_states, _ = self.norm(hidden_states, residual)
333
return hidden_states
334
335
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
335
+ def load_weights(self, weights: Iterable[Tuple[str,
336
+ torch.Tensor]]) -> Set[str]:
337
stacked_params_mapping = [
338
# (param_name, shard_name, shard_id)
339
("qkv_proj", "q_proj", "q"),
@@ -342,6 +343,7 @@ class Qwen2Model(nn.Module):
343
("gate_up_proj", "up_proj", 1),
344
]
345
params_dict = dict(self.named_parameters(remove_duplicate=False))
346
+ loaded_params: Set[str] = set()
347
for name, loaded_weight in weights:
348
if "rotary_emb.inv_freq" in name:
349
continue
@@ -372,6 +374,8 @@ class Qwen2Model(nn.Module):
374
weight_loader = getattr(param, "weight_loader",
375
default_weight_loader)
376
weight_loader(param, loaded_weight)
377
+ loaded_params.add(name)
378
+ return loaded_params
379
380
381
class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
@@ -494,13 +498,14 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
498
) -> Optional[PoolerOutput]:
499
return self._pooler(hidden_states, pooling_metadata)
500
497
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
501
+ def load_weights(self, weights: Iterable[Tuple[str,
502
+ torch.Tensor]]) -> Set[str]:
503
loader = AutoWeightsLoader(
504
self,
505
skip_prefixes=(["lm_head."]
506
if self.config.tie_word_embeddings else None),
507
)
503
- loader.load_weights(weights)
508
+ return loader.load_weights(weights)
509
510
511
class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
@@ -564,7 +569,8 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
569
) -> Optional[PoolerOutput]:
570
return self._pooler(hidden_states, pooling_metadata)
571
567
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
572
+ def load_weights(self, weights: Iterable[Tuple[str,
573
+ torch.Tensor]]) -> Set[str]:
574
loader = AutoWeightsLoader(self,
575
ignore_unexpected_prefixes=["lm_head."])
570
- loader.load_weights(weights)
576
+ return loader.load_weights(weights)
vllm/model_executor/models/qwen2_audio.py
+7
-2
@@ -20,7 +20,8 @@
20
# limitations under the License.
21
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22
from functools import lru_cache
23
-from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
23
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24
+ Union)
25
26
import librosa
27
import numpy as np
@@ -420,7 +421,8 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
421
next_tokens = self.sampler(logits, sampling_metadata)
422
return next_tokens
423
423
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
424
+ def load_weights(self, weights: Iterable[Tuple[str,
425
+ torch.Tensor]]) -> Set[str]:
426
stacked_params_mapping = [
427
# (param_name, shard_name, shard_id)
428
("qkv_proj", "q_proj", "q"),
@@ -430,6 +432,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
432
("gate_up_proj", "up_proj", 1),
433
]
434
params_dict = dict(self.named_parameters(remove_duplicate=False))
435
+ loaded_params: Set[str] = set()
436
for name, loaded_weight in weights:
437
if "rotary_emb.inv_freq" in name:
438
continue
@@ -463,3 +466,5 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
466
weight_loader = getattr(param, "weight_loader",
467
default_weight_loader)
468
weight_loader(param, loaded_weight)
469
+ loaded_params.add(name)
470
+ return loaded_params
vllm/model_executor/models/qwen2_cls.py
+4
-3
@@ -4,7 +4,7 @@
4
# Copyright 2024 The Qwen team.
5
# Copyright 2023 The vLLM team.
6
"""Inference-only Qwen2-Classification model compatible with HF weights."""
7
-from typing import Iterable, List, Optional, Tuple
7
+from typing import Iterable, List, Optional, Set, Tuple
8
9
import torch
10
from torch import nn
@@ -97,7 +97,8 @@ class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
97
) -> Optional[PoolerOutput]:
98
return self._pooler(hidden_states, pooling_metadata)
99
100
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
100
+ def load_weights(self, weights: Iterable[Tuple[str,
101
+ torch.Tensor]]) -> Set[str]:
102
loader = AutoWeightsLoader(self,
103
ignore_unexpected_prefixes=["lm_head."])
103
- loader.load_weights(weights)
104
+ return loader.load_weights(weights)
vllm/model_executor/models/qwen2_moe.py
+6
-2
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Qwen2MoE model compatible with HuggingFace weights."""
24
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
24
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
25
26
import torch
27
import torch.nn.functional as F
@@ -436,7 +436,8 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
436
next_tokens = self.sampler(logits, sampling_metadata)
437
return next_tokens
438
439
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
439
+ def load_weights(self, weights: Iterable[Tuple[str,
440
+ torch.Tensor]]) -> Set[str]:
441
stacked_params_mapping = [
442
# (param_name, shard_name, shard_id)
443
("qkv_proj", "q_proj", "q"),
@@ -455,6 +456,7 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
456
num_experts=self.config.num_experts)
457
458
params_dict = dict(self.named_parameters())
459
+ loaded_params: Set[str] = set()
460
for name, loaded_weight in weights:
461
if "rotary_emb.inv_freq" in name:
462
continue
@@ -532,3 +534,5 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
534
weight_loader = getattr(param, "weight_loader",
535
default_weight_loader)
536
weight_loader(param, loaded_weight)
537
+ loaded_params.add(name)
538
+ return loaded_params
vllm/model_executor/models/qwen2_rm.py
+4
-3
@@ -3,7 +3,7 @@
3
# Copyright 2024 The Qwen team.
4
# Copyright 2023 The vLLM team.
5
"""Inference-only Qwen2-RM model compatible with HuggingFace weights."""
6
-from typing import Iterable, List, Optional, Tuple, Union
6
+from typing import Iterable, List, Optional, Set, Tuple, Union
7
8
import torch
9
from torch import nn
@@ -110,7 +110,8 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
110
) -> Optional[PoolerOutput]:
111
return self._pooler(hidden_states, pooling_metadata)
112
113
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
113
+ def load_weights(self, weights: Iterable[Tuple[str,
114
+ torch.Tensor]]) -> Set[str]:
115
loader = AutoWeightsLoader(self,
116
ignore_unexpected_prefixes=["lm_head."])
116
- loader.load_weights(weights)
117
+ return loader.load_weights(weights)
vllm/model_executor/models/qwen2_vl.py
+6
-2
@@ -23,7 +23,7 @@
23
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24
from functools import partial
25
from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26
- Optional, Tuple, Type, TypedDict, Union)
26
+ Optional, Set, Tuple, Type, TypedDict, Union)
27
28
import torch
29
import torch.nn as nn
@@ -1333,7 +1333,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1333
) -> Optional[PoolerOutput]:
1334
return self._pooler(hidden_states, pooling_metadata)
1335
1336
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1336
+ def load_weights(self, weights: Iterable[Tuple[str,
1337
+ torch.Tensor]]) -> Set[str]:
1338
stacked_params_mapping = [
1339
# (param_name, shard_name, shard_id)
1340
("qkv_proj", "q_proj", "q"),
@@ -1343,6 +1344,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1344
("gate_up_proj", "gate_proj", 0),
1345
]
1346
params_dict = dict(self.named_parameters(remove_duplicate=False))
1347
+ loaded_params: Set[str] = set()
1348
for name, loaded_weight in weights:
1349
if "rotary_emb.inv_freq" in name:
1350
continue
@@ -1392,3 +1394,5 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1394
weight_loader = getattr(param, "weight_loader",
1395
default_weight_loader)
1396
weight_loader(param, loaded_weight)
1397
+ loaded_params.add(name)
1398
+ return loaded_params
vllm/model_executor/models/siglip.py
+8
-3
@@ -2,7 +2,7 @@
2
within a vision language model."""
3
4
import math
5
-from typing import Iterable, List, Optional, Tuple, Union
5
+from typing import Iterable, List, Optional, Set, Tuple, Union
6
7
import numpy as np
8
import torch
@@ -594,7 +594,8 @@ class SiglipVisionModel(nn.Module):
594
interpolate_pos_encoding=interpolate_pos_encoding,
595
)
596
597
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
597
+ def load_weights(self, weights: Iterable[Tuple[str,
598
+ torch.Tensor]]) -> Set[str]:
599
stacked_params_mapping = [
600
# (param_name, shard_name, shard_id)
601
("qkv_proj", "q_proj", "q"),
@@ -602,6 +603,7 @@ class SiglipVisionModel(nn.Module):
603
("qkv_proj", "v_proj", "v"),
604
] if self.shard_weight else []
605
params_dict = dict(self.named_parameters())
606
+ loaded_params: Set[str] = set()
607
layer_count = len(self.vision_model.encoder.layers)
608
609
for name, loaded_weight in weights:
@@ -619,8 +621,9 @@ class SiglipVisionModel(nn.Module):
621
for (param_name, weight_name, shard_id) in stacked_params_mapping:
622
if weight_name not in name:
623
continue
624
+ name = name.replace(weight_name, param_name)
625
623
- param = params_dict[name.replace(weight_name, param_name)]
626
+ param = params_dict[name]
627
weight_loader = param.weight_loader
628
weight_loader(param, loaded_weight, shard_id)
629
break
@@ -629,3 +632,5 @@ class SiglipVisionModel(nn.Module):
632
weight_loader = getattr(param, "weight_loader",
633
default_weight_loader)
634
weight_loader(param, loaded_weight)
635
+ loaded_params.add(name)
636
+ return loaded_params
vllm/model_executor/models/solar.py
+7
-2
@@ -21,7 +21,7 @@
21
# limitations under the License.
22
"""Inference-only Solar model compatible with HuggingFace weights."""
23
24
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
24
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
25
26
import torch
27
from torch import nn
@@ -477,7 +477,8 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
477
next_tokens = self.sampler(logits, sampling_metadata)
478
return next_tokens
479
480
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
480
+ def load_weights(self, weights: Iterable[Tuple[str,
481
+ torch.Tensor]]) -> Set[str]:
482
stacked_params_mapping = [
483
# (param_name, shard_name, shard_id)
484
(".qkv_proj", ".q_proj", "q"),
@@ -487,6 +488,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
488
(".gate_up_proj", ".up_proj", 1),
489
]
490
params_dict = dict(self.named_parameters())
491
+ loaded_params: Set[str] = set()
492
for name, loaded_weight in weights:
493
if "rotary_emb.inv_freq" in name:
494
continue
@@ -502,6 +504,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
504
default_weight_loader)
505
loaded_weight = loaded_weight[0]
506
weight_loader(param, loaded_weight)
507
+ loaded_params.add(scale_name)
508
continue
509
for param_name, weight_name, shard_id in stacked_params_mapping:
510
if weight_name not in name:
@@ -535,6 +538,8 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
538
weight_loader = getattr(param, "weight_loader",
539
default_weight_loader)
540
weight_loader(param, loaded_weight)
541
+ loaded_params.add(name)
542
+ return loaded_params
543
544
# If this function is called, it should always initialize KV cache scale
545
# factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/stablelm.py
+6
-2
@@ -18,7 +18,7 @@
18
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19
"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM)
20
model compatible with HuggingFace weights."""
21
-from typing import Iterable, List, Optional, Tuple, Union
21
+from typing import Iterable, List, Optional, Set, Tuple, Union
22
23
import torch
24
from torch import nn
@@ -306,7 +306,8 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
306
next_tokens = self.sampler(logits, sampling_metadata)
307
return next_tokens
308
309
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
309
+ def load_weights(self, weights: Iterable[Tuple[str,
310
+ torch.Tensor]]) -> Set[str]:
311
stacked_params_mapping = [
312
# (param_name, shard_name, shard_id)
313
("qkv_proj", "q_proj", "q"),
@@ -316,6 +317,7 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
317
("gate_up_proj", "up_proj", 1),
318
]
319
params_dict = dict(self.named_parameters())
320
+ loaded_params: Set[str] = set()
321
for name, loaded_weight in weights:
322
if "rotary_emb.inv_freq" in name:
323
continue
@@ -347,3 +349,5 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
349
weight_loader = getattr(param, "weight_loader",
350
default_weight_loader)
351
weight_loader(param, loaded_weight)
352
+ loaded_params.add(name)
353
+ return loaded_params
vllm/model_executor/models/starcoder2.py
+6
-2
@@ -17,7 +17,7 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
""" PyTorch Starcoder2 model."""
20
-from typing import Iterable, List, Optional, Tuple, Union
20
+from typing import Iterable, List, Optional, Set, Tuple, Union
21
22
import torch
23
from torch import nn
@@ -314,7 +314,8 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
314
next_tokens = self.sampler(logits, sampling_metadata)
315
return next_tokens
316
317
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
317
+ def load_weights(self, weights: Iterable[Tuple[str,
318
+ torch.Tensor]]) -> Set[str]:
319
stacked_params_mapping = [
320
# (param_name, shard_name, shard_id)
321
("qkv_proj", "q_proj", "q"),
@@ -323,6 +324,7 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
324
]
325
326
params_dict = dict(self.named_parameters(remove_duplicate=False))
327
+ loaded_params: Set[str] = set()
328
for name, loaded_weight in weights:
329
if "rotary_emb.inv_freq" in name:
330
continue
@@ -346,3 +348,5 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
348
weight_loader = getattr(param, "weight_loader",
349
default_weight_loader)
350
weight_loader(param, loaded_weight)
351
+ loaded_params.add(name)
352
+ return loaded_params
vllm/model_executor/models/ultravox.py
+4
-3
@@ -3,7 +3,7 @@
3
4
import math
5
from functools import cached_property, lru_cache
6
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
6
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7
TypedDict, Union, cast)
8
9
import numpy as np
@@ -504,10 +504,11 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
504
) -> Optional[SamplerOutput]:
505
return self.language_model.sample(logits, sampling_metadata)
506
507
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
507
+ def load_weights(self, weights: Iterable[Tuple[str,
508
+ torch.Tensor]]) -> Set[str]:
509
hf_to_vllm_mapper = WeightsMapper(
510
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
511
512
loader = AutoWeightsLoader(self,
513
ignore_unexpected_prefixes=["audio_tower."])
513
- loader.load_weights(weights, mapper=hf_to_vllm_mapper)
514
+ return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/utils.py
+6
-5
@@ -1,7 +1,7 @@
1
import itertools
2
from dataclasses import dataclass, field
3
from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
4
- Optional, Protocol, Tuple, Union, overload)
4
+ Optional, Protocol, Set, Tuple, Union, overload)
5
6
import torch
7
import torch.nn as nn
@@ -172,8 +172,9 @@ class AutoWeightsLoader:
172
if module != self.module:
173
module_load_weights = getattr(module, "load_weights", None)
174
if callable(module_load_weights):
175
- module_load_weights(weights)
176
- return
175
+ loaded_params = module_load_weights(weights)
176
+ yield from map(lambda x: self._get_qualname(base_prefix, x),
177
+ loaded_params)
178
179
child_modules = dict(module.named_children())
180
child_params = dict(module.named_parameters(recurse=False))
@@ -222,11 +223,11 @@ class AutoWeightsLoader:
223
weights: Iterable[Tuple[str, torch.Tensor]],
224
*,
225
mapper: Optional[WeightsMapper] = None,
225
- ) -> List[str]:
226
+ ) -> Set[str]:
227
if mapper is not None:
228
weights = mapper.apply(weights)
229
229
- autoloaded_weights = list(self._load_module("", self.module, weights))
230
+ autoloaded_weights = set(self._load_module("", self.module, weights))
231
return autoloaded_weights
232
233
vllm/model_executor/models/xverse.py
+6
-2
@@ -19,7 +19,7 @@
19
# See the License for the specific language governing permissions and
20
# limitations under the License.
21
"""Inference-only Xverse model compatible with HuggingFace weights."""
22
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
22
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
23
24
import torch
25
from torch import nn
@@ -376,7 +376,8 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
376
next_tokens = self.sampler(logits, sampling_metadata)
377
return next_tokens
378
379
- def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
379
+ def load_weights(self, weights: Iterable[Tuple[str,
380
+ torch.Tensor]]) -> Set[str]:
381
stacked_params_mapping = [
382
("qkv_proj", "q_proj", "q"),
383
("qkv_proj", "k_proj", "k"),
@@ -385,6 +386,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
386
("gate_up_proj", "up_proj", 1),
387
]
388
params_dict = dict(self.named_parameters())
389
+ loaded_params: Set[str] = set()
390
for name, loaded_weight in weights:
391
if ("rotary_emb.inv_freq" in name
392
or "rotary_emb.cos_cached" in name
@@ -413,3 +415,5 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
415
weight_loader = getattr(param, "weight_loader",
416
default_weight_loader)
417
weight_loader(param, loaded_weight)
418
+ loaded_params.add(name)
419
+ return loaded_params