@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4e464333e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Misc] Add uninitialized params tracking for `AutoWeightsLoader` (#10327)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 18, 2024 at 09:07 UTC c4e464333eac5a46e1cc2701e095a44057c82927
74 files changed +454 -185
vllm/model_executor/model_loader/loader.py
+11 -1
@@ -334,7 +334,17 @@ class DefaultModelLoader(BaseModelLoader):
334 with target_device:
335 model = _initialize_model(vllm_config=vllm_config)
336
337 - model.load_weights(self._get_all_weights(model_config, model))
337 + weights_to_load = {name for name, _ in model.named_parameters()}
338 + loaded_weights = model.load_weights(
339 + self._get_all_weights(model_config, model))
340 + # We only enable strict check for non-quantiized models
341 + # that have loaded weights tracking currently.
342 + if model_config.quantization is None and loaded_weights is not None:
343 + weights_not_loaded = weights_to_load - loaded_weights
344 + if weights_not_loaded:
345 + raise ValueError(
346 + "Following weights were not initialized from "
347 + f"checkpoint: {weights_not_loaded}")
348
349 for _, module in model.named_modules():
350 quant_method = getattr(module, "quant_method", None)
vllm/model_executor/models/arctic.py
+6 -2
@@ -1,5 +1,5 @@
1 """Inference-only Snowflake Arctic model."""
2 -from typing import Iterable, List, Optional, Tuple, Union
2 +from typing import Iterable, List, Optional, Set, Tuple, Union
3
4 import torch
5 from torch import nn
@@ -480,7 +480,8 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
480 next_tokens = self.sampler(logits, sampling_metadata)
481 return next_tokens
482
483 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
483 + def load_weights(self, weights: Iterable[Tuple[str,
484 + torch.Tensor]]) -> Set[str]:
485 stacked_params_mapping = [
486 # (param_name, shard_name, shard_id)
487 ("qkv_proj", "q_proj", "q"),
@@ -518,6 +519,7 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
519 ("ws", f"experts.{expert_id}.w3.weight", expert_id))
520
521 params_dict = dict(self.named_parameters())
522 + loaded_params: Set[str] = set()
523
524 logger.info(
525 "It will take ~10 minutes loading from the 16-bit weights. "
@@ -573,3 +575,5 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
575 weight_loader = getattr(param, "weight_loader",
576 default_weight_loader)
577 weight_loader(param, loaded_weight)
578 + loaded_params.add(name)
579 + return loaded_params
vllm/model_executor/models/baichuan.py
+6 -2
@@ -18,7 +18,7 @@
18 # limitations under the License.
19 """Inference-only BaiChuan model compatible with HuggingFace weights."""
20 import math
21 -from typing import Iterable, List, Optional, Tuple, Union
21 +from typing import Iterable, List, Optional, Set, Tuple, Union
22
23 import torch
24 from torch import nn
@@ -404,13 +404,15 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
404 next_tokens = self.sampler(logits, sampling_metadata)
405 return next_tokens
406
407 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
407 + def load_weights(self, weights: Iterable[Tuple[str,
408 + torch.Tensor]]) -> Set[str]:
409 stacked_params_mapping = [
410 # (param_name, shard_name, shard_id)
411 ("gate_up_proj", "gate_proj", 0),
412 ("gate_up_proj", "up_proj", 1),
413 ]
414 params_dict = dict(self.named_parameters())
415 + loaded_params: Set[str] = set()
416 for name, loaded_weight in weights:
417 if "rotary_emb.inv_freq" in name:
418 continue
@@ -449,6 +451,8 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
451 weight_loader = getattr(param, "weight_loader",
452 default_weight_loader)
453 weight_loader(param, loaded_weight)
454 + loaded_params.add(name)
455 + return loaded_params
456
457
458 class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
vllm/model_executor/models/bert.py
+6 -2
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Optional, Tuple
1 +from typing import Iterable, List, Optional, Set, Tuple
2
3 import torch
4 from torch import nn
@@ -337,7 +337,8 @@ class BertModel(nn.Module):
337
338 return self.encoder(hidden_states, kv_caches, attn_metadata)
339
340 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
340 + def load_weights(self, weights: Iterable[Tuple[str,
341 + torch.Tensor]]) -> Set[str]:
342 stacked_params_mapping = [
343 # (param_name, shard_name, shard_id)
344 ("qkv_proj", "query", "q"),
@@ -346,6 +347,7 @@ class BertModel(nn.Module):
347 ]
348
349 params_dict = dict(self.named_parameters())
350 + loaded_params: Set[str] = set()
351 for name, loaded_weight in weights:
352 if "pooler" in name:
353 continue
@@ -368,6 +370,8 @@ class BertModel(nn.Module):
370 weight_loader = getattr(param, "weight_loader",
371 default_weight_loader)
372 weight_loader(param, loaded_weight)
373 + loaded_params.add(name)
374 + return loaded_params
375
376
377 class BertEmbeddingModel(nn.Module):
vllm/model_executor/models/blip.py
+8 -4
@@ -1,6 +1,6 @@
1 """Minimal implementation of BlipVisionModel intended to be only used
2 within a vision language model."""
3 -from typing import Iterable, Optional, Tuple, Union
3 +from typing import Iterable, Optional, Set, Tuple, Union
4
5 import torch
6 import torch.nn as nn
@@ -415,7 +415,8 @@ class BlipVisionModel(nn.Module):
415
416 return self.post_layernorm(hidden_states)
417
418 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
418 + def load_weights(self, weights: Iterable[Tuple[str,
419 + torch.Tensor]]) -> Set[str]:
420 stacked_params_mapping = [
421 # (param_name, shard_name, shard_id)
422 ("qkv_proj", "q_proj", "q"),
@@ -423,6 +424,7 @@ class BlipVisionModel(nn.Module):
424 ("qkv_proj", "v_proj", "v"),
425 ] if self.shard_weight else []
426 params_dict = dict(self.named_parameters())
427 + loaded_params: Set[str] = set()
428 layer_count = len(self.encoder.layers)
429
430 for name, loaded_weight in weights:
@@ -440,8 +442,8 @@ class BlipVisionModel(nn.Module):
442 for (param_name, weight_name, shard_id) in stacked_params_mapping:
443 if weight_name not in name:
444 continue
443 -
444 - param = params_dict[name.replace(weight_name, param_name)]
445 + name = name.replace(weight_name, param_name)
446 + param = params_dict[name]
447 weight_loader = param.weight_loader
448 weight_loader(param, loaded_weight, shard_id)
449 break
@@ -450,3 +452,5 @@ class BlipVisionModel(nn.Module):
452 weight_loader = getattr(param, "weight_loader",
453 default_weight_loader)
454 weight_loader(param, loaded_weight)
455 + loaded_params.add(name)
456 + return loaded_params
vllm/model_executor/models/blip2.py
+4 -3
@@ -1,5 +1,5 @@
1 from functools import cached_property
2 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3 TypedDict, Union)
4
5 import torch
@@ -692,6 +692,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
692 ) -> Optional[SamplerOutput]:
693 return self.language_model.sample(logits, sampling_metadata)
694
695 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
695 + def load_weights(self, weights: Iterable[Tuple[str,
696 + torch.Tensor]]) -> Set[str]:
697 loader = AutoWeightsLoader(self)
697 - loader.load_weights(weights)
698 + return loader.load_weights(weights)
vllm/model_executor/models/bloom.py
+6 -2
@@ -16,7 +16,7 @@
16 # limitations under the License.
17 """Inference-only BLOOM model compatible with HuggingFace weights."""
18 import math
19 -from typing import Iterable, List, Optional, Tuple, Union
19 +from typing import Iterable, List, Optional, Set, Tuple, Union
20
21 import torch
22 from torch import nn
@@ -341,8 +341,10 @@ class BloomForCausalLM(nn.Module, SupportsPP):
341 next_tokens = self.sampler(logits, sampling_metadata)
342 return next_tokens
343
344 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
344 + def load_weights(self, weights: Iterable[Tuple[str,
345 + torch.Tensor]]) -> Set[str]:
346 params_dict = dict(self.named_parameters(remove_duplicate=False))
347 + loaded_params: Set[str] = set()
348 for name, loaded_weight in weights:
349 if name == "lm_head.weight":
350 continue
@@ -371,3 +373,5 @@ class BloomForCausalLM(nn.Module, SupportsPP):
373 weight_loader = getattr(param, "weight_loader",
374 default_weight_loader)
375 weight_loader(param, loaded_weight)
376 + loaded_params.add(name)
377 + return loaded_params
vllm/model_executor/models/chameleon.py
+6 -2
@@ -1,5 +1,5 @@
1 from functools import cached_property
2 -from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
2 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
3 Tuple, TypedDict, Union)
4
5 import torch
@@ -1034,7 +1034,8 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1034 next_tokens = self.sampler(logits, sampling_metadata)
1035 return next_tokens
1036
1037 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1037 + def load_weights(self, weights: Iterable[Tuple[str,
1038 + torch.Tensor]]) -> Set[str]:
1039 stacked_params_mapping = [
1040 # (param_name, shard_name, shard_id)
1041 (".qkv_proj", ".q_proj", "q"),
@@ -1044,6 +1045,7 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1045 (".gate_up_proj", ".up_proj", 1),
1046 ]
1047 params_dict = dict(self.named_parameters())
1048 + loaded_params: Set[str] = set()
1049 for name, loaded_weight in weights:
1050 if "rotary_emb.inv_freq" in name:
1051 continue
@@ -1111,3 +1113,5 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1113 weight_loader = getattr(param, "weight_loader",
1114 default_weight_loader)
1115 weight_loader(param, loaded_weight)
1116 + loaded_params.add(name)
1117 + return loaded_params
vllm/model_executor/models/chatglm.py
+8 -2
@@ -3,7 +3,8 @@
3 """Inference-only ChatGLM model compatible with THUDM weights."""
4 from argparse import Namespace
5 from array import array
6 -from typing import Dict, Iterable, List, Mapping, Optional, Tuple, TypedDict
6 +from typing import (Dict, Iterable, List, Mapping, Optional, Set, Tuple,
7 + TypedDict)
8
9 import torch
10 from PIL import Image
@@ -645,7 +646,8 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
646 next_tokens = self.sampler(logits, sampling_metadata)
647 return next_tokens
648
648 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
649 + def load_weights(self, weights: Iterable[Tuple[str,
650 + torch.Tensor]]) -> Set[str]:
651 # Merge two ColumnParallelLinear into one MergedColumnParallelLinear
652 merged_weights_dict: Dict[str, Dict[str, Optional[torch.Tensor]]] = {
653 "transformer.vision.linear_proj.merged_proj.weight": {
@@ -655,6 +657,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
657 }
658
659 params_dict = dict(self.named_parameters(remove_duplicate=False))
660 + loaded_params: Set[str] = set()
661 for name, loaded_weight in weights:
662 is_weight_to_be_merge = False
663 for _, merged_weight_dict in merged_weights_dict.items():
@@ -677,6 +680,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
680 weight_loader = getattr(param, "weight_loader",
681 default_weight_loader)
682 weight_loader(param, loaded_weight)
683 + loaded_params.add(name)
684
685 for combined_name, merged_weight_dict in merged_weights_dict.items():
686 if combined_name in params_dict:
@@ -686,3 +690,5 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
690 weight_loader = getattr(param, "weight_loader",
691 default_weight_loader)
692 weight_loader(param, combined_weight)
693 + loaded_params.add(combined_name)
694 + return loaded_params
vllm/model_executor/models/clip.py
+8 -3
@@ -1,6 +1,6 @@
1 """Minimal implementation of CLIPVisionModel intended to be only used
2 within a vision language model."""
3 -from typing import Iterable, List, Optional, Tuple, Union
3 +from typing import Iterable, List, Optional, Set, Tuple, Union
4
5 import numpy as np
6 import torch
@@ -483,7 +483,8 @@ class CLIPVisionModel(nn.Module):
483
484 # (TODO) Add prefix argument for filtering out weights to be loaded
485 # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
486 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
486 + def load_weights(self, weights: Iterable[Tuple[str,
487 + torch.Tensor]]) -> Set[str]:
488 stacked_params_mapping = [
489 # (param_name, shard_name, shard_id)
490 ("qkv_proj", "q_proj", "q"),
@@ -491,6 +492,7 @@ class CLIPVisionModel(nn.Module):
492 ("qkv_proj", "v_proj", "v"),
493 ] if self.shard_weight else []
494 params_dict = dict(self.named_parameters())
495 + loaded_params: Set[str] = set()
496 layer_count = len(self.vision_model.encoder.layers)
497
498 for name, loaded_weight in weights:
@@ -508,8 +510,9 @@ class CLIPVisionModel(nn.Module):
510 for (param_name, weight_name, shard_id) in stacked_params_mapping:
511 if weight_name not in name:
512 continue
513 + name = name.replace(weight_name, param_name)
514
512 - param = params_dict[name.replace(weight_name, param_name)]
515 + param = params_dict[name]
516 weight_loader = param.weight_loader
517 weight_loader(param, loaded_weight, shard_id)
518 break
@@ -518,3 +521,5 @@ class CLIPVisionModel(nn.Module):
521 weight_loader = getattr(param, "weight_loader",
522 default_weight_loader)
523 weight_loader(param, loaded_weight)
524 + loaded_params.add(name)
525 + return loaded_params
vllm/model_executor/models/commandr.py
+3 -1
@@ -402,7 +402,8 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
402 next_tokens = self.sampler(logits, sampling_metadata)
403 return next_tokens
404
405 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
405 + def load_weights(self, weights: Iterable[Tuple[str,
406 + torch.Tensor]]) -> Set[str]:
407 stacked_params_mapping = [
408 # (param_name, shard_name, shard_id)
409 ("qkv_proj", "q_proj", "q"),
@@ -447,3 +448,4 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
448 default_weight_loader)
449 weight_loader(param, loaded_weight)
450 loaded_params.add(name)
451 + return loaded_params
vllm/model_executor/models/dbrx.py
+6 -2
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Optional, Tuple, Union
1 +from typing import Iterable, List, Optional, Set, Tuple, Union
2
3 import torch
4 import torch.nn as nn
@@ -417,13 +417,15 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
417 next_tokens = self.sampler(logits, sampling_metadata)
418 return next_tokens
419
420 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
420 + def load_weights(self, weights: Iterable[Tuple[str,
421 + torch.Tensor]]) -> Set[str]:
422
423 expert_params_mapping = [(
424 "w13_weight" if weight_name in ["w1", "v1"] else "w2_weight",
425 f"mlp.{weight_name}",
426 ) for weight_name in ["w1", "v1", "w2"]]
427 params_dict = dict(self.named_parameters(remove_duplicate=False))
428 + loaded_params: Set[str] = set()
429 for name, loaded_weight in weights:
430 for param_name, weight_name in expert_params_mapping:
431 if weight_name not in name:
@@ -447,3 +449,5 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
449 weight_loader = getattr(param, "weight_loader",
450 default_weight_loader)
451 weight_loader(param, loaded_weight)
452 + loaded_params.add(name)
453 + return loaded_params
vllm/model_executor/models/decilm.py
+6 -2
@@ -22,7 +22,7 @@
22 # limitations under the License.
23 """Inference-only DeciLM model compatible with HuggingFace weights."""
24
25 -from typing import Iterable, Tuple
25 +from typing import Iterable, Set, Tuple
26
27 import torch
28
@@ -57,7 +57,8 @@ class DeciLMForCausalLM(LlamaForCausalLM):
57 delattr(config, "num_key_value_heads_per_layer")
58 super().__init__(vllm_config=vllm_config)
59
60 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
60 + def load_weights(self, weights: Iterable[Tuple[str,
61 + torch.Tensor]]) -> Set[str]:
62 stacked_params_mapping = [
63 # (param_name, shard_name, shard_id)
64 ("qkv_proj", "q_proj", "q"),
@@ -67,6 +68,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
68 ("gate_up_proj", "up_proj", 1),
69 ]
70 params_dict = dict(self.named_parameters())
71 + loaded_params: Set[str] = set()
72 for name, loaded_weight in weights:
73 if "rotary_emb.inv_freq" in name:
74 continue
@@ -97,6 +99,8 @@ class DeciLMForCausalLM(LlamaForCausalLM):
99 weight_loader = getattr(param, "weight_loader",
100 default_weight_loader)
101 weight_loader(param, loaded_weight)
102 + loaded_params.add(name)
103 + return loaded_params
104
105 def _degroup_weight(self, loaded_weight: torch.Tensor) -> torch.Tensor:
106 hidden_size = self.config.hidden_size
vllm/model_executor/models/deepseek.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only Deepseek model."""
23 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -442,7 +442,8 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
442 next_tokens = self.sampler(logits, sampling_metadata)
443 return next_tokens
444
445 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
445 + def load_weights(self, weights: Iterable[Tuple[str,
446 + torch.Tensor]]) -> Set[str]:
447 stacked_params_mapping = [
448 # (param_name, shard_name, shard_id)
449 ("qkv_proj", "q_proj", "q"),
@@ -453,6 +454,7 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
454 ]
455
456 params_dict = dict(self.named_parameters())
457 + loaded_params: Set[str] = set()
458 for name, loaded_weight in weights:
459 if "rotary_emb.inv_freq" in name:
460 continue
@@ -487,3 +489,5 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
489 weight_loader = getattr(param, "weight_loader",
490 default_weight_loader)
491 weight_loader(param, loaded_weight)
492 + loaded_params.add(name)
493 + return loaded_params
vllm/model_executor/models/deepseek_v2.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only DeepseekV2 model."""
23 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -550,7 +550,8 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
550 device=device),
551 })
552
553 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
553 + def load_weights(self, weights: Iterable[Tuple[str,
554 + torch.Tensor]]) -> Set[str]:
555 stacked_params_mapping = [
556 # (param_name, shard_name, shard_id)
557 ("gate_up_proj", "gate_proj", 0),
@@ -566,6 +567,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
567 num_experts=self.config.n_routed_experts)
568
569 params_dict = dict(self.named_parameters())
570 + loaded_params: Set[str] = set()
571 for name, loaded_weight in weights:
572 if "rotary_emb.inv_freq" in name:
573 continue
@@ -623,3 +625,5 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
625 weight_loader = getattr(param, "weight_loader",
626 default_weight_loader)
627 weight_loader(param, loaded_weight)
628 + loaded_params.add(name)
629 + return loaded_params
vllm/model_executor/models/exaone.py
+7 -2
@@ -22,7 +22,7 @@
22 # limitations under the License.
23 """Inference-only Exaone model compatible with HuggingFace weights."""
24
25 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
26
27 import torch
28 from torch import nn
@@ -513,7 +513,8 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
513 next_tokens = self.sampler(logits, sampling_metadata)
514 return next_tokens
515
516 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
516 + def load_weights(self, weights: Iterable[Tuple[str,
517 + torch.Tensor]]) -> Set[str]:
518 stacked_params_mapping = [
519 # (param_name, shard_name, shard_id)
520 (".qkv_proj", ".q_proj", "q"),
@@ -523,6 +524,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
524 (".gate_up_proj", ".c_fc_1", 1),
525 ]
526 params_dict = dict(self.named_parameters())
527 + loaded_params: Set[str] = set()
528 for name, loaded_weight in weights:
529 if "rotary_emb.inv_freq" in name:
530 continue
@@ -543,6 +545,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
545 default_weight_loader)
546 loaded_weight = loaded_weight[0]
547 weight_loader(param, loaded_weight)
548 + loaded_params.add(scale_name)
549 continue
550 for param_name, weight_name, shard_id in stacked_params_mapping:
551 if weight_name not in name:
@@ -576,6 +579,8 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
579 weight_loader = getattr(param, "weight_loader",
580 default_weight_loader)
581 weight_loader(param, loaded_weight)
582 + loaded_params.add(name)
583 + return loaded_params
584
585 # If this function is called, it should always initialize KV cache scale
586 # factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/falcon.py
+6 -2
@@ -18,7 +18,7 @@
18 """PyTorch Falcon model."""
19
20 import math
21 -from typing import Iterable, List, Optional, Tuple, Union
21 +from typing import Iterable, List, Optional, Set, Tuple, Union
22
23 import torch
24 from torch import nn
@@ -473,7 +473,8 @@ class FalconForCausalLM(nn.Module, SupportsPP):
473 next_tokens = self.sampler(logits, sampling_metadata)
474 return next_tokens
475
476 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
476 + def load_weights(self, weights: Iterable[Tuple[str,
477 + torch.Tensor]]) -> Set[str]:
478 total_num_heads = self.config.num_attention_heads
479 if self.config.new_decoder_architecture:
480 total_num_kv_heads = self.config.num_kv_heads
@@ -483,6 +484,7 @@ class FalconForCausalLM(nn.Module, SupportsPP):
484 total_num_kv_heads = total_num_heads
485 num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
486 params_dict = dict(self.named_parameters(remove_duplicate=False))
487 + loaded_params: Set[str] = set()
488 for name, loaded_weight in weights:
489 if name == "lm_head.weight" and self.tie_word_embeddings:
490 # Falcon uses tied embeddings except Falcon-11b.
@@ -519,3 +521,5 @@ class FalconForCausalLM(nn.Module, SupportsPP):
521 weight_loader = getattr(param, "weight_loader",
522 default_weight_loader)
523 weight_loader(param, loaded_weight)
524 + loaded_params.add(name)
525 + return loaded_params
vllm/model_executor/models/florence2.py
+11 -6
@@ -1,5 +1,5 @@
1 import math
2 -from typing import Iterable, List, Optional, Tuple
2 +from typing import Iterable, List, Optional, Set, Tuple
3
4 import torch
5 import torch.nn as nn
@@ -156,7 +156,8 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
156 next_tokens = self.sampler(logits, sampling_metadata)
157 return next_tokens
158
159 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
159 + def load_weights(self, weights: Iterable[Tuple[str,
160 + torch.Tensor]]) -> Set[str]:
161 stacked_params_mapping = [
162 # (param_name, shard_name, shard_id)
163 ("qkv_proj", "q_proj", "q"),
@@ -165,12 +166,13 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
166 ]
167
168 params_dict = dict(self.named_parameters())
169 + loaded_params: Set[str] = set()
170 for name, loaded_weight in weights:
171 for (param_name, weight_name, shard_id) in stacked_params_mapping:
172 if weight_name not in name:
173 continue
172 -
173 - param = params_dict[name.replace(weight_name, param_name)]
174 + name = name.replace(weight_name, param_name)
175 + param = params_dict[name]
176 weight_loader = param.weight_loader
177 weight_loader(param, loaded_weight, shard_id)
178 break
@@ -183,6 +185,8 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
185 weight_loader = getattr(param, "weight_loader",
186 default_weight_loader)
187 weight_loader(param, loaded_weight)
188 + loaded_params.add(name)
189 + return loaded_params
190
191
192 class Florence2ForConditionalGeneration(nn.Module):
@@ -248,10 +252,11 @@ class Florence2ForConditionalGeneration(nn.Module):
252 ) -> SamplerOutput:
253 return self.language_model.sample(logits, sampling_metadata)
254
251 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
255 + def load_weights(self, weights: Iterable[Tuple[str,
256 + torch.Tensor]]) -> Set[str]:
257 skip_prefixes = [
258 'image_projection', "vision_tower", "image_proj_norm",
259 "image_pos_embed", "visual_temporal_embed"
260 ]
261 loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
257 - loader.load_weights(weights)
262 + return loader.load_weights(weights)
vllm/model_executor/models/fuyu.py
+5 -3
@@ -16,7 +16,8 @@
16 """ PyTorch Fuyu model."""
17 import math
18 from array import array
19 -from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict
19 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
20 + TypedDict)
21
22 import torch
23 import torch.nn as nn
@@ -354,6 +355,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
355 next_tokens = self.language_model.sampler(logits, sampling_metadata)
356 return next_tokens
357
357 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
358 + def load_weights(self, weights: Iterable[Tuple[str,
359 + torch.Tensor]]) -> Set[str]:
360 loader = AutoWeightsLoader(self)
359 - loader.load_weights(weights)
361 + return loader.load_weights(weights)
vllm/model_executor/models/gemma.py
+3 -1
@@ -424,7 +424,8 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
424 next_tokens = self.sampler(logits, sampling_metadata)
425 return next_tokens
426
427 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
427 + def load_weights(self, weights: Iterable[Tuple[str,
428 + torch.Tensor]]) -> Set[str]:
429 stacked_params_mapping = [
430 # (param_name, shard_name, shard_id)
431 ("qkv_proj", "q_proj", "q"),
@@ -469,3 +470,4 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
470 logger.warning(
471 "Some weights are not initialized from checkpoints: %s",
472 unloaded_params)
473 + return loaded_params
vllm/model_executor/models/gemma2.py
+6 -3
@@ -312,7 +312,8 @@ class Gemma2Model(nn.Module):
312 hidden_states, _ = self.norm(hidden_states, residual)
313 return hidden_states
314
315 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
315 + def load_weights(self, weights: Iterable[Tuple[str,
316 + torch.Tensor]]) -> Set[str]:
317 stacked_params_mapping = [
318 # (param_name, shard_name, shard_id)
319 ("qkv_proj", "q_proj", "q"),
@@ -354,6 +355,7 @@ class Gemma2Model(nn.Module):
355 logger.warning(
356 "Some weights are not initialized from checkpoints: %s",
357 unloaded_params)
358 + return loaded_params
359
360
361 class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
@@ -451,13 +453,14 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
453 next_tokens = self.sampler(logits, sampling_metadata)
454 return next_tokens
455
454 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
456 + def load_weights(self, weights: Iterable[Tuple[str,
457 + torch.Tensor]]) -> Set[str]:
458 loader = AutoWeightsLoader(
459 self,
460 skip_prefixes=(["lm_head."]
461 if self.config.tie_word_embeddings else None),
462 )
460 - loader.load_weights(weights)
463 + return loader.load_weights(weights)
464
465
466 class Gemma2EmbeddingModel(nn.Module, SupportsPP):
vllm/model_executor/models/gpt2.py
+6 -2
@@ -16,7 +16,7 @@
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """Inference-only GPT-2 model compatible with HuggingFace weights."""
19 -from typing import Iterable, List, Optional, Tuple, Union
19 +from typing import Iterable, List, Optional, Set, Tuple, Union
20
21 import torch
22 from torch import nn
@@ -298,8 +298,10 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
298 next_tokens = self.sampler(logits, sampling_metadata)
299 return next_tokens
300
301 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
301 + def load_weights(self, weights: Iterable[Tuple[str,
302 + torch.Tensor]]) -> Set[str]:
303 params_dict = dict(self.named_parameters(remove_duplicate=False))
304 + loaded_params: Set[str] = set()
305 for name, loaded_weight in weights:
306 if "lm_head.weight" in name:
307 # GPT-2 ties the weights of the embedding layer and the final
@@ -328,3 +330,5 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
330 weight_loader = getattr(param, "weight_loader",
331 default_weight_loader)
332 weight_loader(param, loaded_weight)
333 + loaded_params.add(name)
334 + return loaded_params
vllm/model_executor/models/gpt_bigcode.py
+6 -2
@@ -17,7 +17,7 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """Inference-only GPTBigCode model compatible with HuggingFace weights."""
20 -from typing import Iterable, List, Optional, Tuple, Union
20 +from typing import Iterable, List, Optional, Set, Tuple, Union
21
22 import torch
23 from torch import nn
@@ -323,8 +323,10 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
323 next_tokens = self.sampler(logits, sampling_metadata)
324 return next_tokens
325
326 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
326 + def load_weights(self, weights: Iterable[Tuple[str,
327 + torch.Tensor]]) -> Set[str]:
328 params_dict = dict(self.named_parameters(remove_duplicate=False))
329 + loaded_params: Set[str] = set()
330 for name, loaded_weight in weights:
331 if "lm_head.weight" in name:
332 continue
@@ -344,3 +346,5 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
346 weight_loader(param, loaded_weight, 'v')
347 else:
348 weight_loader(param, loaded_weight)
349 + loaded_params.add(name)
350 + return loaded_params
vllm/model_executor/models/gpt_j.py
+6 -2
@@ -15,7 +15,7 @@
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 """Inference-only GPT-J model compatible with HuggingFace weights."""
18 -from typing import Iterable, List, Optional, Tuple, Union
18 +from typing import Iterable, List, Optional, Set, Tuple, Union
19
20 import torch
21 from torch import nn
@@ -291,7 +291,8 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
291 next_tokens = self.sampler(logits, sampling_metadata)
292 return next_tokens
293
294 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
294 + def load_weights(self, weights: Iterable[Tuple[str,
295 + torch.Tensor]]) -> Set[str]:
296 stacked_params_mapping = [
297 # (param_name, shard_name, shard_id)
298 ("qkv_proj", "q_proj", "q"),
@@ -301,6 +302,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
302 ("gate_up_proj", "up_proj", 1),
303 ]
304 params_dict = dict(self.named_parameters())
305 + loaded_params: Set[str] = set()
306 for name, loaded_weight in weights:
307 if "attn.bias" in name or "attn.masked_bias" in name:
308 continue
@@ -330,3 +332,5 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
332 weight_loader = getattr(param, "weight_loader",
333 default_weight_loader)
334 weight_loader(param, loaded_weight)
335 + loaded_params.add(name)
336 + return loaded_params
vllm/model_executor/models/gpt_neox.py
+6 -2
@@ -15,7 +15,7 @@
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 """Inference-only GPT-NeoX model compatible with HuggingFace weights."""
18 -from typing import Iterable, List, Optional, Tuple, Union
18 +from typing import Iterable, List, Optional, Set, Tuple, Union
19
20 import torch
21 from torch import nn
@@ -303,8 +303,10 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
303 next_tokens = self.sampler(logits, sampling_metadata)
304 return next_tokens
305
306 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
306 + def load_weights(self, weights: Iterable[Tuple[str,
307 + torch.Tensor]]) -> Set[str]:
308 params_dict = dict(self.named_parameters())
309 + loaded_params: Set[str] = set()
310 for name, loaded_weight in weights:
311 if ("attention.bias" in name or "attention.masked_bias" in name
312 or "rotary_emb.inv_freq" in name):
@@ -337,3 +339,5 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
339 weight_loader = getattr(param, "weight_loader",
340 default_weight_loader)
341 weight_loader(param, loaded_weight)
342 + loaded_params.add(name)
343 + return loaded_params
vllm/model_executor/models/granite.py
+7 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only IBM Granite model compatible with HuggingFace weights."""
23 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -455,7 +455,8 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
455 device=device),
456 })
457
458 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
458 + def load_weights(self, weights: Iterable[Tuple[str,
459 + torch.Tensor]]) -> Set[str]:
460 stacked_params_mapping = [
461 # (param_name, shard_name, shard_id)
462 (".qkv_proj", ".q_proj", "q"),
@@ -465,6 +466,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
466 (".gate_up_proj", ".up_proj", 1),
467 ]
468 params_dict = dict(self.named_parameters())
469 + loaded_params: Set[str] = set()
470 for name, loaded_weight in weights:
471 if "rotary_emb.inv_freq" in name:
472 continue
@@ -485,6 +487,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
487 default_weight_loader)
488 loaded_weight = loaded_weight[0]
489 weight_loader(param, loaded_weight)
490 + loaded_params.add(scale_name)
491 continue
492 for (param_name, weight_name, shard_id) in stacked_params_mapping:
493 if weight_name not in name:
@@ -518,6 +521,8 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
521 weight_loader = getattr(param, "weight_loader",
522 default_weight_loader)
523 weight_loader(param, loaded_weight)
524 + loaded_params.add(name)
525 + return loaded_params
526
527 # If this function is called, it should always initialize KV cache scale
528 # factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/granitemoe.py
+5 -3
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only GraniteMoe model."""
23 -from typing import Iterable, List, Optional, Tuple
23 +from typing import Iterable, List, Optional, Set, Tuple
24
25 import torch
26 from torch import nn
@@ -419,7 +419,8 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419 next_tokens = self.sampler(logits, sampling_metadata)
420 return next_tokens
421
422 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
422 + def load_weights(self, weights: Iterable[Tuple[str,
423 + torch.Tensor]]) -> Set[str]:
424 new_weights = {}
425 for n, p in weights:
426 if n.endswith('.block_sparse_moe.input_linear.weight'):
@@ -452,4 +453,5 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
453 pass
454 else:
455 new_weights[n] = p
455 - mixtral.MixtralForCausalLM.load_weights(self, new_weights.items())
456 + return mixtral.MixtralForCausalLM.load_weights(self,
457 + new_weights.items())
vllm/model_executor/models/idefics2_vision_model.py
+8 -3
@@ -15,7 +15,7 @@
15 # limitations under the License.
16 """PyTorch Idefics2 model."""
17
18 -from typing import Iterable, Optional, Tuple
18 +from typing import Iterable, Optional, Set, Tuple
19
20 import torch
21 from torch import nn
@@ -331,7 +331,8 @@ class Idefics2VisionTransformer(nn.Module):
331 last_hidden_state = self.post_layernorm(encoder_outputs)
332 return last_hidden_state
333
334 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
334 + def load_weights(self, weights: Iterable[Tuple[str,
335 + torch.Tensor]]) -> Set[str]:
336 stacked_params_mapping = [
337 # (param_name, shard_name, shard_id)
338 ("qkv_proj", "q_proj", "q"),
@@ -339,11 +340,13 @@ class Idefics2VisionTransformer(nn.Module):
340 ("qkv_proj", "v_proj", "v"),
341 ]
342 params_dict = dict(self.named_parameters())
343 + loaded_params: Set[str] = set()
344 for name, loaded_weight in weights:
345 for param_name, weight_name, shard_id in stacked_params_mapping:
346 if weight_name not in name:
347 continue
346 - param = params_dict[name.replace(weight_name, param_name)]
348 + name = name.replace(weight_name, param_name)
349 + param = params_dict[name]
350 weight_loader = param.weight_loader
351 weight_loader(param, loaded_weight, shard_id)
352 break
@@ -352,3 +355,5 @@ class Idefics2VisionTransformer(nn.Module):
355 weight_loader = getattr(param, "weight_loader",
356 default_weight_loader)
357 weight_loader(param, loaded_weight)
358 + loaded_params.add(name)
359 + return loaded_params
vllm/model_executor/models/idefics3.py
+4 -3
@@ -15,7 +15,7 @@
15
16 import math
17 from typing import (Dict, Iterable, List, Literal, Mapping, NamedTuple,
18 - Optional, Tuple, TypedDict, Union)
18 + Optional, Set, Tuple, TypedDict, Union)
19
20 import torch
21 import torch.utils.checkpoint
@@ -751,9 +751,10 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
751 next_tokens = self.sampler(logits, sampling_metadata)
752 return next_tokens
753
754 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
754 + def load_weights(self, weights: Iterable[Tuple[str,
755 + torch.Tensor]]) -> Set[str]:
756 loader = AutoWeightsLoader(self)
756 - loader.load_weights(weights)
757 + return loader.load_weights(weights)
758
759 def get_mm_mapping(self) -> MultiModelKeys:
760 """
vllm/model_executor/models/intern_vit.py
+6 -2
@@ -5,7 +5,7 @@
5 # Licensed under The MIT License [see LICENSE for details]
6 # --------------------------------------------------------
7 from functools import partial
8 -from typing import Iterable, Optional, Tuple
8 +from typing import Iterable, Optional, Set, Tuple
9
10 import torch
11 import torch.nn as nn
@@ -469,10 +469,14 @@ class InternVisionModel(nn.Module):
469
470 return encoder_outputs
471
472 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
472 + def load_weights(self, weights: Iterable[Tuple[str,
473 + torch.Tensor]]) -> Set[str]:
474 params_dict = dict(self.named_parameters())
475 + loaded_params: Set[str] = set()
476 for name, loaded_weight in weights:
477 param = params_dict[name]
478 weight_loader = getattr(param, "weight_loader",
479 default_weight_loader)
480 weight_loader(param, loaded_weight)
481 + loaded_params.add(name)
482 + return loaded_params
vllm/model_executor/models/internlm2.py
+6 -2
@@ -1,5 +1,5 @@
1 from functools import partial
2 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
2 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
3
4 import torch
5 from torch import nn
@@ -369,13 +369,15 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
369 next_tokens = self.sampler(logits, sampling_metadata)
370 return next_tokens
371
372 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
372 + def load_weights(self, weights: Iterable[Tuple[str,
373 + torch.Tensor]]) -> Set[str]:
374 stacked_params_mapping = [
375 # (param_name, shard_name, shard_id)
376 ("gate_up_proj", "w1", 0),
377 ("gate_up_proj", "w3", 1),
378 ]
379 params_dict = dict(self.named_parameters())
380 + loaded_params: Set[str] = set()
381 for name, loaded_weight in weights:
382 if "rotary_emb.inv_freq" in name:
383 continue
@@ -402,3 +404,5 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
404 weight_loader = getattr(param, "weight_loader",
405 default_weight_loader)
406 weight_loader(param, loaded_weight)
407 + loaded_params.add(name)
408 + return loaded_params
vllm/model_executor/models/internvl.py
+4 -3
@@ -6,7 +6,7 @@
6 # --------------------------------------------------------
7 import re
8 from functools import cached_property, partial
9 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
10 TypedDict, Union)
11
12 import torch
@@ -663,6 +663,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
663 ) -> Optional[SamplerOutput]:
664 return self.language_model.sample(logits, sampling_metadata)
665
666 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
666 + def load_weights(self, weights: Iterable[Tuple[str,
667 + torch.Tensor]]) -> Set[str]:
668 loader = AutoWeightsLoader(self)
668 - loader.load_weights(weights)
669 + return loader.load_weights(weights)
vllm/model_executor/models/jais.py
+6 -2
@@ -19,7 +19,7 @@
19 """Inference-only Jais model compatible with HuggingFace weights."""
20
21 import math
22 -from typing import Iterable, List, Optional, Tuple, Union
22 +from typing import Iterable, List, Optional, Set, Tuple, Union
23
24 import torch
25 from torch import nn
@@ -350,8 +350,10 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
350 next_tokens = self.sampler(logits, sampling_metadata)
351 return next_tokens
352
353 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
353 + def load_weights(self, weights: Iterable[Tuple[str,
354 + torch.Tensor]]) -> Set[str]:
355 params_dict = dict(self.named_parameters(remove_duplicate=False))
356 + loaded_params: Set[str] = set()
357 for name, loaded_weight in weights:
358 if "lm_head.weight" in name:
359 # GPT-2 ties the weights of the embedding layer and the final
@@ -382,3 +384,5 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
384 weight_loader = getattr(param, "weight_loader",
385 default_weight_loader)
386 weight_loader(param, loaded_weight)
387 + loaded_params.add(name)
388 + return loaded_params
vllm/model_executor/models/jamba.py
+6 -2
@@ -1,5 +1,5 @@
1 """Inference-only Jamba model."""
2 -from typing import Iterable, List, Optional, Tuple
2 +from typing import Iterable, List, Optional, Set, Tuple
3
4 import torch
5 from torch import nn
@@ -462,7 +462,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
462 next_tokens = self.sampler(logits, sampling_metadata)
463 return next_tokens
464
465 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
465 + def load_weights(self, weights: Iterable[Tuple[str,
466 + torch.Tensor]]) -> Set[str]:
467 stacked_params_mapping = [
468 # (param_name, shard_name, shard_id)
469 ("qkv_proj", "q_proj", "q"),
@@ -479,6 +480,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
480 num_experts=self.config.num_experts)
481
482 params_dict = dict(self.named_parameters())
483 + loaded_params: Set[str] = set()
484 for name, loaded_weight in weights:
485 if "rotary_emb.inv_freq" in name:
486 continue
@@ -534,6 +536,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
536 weight_loader = getattr(param, "weight_loader",
537 default_weight_loader)
538 weight_loader(param, loaded_weight)
539 + loaded_params.add(name)
540 + return loaded_params
541
542
543 def _is_moe_layer(name: str):
vllm/model_executor/models/llama.py
+10 -5
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only LLaMA model compatible with HuggingFace weights."""
23 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -350,7 +350,8 @@ class LlamaModel(nn.Module):
350 hidden_states, _ = self.norm(hidden_states, residual)
351 return hidden_states
352
353 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
353 + def load_weights(self, weights: Iterable[Tuple[str,
354 + torch.Tensor]]) -> Set[str]:
355 stacked_params_mapping = [
356 # (param_name, shard_name, shard_id)
357 (".qkv_proj", ".q_proj", "q"),
@@ -360,6 +361,7 @@ class LlamaModel(nn.Module):
361 (".gate_up_proj", ".up_proj", 1),
362 ]
363 params_dict = dict(self.named_parameters())
364 + loaded_params: Set[str] = set()
365 for name, loaded_weight in weights:
366 if "rotary_emb.inv_freq" in name:
367 continue
@@ -375,6 +377,7 @@ class LlamaModel(nn.Module):
377 default_weight_loader)
378 loaded_weight = loaded_weight[0]
379 weight_loader(param, loaded_weight)
380 + loaded_params.add(scale_name)
381 continue
382 for param_name, weight_name, shard_id in stacked_params_mapping:
383 if weight_name not in name:
@@ -390,7 +393,6 @@ class LlamaModel(nn.Module):
393 param = params_dict[name]
394 weight_loader = param.weight_loader
395 weight_loader(param, loaded_weight, shard_id)
393 -
396 break
397 else:
398 # Skip loading extra bias for GPTQ models.
@@ -408,6 +410,8 @@ class LlamaModel(nn.Module):
410 weight_loader = getattr(param, "weight_loader",
411 default_weight_loader)
412 weight_loader(param, loaded_weight)
413 + loaded_params.add(name)
414 + return loaded_params
415
416 # If this function is called, it should always initialize KV cache scale
417 # factors (or else raise an exception). Thus, handled exceptions should
@@ -577,13 +581,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
581 next_tokens = self.sampler(logits, sampling_metadata)
582 return next_tokens
583
580 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
584 + def load_weights(self, weights: Iterable[Tuple[str,
585 + torch.Tensor]]) -> Set[str]:
586 loader = AutoWeightsLoader(
587 self,
588 skip_prefixes=(["lm_head."]
589 if self.config.tie_word_embeddings else None),
590 )
586 - loader.load_weights(
591 + return loader.load_weights(
592 self.maybe_remap_mistral(name, loaded_weight)
593 for name, loaded_weight in weights)
594
vllm/model_executor/models/llava.py
+4 -3
@@ -1,5 +1,5 @@
1 from functools import cached_property
2 -from typing import (Iterable, List, Literal, Mapping, Optional, Protocol,
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
3 Tuple, TypedDict, Union)
4
5 import torch
@@ -547,6 +547,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
547 ) -> Optional[SamplerOutput]:
548 return self.language_model.sample(logits, sampling_metadata)
549
550 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
550 + def load_weights(self, weights: Iterable[Tuple[str,
551 + torch.Tensor]]) -> Set[str]:
552 loader = AutoWeightsLoader(self)
552 - loader.load_weights(weights)
553 + return loader.load_weights(weights)
vllm/model_executor/models/llava_next.py
+4 -3
@@ -1,5 +1,5 @@
1 from functools import cached_property
2 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3 TypedDict, Union)
4
5 import torch
@@ -654,6 +654,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
654 ) -> Optional[PoolerOutput]:
655 return self._pooler(hidden_states, pooling_metadata)
656
657 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
657 + def load_weights(self, weights: Iterable[Tuple[str,
658 + torch.Tensor]]) -> Set[str]:
659 loader = AutoWeightsLoader(self)
659 - loader.load_weights(weights)
660 + return loader.load_weights(weights)
vllm/model_executor/models/llava_next_video.py
+4 -3
@@ -1,6 +1,6 @@
1 import math
2 from functools import cached_property
3 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4 TypedDict, Union)
5
6 import numpy as np
@@ -445,10 +445,11 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
445 ) -> Optional[SamplerOutput]:
446 return self.language_model.sample(logits, sampling_metadata)
447
448 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
448 + def load_weights(self, weights: Iterable[Tuple[str,
449 + torch.Tensor]]) -> Set[str]:
450 loader = AutoWeightsLoader(
451 self,
452 # This model doesn't support images for now
453 ignore_unexpected_prefixes=["image_newline"],
454 )
454 - loader.load_weights(weights)
455 + return loader.load_weights(weights)
vllm/model_executor/models/llava_onevision.py
+4 -3
@@ -1,6 +1,6 @@
1 import math
2 from functools import cached_property
3 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4 TypedDict, Union)
5
6 import numpy as np
@@ -887,6 +887,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
887 ) -> Optional[SamplerOutput]:
888 return self.language_model.sample(logits, sampling_metadata)
889
890 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
890 + def load_weights(self, weights: Iterable[Tuple[str,
891 + torch.Tensor]]) -> Set[str]:
892 loader = AutoWeightsLoader(self)
892 - loader.load_weights(weights)
893 + return loader.load_weights(weights)
vllm/model_executor/models/mamba.py
+6 -2
@@ -1,5 +1,5 @@
1 """PyTorch MAMBA model."""
2 -from typing import Iterable, List, Optional, Tuple
2 +from typing import Iterable, List, Optional, Set, Tuple
3
4 import torch
5 from torch import nn
@@ -243,8 +243,10 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
243 next_tokens = self.sampler(logits, sampling_metadata)
244 return next_tokens
245
246 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
246 + def load_weights(self, weights: Iterable[Tuple[str,
247 + torch.Tensor]]) -> Set[str]:
248 params_dict = dict(self.named_parameters())
249 + loaded_params: Set[str] = set()
250 for name, loaded_weight in weights:
251 if "A_log" in name:
252 name = name.replace("A_log", "A")
@@ -256,3 +258,5 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
258 weight_loader = getattr(param, "weight_loader",
259 default_weight_loader)
260 weight_loader(param, loaded_weight)
261 + loaded_params.add(name)
262 + return loaded_params
vllm/model_executor/models/medusa.py
+7 -2
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Optional, Tuple
1 +from typing import Iterable, List, Optional, Set, Tuple
2
3 import torch
4 import torch.nn as nn
@@ -156,8 +156,10 @@ class Medusa(nn.Module):
156 sampling_metadata=sampling_metadata,
157 )
158
159 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
159 + def load_weights(self, weights: Iterable[Tuple[str,
160 + torch.Tensor]]) -> Set[str]:
161 params_dict = dict(self.named_parameters())
162 + loaded_params: Set[str] = set()
163
164 weights_map = {}
165
@@ -181,9 +183,12 @@ class Medusa(nn.Module):
183 weight_loader = getattr(param, "weight_loader",
184 default_weight_loader)
185 weight_loader(param, loaded_weight)
186 + loaded_params.add(name)
187
188 if self.token_map is not None:
189 self.token_map.to(device=self.lm_heads[0].weight.device)
190
191 assert (self.truncated_vocab_size
192 == self.orig_vocab_size) or (self.token_map is not None)
193 +
194 + return loaded_params
vllm/model_executor/models/minicpm.py
+6 -2
@@ -21,7 +21,7 @@
21 # limitations under the License.
22 """Inference-only MiniCPM model compatible with HuggingFace weights."""
23 import math
24 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
24 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
25
26 import torch
27 from torch import nn
@@ -539,7 +539,8 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
539 next_tokens = self.sampler(logits, sampling_metadata)
540 return next_tokens
541
542 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
542 + def load_weights(self, weights: Iterable[Tuple[str,
543 + torch.Tensor]]) -> Set[str]:
544 stacked_params_mapping = [
545 # (param_name, shard_name, shard_id)
546 ("qkv_proj", "q_proj", "q"),
@@ -556,6 +557,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
557 for weight_name in ["w1", "w2", "w3"]
558 ]
559 params_dict = dict(self.named_parameters())
560 + loaded_params: Set[str] = set()
561 for name, loaded_weight in weights:
562 if "rotary_emb.inv_freq" in name:
563 continue
@@ -606,3 +608,5 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
608 weight_loader = getattr(param, "weight_loader",
609 default_weight_loader)
610 weight_loader(param, loaded_weight)
611 + loaded_params.add(name)
612 + return loaded_params
vllm/model_executor/models/minicpmv.py
+9 -5
@@ -24,7 +24,7 @@ import math
24 import re
25 from functools import partial
26 from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27 - Tuple, TypedDict, Union)
27 + Set, Tuple, TypedDict, Union)
28
29 import torch
30 import torch.types
@@ -602,7 +602,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
602 next_tokens = self.sampler(logits, sampling_metadata)
603 return next_tokens
604
605 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
605 + def load_weights(self, weights: Iterable[Tuple[str,
606 + torch.Tensor]]) -> Set[str]:
607 stacked_params_mapping = [
608 # (param_name, shard_name, shard_id)
609 ("qkv_proj", "q_proj", "q"),
@@ -612,6 +613,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
613 ("gate_up_proj", "up_proj", 1),
614 ]
615 params_dict = dict(self.named_parameters())
616 + loaded_params: Set[str] = set()
617 for name, loaded_weight in weights:
618 for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
619 if key_to_modify in name:
@@ -630,10 +632,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
632 for param_name, weight_name, shard_id in stacked_params_mapping:
633 if weight_name not in name:
634 continue
633 - if is_pp_missing_parameter(
634 - name.replace(weight_name, param_name), self):
635 + name = name.replace(weight_name, param_name)
636 + if is_pp_missing_parameter(name, self):
637 continue
636 - param = params_dict[name.replace(weight_name, param_name)]
638 + param = params_dict[name]
639 weight_loader = param.weight_loader
640 weight_loader(param, loaded_weight, shard_id)
641 break
@@ -646,6 +648,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
648 weight_loader = getattr(param, "weight_loader",
649 default_weight_loader)
650 weight_loader(param, loaded_weight)
651 + loaded_params.add(name)
652 + return loaded_params
653
654 def get_mm_mapping(self) -> MultiModelKeys:
655 """
vllm/model_executor/models/mixtral.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only Mixtral model."""
23 -from typing import Iterable, List, Optional, Tuple, Union
23 +from typing import Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -404,7 +404,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
404 next_tokens = self.sampler(logits, sampling_metadata)
405 return next_tokens
406
407 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
407 + def load_weights(self, weights: Iterable[Tuple[str,
408 + torch.Tensor]]) -> Set[str]:
409 stacked_params_mapping = [
410 # (param_name, shard_name, shard_id)
411 ("qkv_proj", "q_proj", "q"),
@@ -421,6 +422,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
422 num_experts=self.config.num_local_experts)
423
424 params_dict = dict(self.named_parameters())
425 + loaded_params: Set[str] = set()
426 for name, loaded_weight in weights:
427 if "rotary_emb.inv_freq" in name:
428 continue
@@ -478,3 +480,5 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
480 weight_loader = getattr(param, "weight_loader",
481 default_weight_loader)
482 weight_loader(param, loaded_weight)
483 + loaded_params.add(name)
484 + return loaded_params
vllm/model_executor/models/mixtral_quant.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only Mixtral model."""
23 -from typing import Iterable, List, Optional, Tuple, Union
23 +from typing import Iterable, List, Optional, Set, Tuple, Union
24
25 import numpy as np
26 import torch
@@ -409,7 +409,8 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
409 next_tokens = self.sampler(logits, sampling_metadata)
410 return next_tokens
411
412 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
412 + def load_weights(self, weights: Iterable[Tuple[str,
413 + torch.Tensor]]) -> Set[str]:
414 stacked_params_mapping = [
415 # (param_name, shard_name, shard_id)
416 ("qkv_proj", "q_proj", "q"),
@@ -418,6 +419,7 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
419 ]
420
421 params_dict = dict(self.named_parameters())
422 + loaded_params: Set[str] = set()
423 for name, loaded_weight in weights:
424 if "rotary_emb.inv_freq" in name:
425 continue
@@ -448,3 +450,5 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
450 weight_loader = getattr(param, "weight_loader",
451 default_weight_loader)
452 weight_loader(param, loaded_weight)
453 + loaded_params.add(name)
454 + return loaded_params
vllm/model_executor/models/mllama.py
+6 -3
@@ -13,7 +13,7 @@
13 # limitations under the License.
14 """PyTorch Mllama model."""
15 import math
16 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
16 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
17 TypedDict, Union)
18
19 import numpy as np
@@ -1427,7 +1427,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1427
1428 return outputs
1429
1430 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1430 + def load_weights(self, weights: Iterable[Tuple[str,
1431 + torch.Tensor]]) -> Set[str]:
1432 stacked_params_mapping = [
1433 # (param_name, shard_name, shard_id)
1434 (".qkv_proj", ".q_proj", "q"),
@@ -1437,7 +1438,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1438 (".gate_up_proj", ".up_proj", 1),
1439 ]
1440 params_dict = dict(self.named_parameters())
1440 - updated_params = set()
1441 + updated_params: Set[str] = set()
1442 for name, loaded_weight in weights:
1443 if 'patch_embedding.weight' in name:
1444 name = name.replace('patch_embedding.weight',
@@ -1457,6 +1458,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1458 weight_loader = getattr(param, "weight_loader",
1459 default_weight_loader)
1460 weight_loader(param, loaded_weight)
1461 + updated_params.add(name)
1462 + return updated_params
1463
1464
1465 def skip_attention_mask(sparse_mask: List[List[int]]) -> bool:
vllm/model_executor/models/mlp_speculator.py
+6 -2
@@ -1,5 +1,5 @@
1 import math
2 -from typing import Iterable, List, Tuple
2 +from typing import Iterable, List, Set, Tuple
3
4 import torch
5 import torch.nn as nn
@@ -188,11 +188,15 @@ class MLPSpeculator(nn.Module):
188
189 return next_tokens
190
191 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
191 + def load_weights(self, weights: Iterable[Tuple[str,
192 + torch.Tensor]]) -> Set[str]:
193 params_dict = dict(self.named_parameters())
194 + loaded_params: Set[str] = set()
195 for name, loaded_weight in weights:
196 param = params_dict.get(name.replace("speculator.", ""))
197 if param is not None:
198 weight_loader = getattr(param, "weight_loader",
199 default_weight_loader)
200 weight_loader(param, loaded_weight)
201 + loaded_params.add(name)
202 + return loaded_params
vllm/model_executor/models/mpt.py
+6 -2
@@ -1,6 +1,6 @@
1 # Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
2 import math
3 -from typing import Iterable, List, Optional, Tuple, Union
3 +from typing import Iterable, List, Optional, Set, Tuple, Union
4
5 import torch
6 import torch.nn as nn
@@ -324,8 +324,10 @@ class MPTForCausalLM(nn.Module, SupportsPP):
324 next_tokens = self.sampler(logits, sampling_metadata)
325 return next_tokens
326
327 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
327 + def load_weights(self, weights: Iterable[Tuple[str,
328 + torch.Tensor]]) -> Set[str]:
329 params_dict = dict(self.named_parameters(remove_duplicate=False))
330 + loaded_params: Set[str] = set()
331 for name, loaded_weight in weights:
332 # Skip loading extra bias for GPTQ models.
333 if name.endswith(".bias") and name not in params_dict:
@@ -336,3 +338,5 @@ class MPTForCausalLM(nn.Module, SupportsPP):
338 weight_loader = getattr(param, "weight_loader",
339 default_weight_loader)
340 weight_loader(param, loaded_weight)
341 + loaded_params.add(name)
342 + return loaded_params
vllm/model_executor/models/nemotron.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only Nemotron model compatible with HuggingFace weights."""
23 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -474,7 +474,8 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
474 next_tokens = self.sampler(logits, sampling_metadata)
475 return next_tokens
476
477 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
477 + def load_weights(self, weights: Iterable[Tuple[str,
478 + torch.Tensor]]) -> Set[str]:
479 stacked_params_mapping = [
480 # (param_name, shard_name, shard_id)
481 (".qkv_proj", ".q_proj", "q"),
@@ -482,6 +483,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
483 (".qkv_proj", ".v_proj", "v"),
484 ]
485 params_dict = dict(self.named_parameters())
486 + loaded_params: Set[str] = set()
487 for name, loaded_weight in weights:
488 if "rotary_emb.inv_freq" in name:
489 continue
@@ -522,3 +524,5 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
524 weight_loader = getattr(param, "weight_loader",
525 default_weight_loader)
526 weight_loader(param, loaded_weight)
527 + loaded_params.add(name)
528 + return loaded_params
vllm/model_executor/models/olmo.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only OLMo model compatible with HuggingFace weights."""
23 -from typing import Iterable, List, Optional, Tuple, Union
23 +from typing import Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -356,7 +356,8 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
356 next_tokens = self.sampler(logits, sampling_metadata)
357 return next_tokens
358
359 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
359 + def load_weights(self, weights: Iterable[Tuple[str,
360 + torch.Tensor]]) -> Set[str]:
361 stacked_params_mapping = [
362 # (param_name, shard_name, shard_id)
363 ("qkv_proj", "q_proj", "q"),
@@ -366,6 +367,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
367 ("gate_up_proj", "up_proj", 1),
368 ]
369 params_dict = dict(self.named_parameters(remove_duplicate=False))
370 + loaded_params: Set[str] = set()
371 for name, loaded_weight in weights:
372 if "rotary_emb.inv_freq" in name:
373 continue
@@ -402,3 +404,5 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
404 weight_loader = getattr(param, "weight_loader",
405 default_weight_loader)
406 weight_loader(param, loaded_weight)
407 + loaded_params.add(name)
408 + return loaded_params
vllm/model_executor/models/olmoe.py
+6 -2
@@ -10,7 +10,7 @@
10 # See the License for the specific language governing permissions and
11 # limitations under the License.
12 """Inference-only OLMoE model compatible with HuggingFace weights."""
13 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
13 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
14
15 import torch
16 from torch import nn
@@ -364,7 +364,8 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
364 next_tokens = self.sampler(logits, sampling_metadata)
365 return next_tokens
366
367 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
367 + def load_weights(self, weights: Iterable[Tuple[str,
368 + torch.Tensor]]) -> Set[str]:
369 stacked_params_mapping = [
370 # (param_name, shard_name, shard_id)
371 ("qkv_proj", "q_proj", "q"),
@@ -383,6 +384,7 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
384 num_experts=self.config.num_experts)
385
386 params_dict = dict(self.named_parameters())
387 + loaded_params: Set[str] = set()
388 for name, loaded_weight in weights:
389 if "rotary_emb.inv_freq" in name:
390 continue
@@ -455,3 +457,5 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
457 weight_loader = getattr(param, "weight_loader",
458 default_weight_loader)
459 weight_loader(param, loaded_weight)
460 + loaded_params.add(name)
461 + return loaded_params
vllm/model_executor/models/opt.py
+6 -2
@@ -16,7 +16,7 @@
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """Inference-only OPT model compatible with HuggingFace weights."""
19 -from typing import Iterable, List, Optional, Tuple, Union
19 +from typing import Iterable, List, Optional, Set, Tuple, Union
20
21 import torch
22 from torch import nn
@@ -394,7 +394,8 @@ class OPTForCausalLM(nn.Module, SupportsPP):
394 next_tokens = self.sampler(logits, sampling_metadata)
395 return next_tokens
396
397 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
397 + def load_weights(self, weights: Iterable[Tuple[str,
398 + torch.Tensor]]) -> Set[str]:
399 stacked_params_mapping = [
400 # (param_name, shard_name, shard_id)
401 ("qkv_proj", "q_proj", "q"),
@@ -402,6 +403,7 @@ class OPTForCausalLM(nn.Module, SupportsPP):
403 ("qkv_proj", "v_proj", "v"),
404 ]
405 params_dict = dict(self.named_parameters(remove_duplicate=False))
406 + loaded_params: Set[str] = set()
407 for name, loaded_weight in weights:
408 if "lm_head.weight" in name and self.config.tie_word_embeddings:
409 continue
@@ -431,3 +433,5 @@ class OPTForCausalLM(nn.Module, SupportsPP):
433 weight_loader = getattr(param, "weight_loader",
434 default_weight_loader)
435 weight_loader(param, loaded_weight)
436 + loaded_params.add(name)
437 + return loaded_params
vllm/model_executor/models/orion.py
+6 -2
@@ -3,7 +3,7 @@
3 # Copyright (c) OrionStar Inc.
4 # LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE
5 """Inference-only Orion-14B model compatible with HuggingFace weights."""
6 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
6 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
7
8 import torch
9 from torch import nn
@@ -327,7 +327,8 @@ class OrionForCausalLM(nn.Module, SupportsPP):
327 next_tokens = self.sampler(logits, sampling_metadata)
328 return next_tokens
329
330 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
330 + def load_weights(self, weights: Iterable[Tuple[str,
331 + torch.Tensor]]) -> Set[str]:
332 stacked_params_mapping = [
333 # (param_name, shard_name, shard_id)
334 ("qkv_proj", "q_proj", "q"),
@@ -337,6 +338,7 @@ class OrionForCausalLM(nn.Module, SupportsPP):
338 ("gate_up_proj", "up_proj", 1),
339 ]
340 params_dict = dict(self.named_parameters())
341 + loaded_params: Set[str] = set()
342 for name, loaded_weight in weights:
343 if "rotary_emb.inv_freq" in name:
344 continue
@@ -368,3 +370,5 @@ class OrionForCausalLM(nn.Module, SupportsPP):
370 weight_loader = getattr(param, "weight_loader",
371 default_weight_loader)
372 weight_loader(param, loaded_weight)
373 + loaded_params.add(name)
374 + return loaded_params
vllm/model_executor/models/paligemma.py
+4 -3
@@ -1,4 +1,4 @@
1 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
1 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
2 TypedDict, Union)
3
4 import torch
@@ -295,6 +295,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
295 ) -> Optional[SamplerOutput]:
296 return self.language_model.sample(logits, sampling_metadata)
297
298 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
298 + def load_weights(self, weights: Iterable[Tuple[str,
299 + torch.Tensor]]) -> Set[str]:
300 loader = AutoWeightsLoader(self)
300 - loader.load_weights(weights)
301 + return loader.load_weights(weights)
vllm/model_executor/models/persimmon.py
+6 -2
@@ -19,7 +19,7 @@
19 # See the License for the specific language governing permissions and
20 # limitations under the License.
21 """Inference-only persimmon model compatible with HuggingFace weights."""
22 -from typing import Iterable, List, Optional, Tuple, Union
22 +from typing import Iterable, List, Optional, Set, Tuple, Union
23
24 import torch
25 from torch import nn
@@ -324,8 +324,10 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
324 next_tokens = self.sampler(logits, sampling_metadata)
325 return next_tokens
326
327 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
327 + def load_weights(self, weights: Iterable[Tuple[str,
328 + torch.Tensor]]) -> Set[str]:
329 params_dict = dict(self.named_parameters(remove_duplicate=False))
330 + loaded_params: Set[str] = set()
331 for name, loaded_weight in weights:
332 if "rotary_emb.inv_freq" in name:
333 continue
@@ -358,3 +360,5 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
360 weight_loader = getattr(param, "weight_loader",
361 default_weight_loader)
362 weight_loader(param, loaded_weight)
363 + loaded_params.add(name)
364 + return loaded_params
vllm/model_executor/models/phi.py
+6 -2
@@ -34,7 +34,7 @@
34 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 """Inference-only Phi-1.5 model compatible with HuggingFace weights."""
37 -from typing import Iterable, List, Optional, Tuple, Union
37 +from typing import Iterable, List, Optional, Set, Tuple, Union
38
39 import torch
40 from torch import nn
@@ -345,7 +345,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
345 next_tokens = self.sampler(logits, sampling_metadata)
346 return next_tokens
347
348 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
348 + def load_weights(self, weights: Iterable[Tuple[str,
349 + torch.Tensor]]) -> Set[str]:
350 stacked_params_mapping = [
351 # (param_name, shard_name, shard_id)
352 ("qkv_proj", "q_proj", "q"),
@@ -353,6 +354,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
354 ("qkv_proj", "v_proj", "v")
355 ]
356 params_dict = dict(self.named_parameters())
357 + loaded_params: Set[str] = set()
358
359 for name, loaded_weight in weights:
360 if "rotary_emb.inv_freq" in name:
@@ -383,3 +385,5 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
385 weight_loader = getattr(param, "weight_loader",
386 default_weight_loader)
387 weight_loader(param, loaded_weight)
388 + loaded_params.add(name)
389 + return loaded_params
vllm/model_executor/models/phi3_small.py
+6 -2
@@ -1,5 +1,5 @@
1 import math
2 -from typing import Iterable, List, Optional, Tuple, Union
2 +from typing import Iterable, List, Optional, Set, Tuple, Union
3
4 import torch
5 from torch import nn
@@ -457,9 +457,11 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
457 sampling_metadata)
458 return next_tokens
459
460 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
460 + def load_weights(self, weights: Iterable[Tuple[str,
461 + torch.Tensor]]) -> Set[str]:
462
463 params_dict = dict(self.named_parameters())
464 + loaded_params: Set[str] = set()
465 for name, loaded_weight in weights:
466 if "rotary_emb.inv_freq" in name:
467 continue
@@ -471,3 +473,5 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
473 weight_loader = getattr(param, "weight_loader",
474 default_weight_loader)
475 weight_loader(param, loaded_weight)
476 + loaded_params.add(name)
477 + return loaded_params
vllm/model_executor/models/phi3v.py
+6 -3
@@ -15,7 +15,7 @@
15 import itertools
16 import re
17 from functools import cached_property, lru_cache
18 -from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
18 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
19 Tuple, TypedDict, Union)
20
21 import numpy as np
@@ -744,7 +744,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
744 ) -> Optional[PoolerOutput]:
745 return self._pooler(hidden_states, pooling_metadata)
746
747 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
747 + def load_weights(self, weights: Iterable[Tuple[str,
748 + torch.Tensor]]) -> Set[str]:
749 hf_to_vllm_mapper = WeightsMapper(
750 orig_to_new_prefix={
751 "model.vision_embed_tokens.wte": "embed_tokens",
@@ -759,5 +760,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
760
761 # The HF config doesn't specify whether these are tied,
762 # so we detect it this way
762 - if "embed_tokens" not in autoloaded_weights:
763 + if "embed_tokens.weight" not in autoloaded_weights:
764 self.embed_tokens = self.language_model.model.embed_tokens
765 + autoloaded_weights.add("embed_tokens.weight")
766 + return autoloaded_weights
vllm/model_executor/models/phimoe.py
+6 -2
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only PhiMoE model."""
23 -from typing import Iterable, List, Optional, Tuple, Union
23 +from typing import Iterable, List, Optional, Set, Tuple, Union
24
25 import torch
26 from torch import nn
@@ -598,7 +598,8 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
598 next_tokens = self.sampler(logits, sampling_metadata)
599 return next_tokens
600
601 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
601 + def load_weights(self, weights: Iterable[Tuple[str,
602 + torch.Tensor]]) -> Set[str]:
603 stacked_params_mapping = [
604 # (param_name, shard_name, shard_id)
605 ("qkv_proj", "q_proj", "q"),
@@ -613,6 +614,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
614 num_experts=self.config.num_local_experts)
615
616 params_dict = dict(self.named_parameters())
617 + loaded_params: Set[str] = set()
618 for name, loaded_weight in weights:
619 if "rotary_emb.inv_freq" in name:
620 continue
@@ -666,3 +668,5 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
668 weight_loader = getattr(param, "weight_loader",
669 default_weight_loader)
670 weight_loader(param, loaded_weight)
671 + loaded_params.add(name)
672 + return loaded_params
vllm/model_executor/models/pixtral.py
+8 -4
@@ -1,7 +1,7 @@
1 from dataclasses import dataclass, fields
2 from functools import cached_property
3 from itertools import tee
4 -from typing import Iterable, List, Mapping, Optional, Tuple, Union
4 +from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union
5
6 import numpy
7 import torch
@@ -1053,7 +1053,8 @@ class PixtralHFVisionModel(nn.Module):
1053
1054 # (TODO) Add prefix argument for filtering out weights to be loaded
1055 # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
1056 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1056 + def load_weights(self, weights: Iterable[Tuple[str,
1057 + torch.Tensor]]) -> Set[str]:
1058 stacked_params_mapping = [
1059 # (param_name, shard_name, shard_id)
1060 (".qkv_proj", ".q_proj", "q"),
@@ -1063,6 +1064,7 @@ class PixtralHFVisionModel(nn.Module):
1064 (".gate_up_proj", ".up_proj", 1),
1065 ]
1066 params_dict = dict(self.named_parameters())
1067 + loaded_params: Set[str] = set()
1068 layer_count = len(self.transformer.layers)
1069
1070 for name, loaded_weight in weights:
@@ -1075,8 +1077,8 @@ class PixtralHFVisionModel(nn.Module):
1077 for (param_name, weight_name, shard_id) in stacked_params_mapping:
1078 if weight_name not in name:
1079 continue
1078 -
1079 - param = params_dict[name.replace(weight_name, param_name)]
1080 + name = name.replace(weight_name, param_name)
1081 + param = params_dict[name]
1082 weight_loader = param.weight_loader
1083 weight_loader(param, loaded_weight, shard_id)
1084 break
@@ -1085,3 +1087,5 @@ class PixtralHFVisionModel(nn.Module):
1087 weight_loader = getattr(param, "weight_loader",
1088 default_weight_loader)
1089 weight_loader(param, loaded_weight)
1090 + loaded_params.add(name)
1091 + return loaded_params
vllm/model_executor/models/qwen.py
+6 -2
@@ -8,7 +8,7 @@ import math
8 import re
9 from functools import partial
10 from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
11 - Optional, Tuple, TypedDict, Union)
11 + Optional, Set, Tuple, TypedDict, Union)
12
13 import numpy as np
14 import torch
@@ -964,13 +964,15 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
964 next_tokens = self.sampler(logits, sampling_metadata)
965 return next_tokens
966
967 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
967 + def load_weights(self, weights: Iterable[Tuple[str,
968 + torch.Tensor]]) -> Set[str]:
969 stacked_params_mapping = [
970 # (param_name, shard_name, shard_id)
971 ("gate_up_proj", "w2", 0),
972 ("gate_up_proj", "w1", 1),
973 ]
974 params_dict = dict(self.named_parameters())
975 + loaded_params: Set[str] = set()
976 for name, loaded_weight in weights:
977 if "rotary_emb.inv_freq" in name:
978 continue
@@ -999,6 +1001,8 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
1001 weight_loader = getattr(param, "weight_loader",
1002 default_weight_loader)
1003 weight_loader(param, loaded_weight)
1004 + loaded_params.add(name)
1005 + return loaded_params
1006
1007
1008 class QWenLLM(QWenBaseModel):
vllm/model_executor/models/qwen2.py
+12 -6
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Qwen2 model compatible with HuggingFace weights."""
24 -from typing import Iterable, List, Optional, Tuple, Union
24 +from typing import Iterable, List, Optional, Set, Tuple, Union
25
26 import torch
27 from torch import nn
@@ -332,7 +332,8 @@ class Qwen2Model(nn.Module):
332 hidden_states, _ = self.norm(hidden_states, residual)
333 return hidden_states
334
335 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
335 + def load_weights(self, weights: Iterable[Tuple[str,
336 + torch.Tensor]]) -> Set[str]:
337 stacked_params_mapping = [
338 # (param_name, shard_name, shard_id)
339 ("qkv_proj", "q_proj", "q"),
@@ -342,6 +343,7 @@ class Qwen2Model(nn.Module):
343 ("gate_up_proj", "up_proj", 1),
344 ]
345 params_dict = dict(self.named_parameters(remove_duplicate=False))
346 + loaded_params: Set[str] = set()
347 for name, loaded_weight in weights:
348 if "rotary_emb.inv_freq" in name:
349 continue
@@ -372,6 +374,8 @@ class Qwen2Model(nn.Module):
374 weight_loader = getattr(param, "weight_loader",
375 default_weight_loader)
376 weight_loader(param, loaded_weight)
377 + loaded_params.add(name)
378 + return loaded_params
379
380
381 class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
@@ -494,13 +498,14 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
498 ) -> Optional[PoolerOutput]:
499 return self._pooler(hidden_states, pooling_metadata)
500
497 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
501 + def load_weights(self, weights: Iterable[Tuple[str,
502 + torch.Tensor]]) -> Set[str]:
503 loader = AutoWeightsLoader(
504 self,
505 skip_prefixes=(["lm_head."]
506 if self.config.tie_word_embeddings else None),
507 )
503 - loader.load_weights(weights)
508 + return loader.load_weights(weights)
509
510
511 class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
@@ -564,7 +569,8 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
569 ) -> Optional[PoolerOutput]:
570 return self._pooler(hidden_states, pooling_metadata)
571
567 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
572 + def load_weights(self, weights: Iterable[Tuple[str,
573 + torch.Tensor]]) -> Set[str]:
574 loader = AutoWeightsLoader(self,
575 ignore_unexpected_prefixes=["lm_head."])
570 - loader.load_weights(weights)
576 + return loader.load_weights(weights)
vllm/model_executor/models/qwen2_audio.py
+7 -2
@@ -20,7 +20,8 @@
20 # limitations under the License.
21 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22 from functools import lru_cache
23 -from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
23 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24 + Union)
25
26 import librosa
27 import numpy as np
@@ -420,7 +421,8 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
421 next_tokens = self.sampler(logits, sampling_metadata)
422 return next_tokens
423
423 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
424 + def load_weights(self, weights: Iterable[Tuple[str,
425 + torch.Tensor]]) -> Set[str]:
426 stacked_params_mapping = [
427 # (param_name, shard_name, shard_id)
428 ("qkv_proj", "q_proj", "q"),
@@ -430,6 +432,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
432 ("gate_up_proj", "up_proj", 1),
433 ]
434 params_dict = dict(self.named_parameters(remove_duplicate=False))
435 + loaded_params: Set[str] = set()
436 for name, loaded_weight in weights:
437 if "rotary_emb.inv_freq" in name:
438 continue
@@ -463,3 +466,5 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
466 weight_loader = getattr(param, "weight_loader",
467 default_weight_loader)
468 weight_loader(param, loaded_weight)
469 + loaded_params.add(name)
470 + return loaded_params
vllm/model_executor/models/qwen2_cls.py
+4 -3
@@ -4,7 +4,7 @@
4 # Copyright 2024 The Qwen team.
5 # Copyright 2023 The vLLM team.
6 """Inference-only Qwen2-Classification model compatible with HF weights."""
7 -from typing import Iterable, List, Optional, Tuple
7 +from typing import Iterable, List, Optional, Set, Tuple
8
9 import torch
10 from torch import nn
@@ -97,7 +97,8 @@ class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
97 ) -> Optional[PoolerOutput]:
98 return self._pooler(hidden_states, pooling_metadata)
99
100 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
100 + def load_weights(self, weights: Iterable[Tuple[str,
101 + torch.Tensor]]) -> Set[str]:
102 loader = AutoWeightsLoader(self,
103 ignore_unexpected_prefixes=["lm_head."])
103 - loader.load_weights(weights)
104 + return loader.load_weights(weights)
vllm/model_executor/models/qwen2_moe.py
+6 -2
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Qwen2MoE model compatible with HuggingFace weights."""
24 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
24 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
25
26 import torch
27 import torch.nn.functional as F
@@ -436,7 +436,8 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
436 next_tokens = self.sampler(logits, sampling_metadata)
437 return next_tokens
438
439 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
439 + def load_weights(self, weights: Iterable[Tuple[str,
440 + torch.Tensor]]) -> Set[str]:
441 stacked_params_mapping = [
442 # (param_name, shard_name, shard_id)
443 ("qkv_proj", "q_proj", "q"),
@@ -455,6 +456,7 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
456 num_experts=self.config.num_experts)
457
458 params_dict = dict(self.named_parameters())
459 + loaded_params: Set[str] = set()
460 for name, loaded_weight in weights:
461 if "rotary_emb.inv_freq" in name:
462 continue
@@ -532,3 +534,5 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
534 weight_loader = getattr(param, "weight_loader",
535 default_weight_loader)
536 weight_loader(param, loaded_weight)
537 + loaded_params.add(name)
538 + return loaded_params
vllm/model_executor/models/qwen2_rm.py
+4 -3
@@ -3,7 +3,7 @@
3 # Copyright 2024 The Qwen team.
4 # Copyright 2023 The vLLM team.
5 """Inference-only Qwen2-RM model compatible with HuggingFace weights."""
6 -from typing import Iterable, List, Optional, Tuple, Union
6 +from typing import Iterable, List, Optional, Set, Tuple, Union
7
8 import torch
9 from torch import nn
@@ -110,7 +110,8 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
110 ) -> Optional[PoolerOutput]:
111 return self._pooler(hidden_states, pooling_metadata)
112
113 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
113 + def load_weights(self, weights: Iterable[Tuple[str,
114 + torch.Tensor]]) -> Set[str]:
115 loader = AutoWeightsLoader(self,
116 ignore_unexpected_prefixes=["lm_head."])
116 - loader.load_weights(weights)
117 + return loader.load_weights(weights)
vllm/model_executor/models/qwen2_vl.py
+6 -2
@@ -23,7 +23,7 @@
23 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24 from functools import partial
25 from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26 - Optional, Tuple, Type, TypedDict, Union)
26 + Optional, Set, Tuple, Type, TypedDict, Union)
27
28 import torch
29 import torch.nn as nn
@@ -1333,7 +1333,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1333 ) -> Optional[PoolerOutput]:
1334 return self._pooler(hidden_states, pooling_metadata)
1335
1336 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1336 + def load_weights(self, weights: Iterable[Tuple[str,
1337 + torch.Tensor]]) -> Set[str]:
1338 stacked_params_mapping = [
1339 # (param_name, shard_name, shard_id)
1340 ("qkv_proj", "q_proj", "q"),
@@ -1343,6 +1344,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1344 ("gate_up_proj", "gate_proj", 0),
1345 ]
1346 params_dict = dict(self.named_parameters(remove_duplicate=False))
1347 + loaded_params: Set[str] = set()
1348 for name, loaded_weight in weights:
1349 if "rotary_emb.inv_freq" in name:
1350 continue
@@ -1392,3 +1394,5 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1394 weight_loader = getattr(param, "weight_loader",
1395 default_weight_loader)
1396 weight_loader(param, loaded_weight)
1397 + loaded_params.add(name)
1398 + return loaded_params
vllm/model_executor/models/siglip.py
+8 -3
@@ -2,7 +2,7 @@
2 within a vision language model."""
3
4 import math
5 -from typing import Iterable, List, Optional, Tuple, Union
5 +from typing import Iterable, List, Optional, Set, Tuple, Union
6
7 import numpy as np
8 import torch
@@ -594,7 +594,8 @@ class SiglipVisionModel(nn.Module):
594 interpolate_pos_encoding=interpolate_pos_encoding,
595 )
596
597 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
597 + def load_weights(self, weights: Iterable[Tuple[str,
598 + torch.Tensor]]) -> Set[str]:
599 stacked_params_mapping = [
600 # (param_name, shard_name, shard_id)
601 ("qkv_proj", "q_proj", "q"),
@@ -602,6 +603,7 @@ class SiglipVisionModel(nn.Module):
603 ("qkv_proj", "v_proj", "v"),
604 ] if self.shard_weight else []
605 params_dict = dict(self.named_parameters())
606 + loaded_params: Set[str] = set()
607 layer_count = len(self.vision_model.encoder.layers)
608
609 for name, loaded_weight in weights:
@@ -619,8 +621,9 @@ class SiglipVisionModel(nn.Module):
621 for (param_name, weight_name, shard_id) in stacked_params_mapping:
622 if weight_name not in name:
623 continue
624 + name = name.replace(weight_name, param_name)
625
623 - param = params_dict[name.replace(weight_name, param_name)]
626 + param = params_dict[name]
627 weight_loader = param.weight_loader
628 weight_loader(param, loaded_weight, shard_id)
629 break
@@ -629,3 +632,5 @@ class SiglipVisionModel(nn.Module):
632 weight_loader = getattr(param, "weight_loader",
633 default_weight_loader)
634 weight_loader(param, loaded_weight)
635 + loaded_params.add(name)
636 + return loaded_params
vllm/model_executor/models/solar.py
+7 -2
@@ -21,7 +21,7 @@
21 # limitations under the License.
22 """Inference-only Solar model compatible with HuggingFace weights."""
23
24 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
24 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
25
26 import torch
27 from torch import nn
@@ -477,7 +477,8 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
477 next_tokens = self.sampler(logits, sampling_metadata)
478 return next_tokens
479
480 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
480 + def load_weights(self, weights: Iterable[Tuple[str,
481 + torch.Tensor]]) -> Set[str]:
482 stacked_params_mapping = [
483 # (param_name, shard_name, shard_id)
484 (".qkv_proj", ".q_proj", "q"),
@@ -487,6 +488,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
488 (".gate_up_proj", ".up_proj", 1),
489 ]
490 params_dict = dict(self.named_parameters())
491 + loaded_params: Set[str] = set()
492 for name, loaded_weight in weights:
493 if "rotary_emb.inv_freq" in name:
494 continue
@@ -502,6 +504,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
504 default_weight_loader)
505 loaded_weight = loaded_weight[0]
506 weight_loader(param, loaded_weight)
507 + loaded_params.add(scale_name)
508 continue
509 for param_name, weight_name, shard_id in stacked_params_mapping:
510 if weight_name not in name:
@@ -535,6 +538,8 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
538 weight_loader = getattr(param, "weight_loader",
539 default_weight_loader)
540 weight_loader(param, loaded_weight)
541 + loaded_params.add(name)
542 + return loaded_params
543
544 # If this function is called, it should always initialize KV cache scale
545 # factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/stablelm.py
+6 -2
@@ -18,7 +18,7 @@
18 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19 """Inference-only StabeLM (https://github.com/Stability-AI/StableLM)
20 model compatible with HuggingFace weights."""
21 -from typing import Iterable, List, Optional, Tuple, Union
21 +from typing import Iterable, List, Optional, Set, Tuple, Union
22
23 import torch
24 from torch import nn
@@ -306,7 +306,8 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
306 next_tokens = self.sampler(logits, sampling_metadata)
307 return next_tokens
308
309 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
309 + def load_weights(self, weights: Iterable[Tuple[str,
310 + torch.Tensor]]) -> Set[str]:
311 stacked_params_mapping = [
312 # (param_name, shard_name, shard_id)
313 ("qkv_proj", "q_proj", "q"),
@@ -316,6 +317,7 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
317 ("gate_up_proj", "up_proj", 1),
318 ]
319 params_dict = dict(self.named_parameters())
320 + loaded_params: Set[str] = set()
321 for name, loaded_weight in weights:
322 if "rotary_emb.inv_freq" in name:
323 continue
@@ -347,3 +349,5 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
349 weight_loader = getattr(param, "weight_loader",
350 default_weight_loader)
351 weight_loader(param, loaded_weight)
352 + loaded_params.add(name)
353 + return loaded_params
vllm/model_executor/models/starcoder2.py
+6 -2
@@ -17,7 +17,7 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """ PyTorch Starcoder2 model."""
20 -from typing import Iterable, List, Optional, Tuple, Union
20 +from typing import Iterable, List, Optional, Set, Tuple, Union
21
22 import torch
23 from torch import nn
@@ -314,7 +314,8 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
314 next_tokens = self.sampler(logits, sampling_metadata)
315 return next_tokens
316
317 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
317 + def load_weights(self, weights: Iterable[Tuple[str,
318 + torch.Tensor]]) -> Set[str]:
319 stacked_params_mapping = [
320 # (param_name, shard_name, shard_id)
321 ("qkv_proj", "q_proj", "q"),
@@ -323,6 +324,7 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
324 ]
325
326 params_dict = dict(self.named_parameters(remove_duplicate=False))
327 + loaded_params: Set[str] = set()
328 for name, loaded_weight in weights:
329 if "rotary_emb.inv_freq" in name:
330 continue
@@ -346,3 +348,5 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
348 weight_loader = getattr(param, "weight_loader",
349 default_weight_loader)
350 weight_loader(param, loaded_weight)
351 + loaded_params.add(name)
352 + return loaded_params
vllm/model_executor/models/ultravox.py
+4 -3
@@ -3,7 +3,7 @@
3
4 import math
5 from functools import cached_property, lru_cache
6 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
6 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7 TypedDict, Union, cast)
8
9 import numpy as np
@@ -504,10 +504,11 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
504 ) -> Optional[SamplerOutput]:
505 return self.language_model.sample(logits, sampling_metadata)
506
507 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
507 + def load_weights(self, weights: Iterable[Tuple[str,
508 + torch.Tensor]]) -> Set[str]:
509 hf_to_vllm_mapper = WeightsMapper(
510 orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
511
512 loader = AutoWeightsLoader(self,
513 ignore_unexpected_prefixes=["audio_tower."])
513 - loader.load_weights(weights, mapper=hf_to_vllm_mapper)
514 + return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/utils.py
+6 -5
@@ -1,7 +1,7 @@
1 import itertools
2 from dataclasses import dataclass, field
3 from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
4 - Optional, Protocol, Tuple, Union, overload)
4 + Optional, Protocol, Set, Tuple, Union, overload)
5
6 import torch
7 import torch.nn as nn
@@ -172,8 +172,9 @@ class AutoWeightsLoader:
172 if module != self.module:
173 module_load_weights = getattr(module, "load_weights", None)
174 if callable(module_load_weights):
175 - module_load_weights(weights)
176 - return
175 + loaded_params = module_load_weights(weights)
176 + yield from map(lambda x: self._get_qualname(base_prefix, x),
177 + loaded_params)
178
179 child_modules = dict(module.named_children())
180 child_params = dict(module.named_parameters(recurse=False))
@@ -222,11 +223,11 @@ class AutoWeightsLoader:
223 weights: Iterable[Tuple[str, torch.Tensor]],
224 *,
225 mapper: Optional[WeightsMapper] = None,
225 - ) -> List[str]:
226 + ) -> Set[str]:
227 if mapper is not None:
228 weights = mapper.apply(weights)
229
229 - autoloaded_weights = list(self._load_module("", self.module, weights))
230 + autoloaded_weights = set(self._load_module("", self.module, weights))
231 return autoloaded_weights
232
233
vllm/model_executor/models/xverse.py
+6 -2
@@ -19,7 +19,7 @@
19 # See the License for the specific language governing permissions and
20 # limitations under the License.
21 """Inference-only Xverse model compatible with HuggingFace weights."""
22 -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
22 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
23
24 import torch
25 from torch import nn
@@ -376,7 +376,8 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
376 next_tokens = self.sampler(logits, sampling_metadata)
377 return next_tokens
378
379 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
379 + def load_weights(self, weights: Iterable[Tuple[str,
380 + torch.Tensor]]) -> Set[str]:
381 stacked_params_mapping = [
382 ("qkv_proj", "q_proj", "q"),
383 ("qkv_proj", "k_proj", "k"),
@@ -385,6 +386,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
386 ("gate_up_proj", "up_proj", 1),
387 ]
388 params_dict = dict(self.named_parameters())
389 + loaded_params: Set[str] = set()
390 for name, loaded_weight in weights:
391 if ("rotary_emb.inv_freq" in name
392 or "rotary_emb.cos_cached" in name
@@ -413,3 +415,5 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
415 weight_loader = getattr(param, "weight_loader",
416 default_weight_loader)
417 weight_loader(param, loaded_weight)
418 + loaded_params.add(name)
419 + return loaded_params