A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Model] Support for fairseq2 Llama (#11442)
Signed-off-by: Martin Gleize <mgleize@meta.com> Co-authored-by: mgleize user <mgleize@a100-st-p4de24xlarge-4.fair-a100.hpcaas>
Martin Gleize committed
Jan 19, 2025 at 19:40 UTC
bbe5f9de7dab1fa905807577faa185d85040213a
7 files changed
+197
-21
tests/models/registry.py
+1
@@ -69,6 +69,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
69
"DeepseekV3ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3", # noqa: E501
70
trust_remote_code=True),
71
"ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct"), # noqa: E501
72
+ "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
73
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
74
"GemmaForCausalLM": _HfExamplesInfo("google/gemma-2b"),
75
"Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
tests/weight_loading/models.txt
+2
-1
@@ -30,4 +30,5 @@ marlin, nm-testing/zephyr-beta-7b-marlin-g128, main
30
marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
31
qqq, HandH1998/QQQ-Llama-3-8b-g128, main
32
qqq, HandH1998/QQQ-Llama-3-8b, main
33
-hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
\ No newline at end of file
33
+hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
34
+None, mgleize/fairseq2-dummy-Llama-3.2-1B, main
\ No newline at end of file
tests/weight_loading/test_weight_loading.py
+7
-6
@@ -20,12 +20,13 @@ def test_weight_loading(vllm_runner):
20
"""
21
Test parameter weight loading with tp>1.
22
"""
23
- with vllm_runner(model_name=MODEL_NAME,
24
- revision=REVISION,
25
- dtype=torch.half if QUANTIZATION == "gptq" else "auto",
26
- quantization=QUANTIZATION,
27
- max_model_len=MAX_MODEL_LEN,
28
- tensor_parallel_size=2) as model:
23
+ with vllm_runner(
24
+ model_name=MODEL_NAME,
25
+ revision=REVISION,
26
+ dtype=torch.half if QUANTIZATION == "gptq" else "auto",
27
+ quantization=None if QUANTIZATION == "None" else QUANTIZATION,
28
+ max_model_len=MAX_MODEL_LEN,
29
+ tensor_parallel_size=2) as model:
30
31
output = model.generate_greedy("Hello world!", max_tokens=20)
32
print(output)
vllm/model_executor/layers/linear.py
+22
-12
@@ -344,11 +344,13 @@ class ColumnParallelLinear(LinearBase):
344
param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
345
346
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
347
+ is_sharded_weight = getattr(param, "is_sharded_weight", False)
348
+ # bitsandbytes loads the weights of the specific portion
349
+ # no need to narrow
350
+ is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
351
352
param_data = param.data
349
- # bitsandbytes loads the weights of the specific portion
350
- # no need to narrow here
351
- if output_dim is not None and not use_bitsandbytes_4bit:
353
+ if output_dim is not None and not is_sharded_weight:
354
shard_size = param_data.shape[output_dim]
355
start_idx = tp_rank * shard_size
356
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
@@ -546,6 +548,11 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
548
549
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
550
False)
551
+ is_sharded_weight = getattr(param, "is_sharded_weight", False)
552
+ # bitsandbytes loads the weights of the specific portion
553
+ # no need to narrow
554
+ is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
555
+
556
if use_bitsandbytes_4bit:
557
shard_size = loaded_weight.shape[output_dim]
558
shard_offset = loaded_weight.shape[output_dim] * \
@@ -554,9 +561,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
561
param_data = param_data.narrow(output_dim, shard_offset,
562
shard_size)
563
start_idx = tp_rank * shard_size
557
- # bitsandbytes loads the weights of the specific portion
558
- # no need to narrow here
559
- if not use_bitsandbytes_4bit:
564
+ if not is_sharded_weight:
565
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
566
shard_size)
567
# Special case for AQLM codebooks.
@@ -941,6 +946,11 @@ class QKVParallelLinear(ColumnParallelLinear):
946
947
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
948
False)
949
+ is_sharded_weight = getattr(param, "is_sharded_weight", False)
950
+ # bitsandbytes loads the weights of the specific portion
951
+ # no need to narrow
952
+ is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
953
+
954
if use_bitsandbytes_4bit:
955
orig_qkv_offsets = {
956
"q": (0, self.num_heads * self.head_size),
@@ -964,9 +974,7 @@ class QKVParallelLinear(ColumnParallelLinear):
974
shard_id = tp_rank // self.num_kv_head_replicas
975
start_idx = shard_id * shard_size
976
967
- # bitsandbytes loads the weights of the specific portion
968
- # no need to narrow here
969
- if not use_bitsandbytes_4bit:
977
+ if not is_sharded_weight:
978
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
979
shard_size)
980
@@ -1070,6 +1078,10 @@ class RowParallelLinear(LinearBase):
1078
tp_size = get_tensor_model_parallel_world_size()
1079
input_dim = getattr(param, "input_dim", None)
1080
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
1081
+ is_sharded_weight = getattr(param, "is_sharded_weight", False)
1082
+ # bitsandbytes loads the weights of the specific portion
1083
+ # no need to narrow
1084
+ is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
1085
1086
# Special case for GGUF
1087
is_gguf_weight = getattr(param, "is_gguf_weight", False)
@@ -1085,9 +1097,7 @@ class RowParallelLinear(LinearBase):
1097
param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
1098
1099
param_data = param.data
1088
- # bitsandbytes loads the weights of the specific portion
1089
- # no need to narrow here
1090
- if input_dim is not None and not use_bitsandbytes_4bit:
1100
+ if input_dim is not None and not is_sharded_weight:
1101
shard_size = param_data.shape[input_dim]
1102
start_idx = tp_rank * shard_size
1103
loaded_weight = loaded_weight.narrow(input_dim, start_idx,
vllm/model_executor/model_loader/loader.py
+13
-2
@@ -182,6 +182,9 @@ class DefaultModelLoader(BaseModelLoader):
182
fall_back_to_pt: bool = True
183
"""Whether .pt weights can be used."""
184
185
+ allow_patterns_overrides: Optional[list[str]] = None
186
+ """If defined, weights will load exclusively using these patterns."""
187
+
188
def __init__(self, load_config: LoadConfig):
189
super().__init__(load_config)
190
if load_config.model_loader_extra_config:
@@ -218,6 +221,7 @@ class DefaultModelLoader(BaseModelLoader):
221
model_name_or_path: str,
222
revision: Optional[str],
223
fall_back_to_pt: bool,
224
+ allow_patterns_overrides: Optional[list[str]],
225
) -> Tuple[str, List[str], bool]:
226
"""Prepare weights for the model.
227
@@ -249,6 +253,9 @@ class DefaultModelLoader(BaseModelLoader):
253
if fall_back_to_pt:
254
allow_patterns += ["*.pt"]
255
256
+ if allow_patterns_overrides is not None:
257
+ allow_patterns = allow_patterns_overrides
258
+
259
if not is_local:
260
hf_folder = download_weights_from_hf(
261
model_name_or_path,
@@ -298,7 +305,8 @@ class DefaultModelLoader(BaseModelLoader):
305
) -> Generator[Tuple[str, torch.Tensor], None, None]:
306
"""Get an iterator for the model weights based on the load format."""
307
hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
301
- source.model_or_path, source.revision, source.fall_back_to_pt)
308
+ source.model_or_path, source.revision, source.fall_back_to_pt,
309
+ source.allow_patterns_overrides)
310
if self.load_config.load_format == LoadFormat.NPCACHE:
311
# Currently np_cache only support *.bin checkpoints
312
assert use_safetensors is False
@@ -340,6 +348,8 @@ class DefaultModelLoader(BaseModelLoader):
348
prefix="",
349
fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load",
350
True),
351
+ allow_patterns_overrides=getattr(model, "allow_patterns_overrides",
352
+ None),
353
)
354
yield from self._get_weights_iterator(primary_weights)
355
@@ -353,7 +363,8 @@ class DefaultModelLoader(BaseModelLoader):
363
def download_model(self, model_config: ModelConfig) -> None:
364
self._prepare_weights(model_config.model,
365
model_config.revision,
356
- fall_back_to_pt=True)
366
+ fall_back_to_pt=True,
367
+ allow_patterns_overrides=None)
368
369
def load_model(self, vllm_config: VllmConfig) -> nn.Module:
370
device_config = vllm_config.device_config
vllm/model_executor/models/fairseq2_llama.py
new
+151
@@ -0,0 +1,151 @@
1
+# Copyright 2024 The vLLM team.
2
+# Copyright 2024 Meta Platforms, Inc. and affiliates. All rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+# http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+"""Llama model for fairseq2 weights."""
16
+
17
+from typing import Iterable, Set, Tuple
18
+
19
+import torch
20
+from torch.nn import Parameter
21
+
22
+from vllm.config import VllmConfig
23
+from vllm.distributed import (get_tensor_model_parallel_rank,
24
+ get_tensor_model_parallel_world_size)
25
+from vllm.model_executor.layers.linear import set_weight_attrs
26
+from vllm.model_executor.models.llama import LlamaForCausalLM
27
+
28
+from .utils import AutoWeightsLoader, WeightsMapper
29
+
30
+
31
+class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
32
+
33
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
34
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
35
+ self.tp_rank = get_tensor_model_parallel_rank()
36
+ self.tp_size = get_tensor_model_parallel_world_size()
37
+ # For the model loader to read only the relevant checkpoint files
38
+ self.allow_patterns_overrides = [
39
+ # either the full checkpoint
40
+ "model.pt",
41
+ # or the tp-sharded checkpoint of the current rank
42
+ f"model.{self.tp_rank}.pt",
43
+ ]
44
+
45
+ def load_weights(self, weights: Iterable[Tuple[str,
46
+ torch.Tensor]]) -> Set[str]:
47
+ # fairseq2's serialization adds a wrapper to usual .pt state_dict's:
48
+ # { "model_key": my_model_name, "my_model_name": state_dict }
49
+ # which we first need to unpack
50
+ weights_wrapped = dict(weights)
51
+ weights = weights_wrapped[
52
+ weights_wrapped["model_key"]].items() # type: ignore
53
+
54
+ # remap keys
55
+ fs2_to_vllm_mapper = WeightsMapper(
56
+ orig_to_new_prefix={
57
+ "decoder_frontend.embed.": "model.embed_tokens.",
58
+ "decoder.": "model.",
59
+ "final_proj.": "lm_head.",
60
+ },
61
+ orig_to_new_substr={
62
+ ".self_attn_layer_norm.": ".input_layernorm.",
63
+ ".ffn_layer_norm.": ".post_attention_layernorm.",
64
+ ".self_attn.output_proj.": ".self_attn.o_proj.",
65
+ ".ffn.gate_proj.": ".mlp.gate_proj.",
66
+ ".ffn.inner_proj.": ".mlp.up_proj.",
67
+ ".ffn.output_proj.": ".mlp.down_proj.",
68
+ ".layer_norm.": ".norm.",
69
+ },
70
+ )
71
+ weights = fs2_to_vllm_mapper.apply(weights)
72
+
73
+ params = dict(self.named_parameters())
74
+
75
+ loader = AutoWeightsLoader(
76
+ self,
77
+ skip_prefixes=(["lm_head."]
78
+ if self.config.tie_word_embeddings else None),
79
+ )
80
+ return loader.load_weights(
81
+ (self.reshape_fairseq2_weights(name, loaded_weight, params)
82
+ for name, loaded_weight in weights))
83
+
84
+ def flag_sharded_weights(self, params: dict[str, Parameter]):
85
+ """Sets the `is_sharded_weight` flag to True for all sharded weights"""
86
+ for name, param in params.items():
87
+ modules = name.split(".")
88
+ if "norm" in name and len(param.size()) < 2:
89
+ # layer norms are not sharded
90
+ continue
91
+ elif any(emb in modules for emb in ["embed_tokens", "lm_head"]):
92
+ # for now we repeat embedding layers for compatibility
93
+ continue
94
+ else:
95
+ # all other layers are sharded
96
+ set_weight_attrs(param, {"is_sharded_weight": True})
97
+
98
+ def reshape_fairseq2_weights(
99
+ self,
100
+ name: str,
101
+ loaded_weight: torch.Tensor,
102
+ params: dict[str, Parameter],
103
+ ) -> Tuple[str, torch.Tensor]:
104
+ """Reshape fairseq2's weights."""
105
+
106
+ def permute(w: torch.Tensor, n_heads: int) -> torch.Tensor:
107
+ attn_in = self.config.head_dim * n_heads
108
+ # check for a sharded weight on dim 0
109
+ if attn_in // self.tp_size == w.size()[0]:
110
+ attn_in //= self.tp_size
111
+ n_heads //= self.tp_size
112
+ attn_out = self.config.hidden_size
113
+ return (w.view(n_heads, attn_in // n_heads // 2, 2,
114
+ attn_out).transpose(1,
115
+ 2).reshape(attn_in, attn_out))
116
+
117
+ modules = name.split(".")
118
+
119
+ # rotary embeds should be sliced
120
+ if "k_proj" in modules:
121
+ loaded_weight = permute(loaded_weight,
122
+ self.config.num_key_value_heads)
123
+
124
+ elif "q_proj" in modules:
125
+ loaded_weight = permute(loaded_weight,
126
+ self.config.num_attention_heads)
127
+
128
+ # We make the loaded weights compatible with both
129
+ # full checkpoints and tp sharded checkpoints.
130
+ # Embeddings are repeated to fit the vocab size.
131
+ # Other weights are flagged for the weight_loader calls.
132
+ if any(emb in modules for emb in ["embed_tokens", "lm_head"]):
133
+ # Embeddings are sharded on dim 0
134
+ dim = 0
135
+ # In fairseq2, vocab size has to be divisible by tp_size
136
+ # so we don't worry about padding
137
+ if self.tp_size > 1 and loaded_weight.shape[
138
+ dim] < self.config.vocab_size:
139
+ assert loaded_weight.shape[
140
+ dim] * self.tp_size == self.config.vocab_size, \
141
+ "vocab_size should be divisible by tp_size."
142
+ repeats = [1] * len(loaded_weight.size())
143
+ repeats[dim] = self.tp_size
144
+ # repeat to match vocab size and to be easily 'narrow'able
145
+ loaded_weight = loaded_weight.repeat(repeats)
146
+ set_weight_attrs(params[name], {"is_sharded_weight": False})
147
+ # if embeddings are sharded, the rest is too
148
+ if "embed_tokens" in modules:
149
+ self.flag_sharded_weights(params)
150
+
151
+ return name, loaded_weight
vllm/model_executor/models/registry.py
+1
@@ -47,6 +47,7 @@ _TEXT_GENERATION_MODELS = {
47
"DeepseekV3ForCausalLM": ("deepseek_v3", "DeepseekV3ForCausalLM"),
48
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
49
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
50
+ "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
51
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
52
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
53
"GlmForCausalLM": ("glm", "GlmForCausalLM"),