@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bbe5f9de7d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model] Support for fairseq2 Llama (#11442)

Signed-off-by: Martin Gleize <mgleize@meta.com> Co-authored-by: mgleize user <mgleize@a100-st-p4de24xlarge-4.fair-a100.hpcaas>

Martin Gleize committed Jan 19, 2025 at 19:40 UTC bbe5f9de7dab1fa905807577faa185d85040213a
7 files changed +197 -21
tests/models/registry.py
+1
@@ -69,6 +69,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
69 "DeepseekV3ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3", # noqa: E501
70 trust_remote_code=True),
71 "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct"), # noqa: E501
72 + "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
73 "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
74 "GemmaForCausalLM": _HfExamplesInfo("google/gemma-2b"),
75 "Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
tests/weight_loading/models.txt
+2 -1
@@ -30,4 +30,5 @@ marlin, nm-testing/zephyr-beta-7b-marlin-g128, main
30 marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
31 qqq, HandH1998/QQQ-Llama-3-8b-g128, main
32 qqq, HandH1998/QQQ-Llama-3-8b, main
33 -hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
\ No newline at end of file
33 +hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
34 +None, mgleize/fairseq2-dummy-Llama-3.2-1B, main
\ No newline at end of file
tests/weight_loading/test_weight_loading.py
+7 -6
@@ -20,12 +20,13 @@ def test_weight_loading(vllm_runner):
20 """
21 Test parameter weight loading with tp>1.
22 """
23 - with vllm_runner(model_name=MODEL_NAME,
24 - revision=REVISION,
25 - dtype=torch.half if QUANTIZATION == "gptq" else "auto",
26 - quantization=QUANTIZATION,
27 - max_model_len=MAX_MODEL_LEN,
28 - tensor_parallel_size=2) as model:
23 + with vllm_runner(
24 + model_name=MODEL_NAME,
25 + revision=REVISION,
26 + dtype=torch.half if QUANTIZATION == "gptq" else "auto",
27 + quantization=None if QUANTIZATION == "None" else QUANTIZATION,
28 + max_model_len=MAX_MODEL_LEN,
29 + tensor_parallel_size=2) as model:
30
31 output = model.generate_greedy("Hello world!", max_tokens=20)
32 print(output)
vllm/model_executor/layers/linear.py
+22 -12
@@ -344,11 +344,13 @@ class ColumnParallelLinear(LinearBase):
344 param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
345
346 use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
347 + is_sharded_weight = getattr(param, "is_sharded_weight", False)
348 + # bitsandbytes loads the weights of the specific portion
349 + # no need to narrow
350 + is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
351
352 param_data = param.data
349 - # bitsandbytes loads the weights of the specific portion
350 - # no need to narrow here
351 - if output_dim is not None and not use_bitsandbytes_4bit:
353 + if output_dim is not None and not is_sharded_weight:
354 shard_size = param_data.shape[output_dim]
355 start_idx = tp_rank * shard_size
356 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
@@ -546,6 +548,11 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
548
549 use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
550 False)
551 + is_sharded_weight = getattr(param, "is_sharded_weight", False)
552 + # bitsandbytes loads the weights of the specific portion
553 + # no need to narrow
554 + is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
555 +
556 if use_bitsandbytes_4bit:
557 shard_size = loaded_weight.shape[output_dim]
558 shard_offset = loaded_weight.shape[output_dim] * \
@@ -554,9 +561,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
561 param_data = param_data.narrow(output_dim, shard_offset,
562 shard_size)
563 start_idx = tp_rank * shard_size
557 - # bitsandbytes loads the weights of the specific portion
558 - # no need to narrow here
559 - if not use_bitsandbytes_4bit:
564 + if not is_sharded_weight:
565 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
566 shard_size)
567 # Special case for AQLM codebooks.
@@ -941,6 +946,11 @@ class QKVParallelLinear(ColumnParallelLinear):
946
947 use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
948 False)
949 + is_sharded_weight = getattr(param, "is_sharded_weight", False)
950 + # bitsandbytes loads the weights of the specific portion
951 + # no need to narrow
952 + is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
953 +
954 if use_bitsandbytes_4bit:
955 orig_qkv_offsets = {
956 "q": (0, self.num_heads * self.head_size),
@@ -964,9 +974,7 @@ class QKVParallelLinear(ColumnParallelLinear):
974 shard_id = tp_rank // self.num_kv_head_replicas
975 start_idx = shard_id * shard_size
976
967 - # bitsandbytes loads the weights of the specific portion
968 - # no need to narrow here
969 - if not use_bitsandbytes_4bit:
977 + if not is_sharded_weight:
978 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
979 shard_size)
980
@@ -1070,6 +1078,10 @@ class RowParallelLinear(LinearBase):
1078 tp_size = get_tensor_model_parallel_world_size()
1079 input_dim = getattr(param, "input_dim", None)
1080 use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
1081 + is_sharded_weight = getattr(param, "is_sharded_weight", False)
1082 + # bitsandbytes loads the weights of the specific portion
1083 + # no need to narrow
1084 + is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
1085
1086 # Special case for GGUF
1087 is_gguf_weight = getattr(param, "is_gguf_weight", False)
@@ -1085,9 +1097,7 @@ class RowParallelLinear(LinearBase):
1097 param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
1098
1099 param_data = param.data
1088 - # bitsandbytes loads the weights of the specific portion
1089 - # no need to narrow here
1090 - if input_dim is not None and not use_bitsandbytes_4bit:
1100 + if input_dim is not None and not is_sharded_weight:
1101 shard_size = param_data.shape[input_dim]
1102 start_idx = tp_rank * shard_size
1103 loaded_weight = loaded_weight.narrow(input_dim, start_idx,
vllm/model_executor/model_loader/loader.py
+13 -2
@@ -182,6 +182,9 @@ class DefaultModelLoader(BaseModelLoader):
182 fall_back_to_pt: bool = True
183 """Whether .pt weights can be used."""
184
185 + allow_patterns_overrides: Optional[list[str]] = None
186 + """If defined, weights will load exclusively using these patterns."""
187 +
188 def __init__(self, load_config: LoadConfig):
189 super().__init__(load_config)
190 if load_config.model_loader_extra_config:
@@ -218,6 +221,7 @@ class DefaultModelLoader(BaseModelLoader):
221 model_name_or_path: str,
222 revision: Optional[str],
223 fall_back_to_pt: bool,
224 + allow_patterns_overrides: Optional[list[str]],
225 ) -> Tuple[str, List[str], bool]:
226 """Prepare weights for the model.
227
@@ -249,6 +253,9 @@ class DefaultModelLoader(BaseModelLoader):
253 if fall_back_to_pt:
254 allow_patterns += ["*.pt"]
255
256 + if allow_patterns_overrides is not None:
257 + allow_patterns = allow_patterns_overrides
258 +
259 if not is_local:
260 hf_folder = download_weights_from_hf(
261 model_name_or_path,
@@ -298,7 +305,8 @@ class DefaultModelLoader(BaseModelLoader):
305 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
306 """Get an iterator for the model weights based on the load format."""
307 hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
301 - source.model_or_path, source.revision, source.fall_back_to_pt)
308 + source.model_or_path, source.revision, source.fall_back_to_pt,
309 + source.allow_patterns_overrides)
310 if self.load_config.load_format == LoadFormat.NPCACHE:
311 # Currently np_cache only support *.bin checkpoints
312 assert use_safetensors is False
@@ -340,6 +348,8 @@ class DefaultModelLoader(BaseModelLoader):
348 prefix="",
349 fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load",
350 True),
351 + allow_patterns_overrides=getattr(model, "allow_patterns_overrides",
352 + None),
353 )
354 yield from self._get_weights_iterator(primary_weights)
355
@@ -353,7 +363,8 @@ class DefaultModelLoader(BaseModelLoader):
363 def download_model(self, model_config: ModelConfig) -> None:
364 self._prepare_weights(model_config.model,
365 model_config.revision,
356 - fall_back_to_pt=True)
366 + fall_back_to_pt=True,
367 + allow_patterns_overrides=None)
368
369 def load_model(self, vllm_config: VllmConfig) -> nn.Module:
370 device_config = vllm_config.device_config
vllm/model_executor/models/fairseq2_llama.py new
+151
@@ -0,0 +1,151 @@
1 +# Copyright 2024 The vLLM team.
2 +# Copyright 2024 Meta Platforms, Inc. and affiliates. All rights reserved.
3 +#
4 +# Licensed under the Apache License, Version 2.0 (the "License");
5 +# you may not use this file except in compliance with the License.
6 +# You may obtain a copy of the License at
7 +#
8 +# http://www.apache.org/licenses/LICENSE-2.0
9 +#
10 +# Unless required by applicable law or agreed to in writing, software
11 +# distributed under the License is distributed on an "AS IS" BASIS,
12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 +# See the License for the specific language governing permissions and
14 +# limitations under the License.
15 +"""Llama model for fairseq2 weights."""
16 +
17 +from typing import Iterable, Set, Tuple
18 +
19 +import torch
20 +from torch.nn import Parameter
21 +
22 +from vllm.config import VllmConfig
23 +from vllm.distributed import (get_tensor_model_parallel_rank,
24 + get_tensor_model_parallel_world_size)
25 +from vllm.model_executor.layers.linear import set_weight_attrs
26 +from vllm.model_executor.models.llama import LlamaForCausalLM
27 +
28 +from .utils import AutoWeightsLoader, WeightsMapper
29 +
30 +
31 +class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
32 +
33 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
34 + super().__init__(vllm_config=vllm_config, prefix=prefix)
35 + self.tp_rank = get_tensor_model_parallel_rank()
36 + self.tp_size = get_tensor_model_parallel_world_size()
37 + # For the model loader to read only the relevant checkpoint files
38 + self.allow_patterns_overrides = [
39 + # either the full checkpoint
40 + "model.pt",
41 + # or the tp-sharded checkpoint of the current rank
42 + f"model.{self.tp_rank}.pt",
43 + ]
44 +
45 + def load_weights(self, weights: Iterable[Tuple[str,
46 + torch.Tensor]]) -> Set[str]:
47 + # fairseq2's serialization adds a wrapper to usual .pt state_dict's:
48 + # { "model_key": my_model_name, "my_model_name": state_dict }
49 + # which we first need to unpack
50 + weights_wrapped = dict(weights)
51 + weights = weights_wrapped[
52 + weights_wrapped["model_key"]].items() # type: ignore
53 +
54 + # remap keys
55 + fs2_to_vllm_mapper = WeightsMapper(
56 + orig_to_new_prefix={
57 + "decoder_frontend.embed.": "model.embed_tokens.",
58 + "decoder.": "model.",
59 + "final_proj.": "lm_head.",
60 + },
61 + orig_to_new_substr={
62 + ".self_attn_layer_norm.": ".input_layernorm.",
63 + ".ffn_layer_norm.": ".post_attention_layernorm.",
64 + ".self_attn.output_proj.": ".self_attn.o_proj.",
65 + ".ffn.gate_proj.": ".mlp.gate_proj.",
66 + ".ffn.inner_proj.": ".mlp.up_proj.",
67 + ".ffn.output_proj.": ".mlp.down_proj.",
68 + ".layer_norm.": ".norm.",
69 + },
70 + )
71 + weights = fs2_to_vllm_mapper.apply(weights)
72 +
73 + params = dict(self.named_parameters())
74 +
75 + loader = AutoWeightsLoader(
76 + self,
77 + skip_prefixes=(["lm_head."]
78 + if self.config.tie_word_embeddings else None),
79 + )
80 + return loader.load_weights(
81 + (self.reshape_fairseq2_weights(name, loaded_weight, params)
82 + for name, loaded_weight in weights))
83 +
84 + def flag_sharded_weights(self, params: dict[str, Parameter]):
85 + """Sets the `is_sharded_weight` flag to True for all sharded weights"""
86 + for name, param in params.items():
87 + modules = name.split(".")
88 + if "norm" in name and len(param.size()) < 2:
89 + # layer norms are not sharded
90 + continue
91 + elif any(emb in modules for emb in ["embed_tokens", "lm_head"]):
92 + # for now we repeat embedding layers for compatibility
93 + continue
94 + else:
95 + # all other layers are sharded
96 + set_weight_attrs(param, {"is_sharded_weight": True})
97 +
98 + def reshape_fairseq2_weights(
99 + self,
100 + name: str,
101 + loaded_weight: torch.Tensor,
102 + params: dict[str, Parameter],
103 + ) -> Tuple[str, torch.Tensor]:
104 + """Reshape fairseq2's weights."""
105 +
106 + def permute(w: torch.Tensor, n_heads: int) -> torch.Tensor:
107 + attn_in = self.config.head_dim * n_heads
108 + # check for a sharded weight on dim 0
109 + if attn_in // self.tp_size == w.size()[0]:
110 + attn_in //= self.tp_size
111 + n_heads //= self.tp_size
112 + attn_out = self.config.hidden_size
113 + return (w.view(n_heads, attn_in // n_heads // 2, 2,
114 + attn_out).transpose(1,
115 + 2).reshape(attn_in, attn_out))
116 +
117 + modules = name.split(".")
118 +
119 + # rotary embeds should be sliced
120 + if "k_proj" in modules:
121 + loaded_weight = permute(loaded_weight,
122 + self.config.num_key_value_heads)
123 +
124 + elif "q_proj" in modules:
125 + loaded_weight = permute(loaded_weight,
126 + self.config.num_attention_heads)
127 +
128 + # We make the loaded weights compatible with both
129 + # full checkpoints and tp sharded checkpoints.
130 + # Embeddings are repeated to fit the vocab size.
131 + # Other weights are flagged for the weight_loader calls.
132 + if any(emb in modules for emb in ["embed_tokens", "lm_head"]):
133 + # Embeddings are sharded on dim 0
134 + dim = 0
135 + # In fairseq2, vocab size has to be divisible by tp_size
136 + # so we don't worry about padding
137 + if self.tp_size > 1 and loaded_weight.shape[
138 + dim] < self.config.vocab_size:
139 + assert loaded_weight.shape[
140 + dim] * self.tp_size == self.config.vocab_size, \
141 + "vocab_size should be divisible by tp_size."
142 + repeats = [1] * len(loaded_weight.size())
143 + repeats[dim] = self.tp_size
144 + # repeat to match vocab size and to be easily 'narrow'able
145 + loaded_weight = loaded_weight.repeat(repeats)
146 + set_weight_attrs(params[name], {"is_sharded_weight": False})
147 + # if embeddings are sharded, the rest is too
148 + if "embed_tokens" in modules:
149 + self.flag_sharded_weights(params)
150 +
151 + return name, loaded_weight
vllm/model_executor/models/registry.py
+1
@@ -47,6 +47,7 @@ _TEXT_GENERATION_MODELS = {
47 "DeepseekV3ForCausalLM": ("deepseek_v3", "DeepseekV3ForCausalLM"),
48 "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
49 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
50 + "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
51 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
52 "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
53 "GlmForCausalLM": ("glm", "GlmForCausalLM"),