A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Misc ] non-uniform quantization via `compressed-tensors` for `Llama` (#6515)
Robert Shaw committed
Jul 18, 2024 at 22:39 UTC
dbe55885541fcf27c0bc229df50b3ea3c53e38ce
11 files changed
+300
-90
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test -b auto -l 1000 -f 5 -t 1
2
+model_name: "nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.758
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.759
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -2,4 +2,5 @@ Meta-Llama-3-8B-Instruct.yaml
2
Meta-Llama-3-8B-Instruct-FP8.yaml
3
Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5
+Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
vllm/model_executor/layers/fused_moe/layer.py
+1
@@ -158,6 +158,7 @@ class FusedMoE(torch.nn.Module):
158
topk_group: Optional[int] = None,
159
quant_config: Optional[QuantizationConfig] = None,
160
tp_size: Optional[int] = None,
161
+ prefix: str = "",
162
):
163
super().__init__()
164
vllm/model_executor/layers/linear.py
+32
-12
@@ -171,6 +171,8 @@ class ReplicatedLinear(LinearBase):
171
skip_bias_add: If true, skip adding bias but instead return it.
172
params_dtype: Data type for the parameters.
173
quant_config: Quantization configure.
174
+ prefix: The name of the layer in the state dict, including all parents
175
+ (e.g. model.layers.0.qkv_proj)
176
"""
177
178
def __init__(self,
@@ -179,15 +181,19 @@ class ReplicatedLinear(LinearBase):
181
bias: bool = True,
182
skip_bias_add: bool = False,
183
params_dtype: Optional[torch.dtype] = None,
182
- quant_config: Optional[QuantizationConfig] = None):
184
+ quant_config: Optional[QuantizationConfig] = None,
185
+ prefix: Optional[str] = None):
186
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
187
quant_config)
188
189
# All the linear layer supports quant method.
190
assert self.quant_method is not None
188
- self.quant_method.create_weights(self, self.input_size,
189
- [self.output_size], self.input_size,
190
- self.output_size, self.params_dtype)
191
+ self.quant_method.create_weights(self,
192
+ self.input_size, [self.output_size],
193
+ self.input_size,
194
+ self.output_size,
195
+ self.params_dtype,
196
+ prefix=prefix)
197
198
if bias:
199
self.bias = Parameter(
@@ -239,6 +245,8 @@ class ColumnParallelLinear(LinearBase):
245
quant_config: Quantization configure.
246
output_sizes: list of output sizes packed into one output, like for QKV
247
the list would be size 3.
248
+ prefix: The name of the layer in the state dict, including all parents
249
+ (e.g. model.layers.0.qkv_proj)
250
"""
251
252
def __init__(self,
@@ -249,7 +257,8 @@ class ColumnParallelLinear(LinearBase):
257
skip_bias_add: bool = False,
258
params_dtype: Optional[torch.dtype] = None,
259
quant_config: Optional[QuantizationConfig] = None,
252
- output_sizes: Optional[List[int]] = None):
260
+ output_sizes: Optional[List[int]] = None,
261
+ prefix: Optional[str] = None):
262
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
263
quant_config)
264
@@ -276,7 +285,8 @@ class ColumnParallelLinear(LinearBase):
285
input_size=self.input_size,
286
output_size=self.output_size,
287
params_dtype=self.params_dtype,
279
- weight_loader=self.weight_loader)
288
+ weight_loader=self.weight_loader,
289
+ prefix=prefix)
290
if bias:
291
self.bias = Parameter(
292
torch.empty(self.output_size_per_partition,
@@ -348,6 +358,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
358
skip adding bias but instead return it.
359
params_dtype: Data type for the parameters.
360
quant_config: Quantization configure.
361
+ prefix: The name of the layer in the state dict, including all parents
362
+ (e.g. model.layers.0.qkv_proj)
363
"""
364
365
def __init__(self,
@@ -357,7 +369,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
369
gather_output: bool = False,
370
skip_bias_add: bool = False,
371
params_dtype: Optional[torch.dtype] = None,
360
- quant_config: Optional[QuantizationConfig] = None):
372
+ quant_config: Optional[QuantizationConfig] = None,
373
+ prefix: Optional[str] = None):
374
self.output_sizes = output_sizes
375
tp_size = get_tensor_model_parallel_world_size()
376
assert all(output_size % tp_size == 0 for output_size in output_sizes)
@@ -367,7 +380,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
380
gather_output=gather_output,
381
skip_bias_add=skip_bias_add,
382
params_dtype=params_dtype,
370
- quant_config=quant_config)
383
+ quant_config=quant_config,
384
+ prefix=prefix)
385
386
def weight_loader(self,
387
param: Parameter,
@@ -487,6 +501,8 @@ class QKVParallelLinear(ColumnParallelLinear):
501
skip adding bias but instead return it.
502
params_dtype: Data type for the parameters.
503
quant_config: Quantization configure.
504
+ prefix: The name of the layer in the state dict, including all parents
505
+ (e.g. model.layers.0.qkv_proj)
506
"""
507
508
def __init__(self,
@@ -497,7 +513,8 @@ class QKVParallelLinear(ColumnParallelLinear):
513
bias: bool = True,
514
skip_bias_add: bool = False,
515
params_dtype: Optional[torch.dtype] = None,
500
- quant_config: Optional[QuantizationConfig] = None):
516
+ quant_config: Optional[QuantizationConfig] = None,
517
+ prefix: Optional[str] = None):
518
self.hidden_size = hidden_size
519
self.head_size = head_size
520
self.total_num_heads = total_num_heads
@@ -529,7 +546,8 @@ class QKVParallelLinear(ColumnParallelLinear):
546
gather_output=False,
547
skip_bias_add=skip_bias_add,
548
params_dtype=params_dtype,
532
- quant_config=quant_config)
549
+ quant_config=quant_config,
550
+ prefix=prefix)
551
552
def weight_loader(self,
553
param: Parameter,
@@ -688,7 +706,8 @@ class RowParallelLinear(LinearBase):
706
skip_bias_add: bool = False,
707
params_dtype: Optional[torch.dtype] = None,
708
reduce_results: bool = True,
691
- quant_config: Optional[QuantizationConfig] = None):
709
+ quant_config: Optional[QuantizationConfig] = None,
710
+ prefix: Optional[str] = None):
711
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
712
quant_config)
713
@@ -706,7 +725,8 @@ class RowParallelLinear(LinearBase):
725
input_size=self.input_size,
726
output_size=self.output_size,
727
params_dtype=self.params_dtype,
709
- weight_loader=self.weight_loader)
728
+ weight_loader=self.weight_loader,
729
+ prefix=prefix)
730
if not reduce_results and (bias and not skip_bias_add):
731
raise ValueError("When not reduce the results, adding bias to the "
732
"results can lead to incorrect results")
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+58
-34
@@ -8,23 +8,25 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8
QuantizationConfig)
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
- CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12
- CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13
- CompressedTensorsWNA16)
11
+ CompressedTensorsScheme, CompressedTensorsUnquantized,
12
+ CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
13
+ CompressedTensorsW8A8Int8, CompressedTensorsWNA16)
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
CompressionFormat, QuantizationArgs, QuantizationStrategy,
16
- QuantizationType, find_first_name_or_class_match,
17
- is_activation_quantization_format)
16
+ QuantizationType, find_matched_target, is_activation_quantization_format,
17
+ should_ignore_layer)
18
from vllm.platforms import current_platform
19
20
21
class CompressedTensorsConfig(QuantizationConfig):
22
23
- def __init__(self, layer_quant_details: Dict[str, Any], ignore: List[str],
23
+ def __init__(self, target_scheme_map: Dict[str, Any], ignore: List[str],
24
quant_format: str):
25
+
26
self.ignore = ignore
26
- self.layer_quant_details = layer_quant_details
27
self.quant_format = quant_format
28
+ # Map from [target -> scheme]
29
+ self.target_scheme_map = target_scheme_map
30
31
def get_linear_method(self) -> "CompressedTensorsLinearMethod":
32
return CompressedTensorsLinearMethod(self)
@@ -51,7 +53,7 @@ class CompressedTensorsConfig(QuantizationConfig):
53
54
@classmethod
55
def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
54
- layer_quant_details: Dict[str, Any] = dict()
56
+ target_scheme_map: Dict[str, Any] = dict()
57
ignore: List[str] = config.get("ignore", None)
58
quant_format: str = config.get("format", None)
59
@@ -63,21 +65,21 @@ class CompressedTensorsConfig(QuantizationConfig):
65
# details follow the structure defined by the QuantizationArgs
66
# pydantic model, which is used to verify the structure of the
67
# quant_config and also store the details for later use.
66
- for key, quant_config in config["config_groups"].items():
68
+ for _, quant_config in config["config_groups"].items():
69
targets = quant_config.get("targets")
70
for target in targets:
69
- layer_quant_details[target] = {}
70
- layer_quant_details[target][
71
+ target_scheme_map[target] = {}
72
+ target_scheme_map[target][
73
"weights"] = QuantizationArgs.parse_obj(
74
quant_config.get("weights"))
75
try:
74
- layer_quant_details[target][
76
+ target_scheme_map[target][
77
"input_activations"] = QuantizationArgs.parse_obj(
78
quant_config.get("input_activations"))
79
except Exception:
78
- layer_quant_details[target]["input_activations"] = None
80
+ target_scheme_map[target]["input_activations"] = None
81
80
- return cls(layer_quant_details=layer_quant_details,
82
+ return cls(target_scheme_map=target_scheme_map,
83
ignore=ignore,
84
quant_format=quant_format)
85
@@ -167,8 +169,9 @@ class CompressedTensorsConfig(QuantizationConfig):
169
return (is_channel_group and input_quant_none and is_symmetric
170
and is_static)
171
170
- def _get_schema(self, weight_quant: BaseModel,
171
- input_quant: BaseModel) -> "CompressedTensorsScheme":
172
+ def _get_scheme_from_parts(
173
+ self, weight_quant: BaseModel,
174
+ input_quant: BaseModel) -> "CompressedTensorsScheme":
175
176
# Detect If Mixed Precision
177
if self._is_wNa16_group_channel(weight_quant, input_quant):
@@ -205,26 +208,47 @@ class CompressedTensorsConfig(QuantizationConfig):
208
raise NotImplementedError(
209
"No compressed-tensors compatible scheme was found.")
210
208
- def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
211
+ def get_scheme(
212
+ self,
213
+ layer: torch.nn.Module,
214
+ layer_name: Optional[str] = None) -> "CompressedTensorsScheme":
215
+ """
216
+ compressed-tensors supports non uniform in the following way:
217
+
218
+ ignore: List of layer_names or nn.Module names to be ignored.
219
+ targets of config_groups: There can be N config_groups which each
220
+ have a quantization scheme. Each config_group has a list of targets
221
+ which can be a full layer_name, a regex for a layer_name, or
222
+ an nn.Module name.
223
210
- layer_type_name = find_first_name_or_class_match(
211
- name="",
212
- module=layer,
213
- targets=self.layer_quant_details.keys(),
214
- check_contains=True)
224
+ We first check whether a layer is in the ignore group and use
225
+ CompressedTensorsUnquantized (i.e. fp16/bf16) scheme for the layer
226
216
- if layer_type_name is None:
217
- raise ValueError(f"Could not matching target for layer {layer}")
227
+ We then detect whether a layer_name is found in any target and
228
+ use the quantization scheme corresponding to the matched target
229
+ to select the CompressedTensorsScheme used for infernece.
230
+ """
231
+
232
+ # Check if the layer is skipped for quantization.
233
+ # TODO (@robertgshaw2): support module names
234
+ if should_ignore_layer(layer_name, ignore=self.ignore):
235
+ return CompressedTensorsUnquantized()
236
+
237
+ # Find the "target" in the compressed-tensors config
238
+ # that our layer conforms to.
239
+ # TODO (@robertgshaw): add compressed-tensors as dep
240
+ # so we do not have to re-write these functions
241
+ matched_target = find_matched_target(
242
+ layer_name=layer_name,
243
+ module=layer,
244
+ targets=self.target_scheme_map.keys())
245
219
- layer_quant_details: Dict[str, Any] = self.layer_quant_details.get(
220
- layer_type_name, None)
221
- if layer_quant_details is None:
222
- raise ValueError(
223
- f"Could not find quantization details for {layer}.")
246
+ # Find the quant_scheme
247
+ scheme = self.target_scheme_map[matched_target]
248
225
- scheme = self._get_schema(
226
- weight_quant=layer_quant_details["weights"],
227
- input_quant=layer_quant_details["input_activations"])
249
+ return self._get_scheme_from_parts(
250
+ weight_quant=scheme["weights"],
251
+ input_quant=scheme["input_activations"])
252
253
# Raise error if device does not support the scheme
254
# (e.g. fp8 needs ada lovelace)
@@ -250,11 +274,11 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
274
Use the CompressedTensorsScheme associated with each layer to create
275
the necessary parameters for the layer. See LinearMethodBase for param
276
details
253
-
277
"""
278
weight_loader = extra_weight_attrs.get("weight_loader")
279
+ layer_name = extra_weight_attrs.get("prefix")
280
257
- scheme = self.quantization_config.get_scheme(layer=layer)
281
+ scheme = self.quantization_config.get_scheme(layer, layer_name)
282
scheme.create_weights(
283
layer=layer,
284
input_size=input_size,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
-1
@@ -33,7 +33,6 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
33
34
weight = Parameter(torch.empty(sum(output_partition_sizes),
35
input_size_per_partition,
36
- device="cuda",
36
dtype=params_dtype),
37
requires_grad=False)
38
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+118
-21
@@ -86,25 +86,106 @@ def is_activation_quantization_format(format: str) -> bool:
86
return format in _ACTIVATION_QUANTIZATION_FORMATS
87
88
89
-def find_first_name_or_class_match(
90
- name: str,
91
- module: Module,
92
- targets: Iterable[str],
93
- check_contains: bool = False) -> Optional[str]:
89
+# fused_name: List[shard_name]
90
+_FUSED_LAYER_NAME_MAPPING = {
91
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
92
+ "gate_up_proj": ["gate_proj", "up_proj"]
93
+}
94
+
95
+
96
+def should_ignore_layer(layer_name: Optional[str],
97
+ ignore: Iterable[str]) -> bool:
98
+ if layer_name is None:
99
+ return False
100
+
101
+ # layer_name = model.layers.0.self_attn.qkv_proj
102
+ # proj_name = qkv_proj
103
+ proj_name = layer_name.split(".")[-1]
104
+
105
+ # Fused layers like gate_up_proj or qkv_proj will not be fused
106
+ # in the safetensors checkpoint. So, we convert the name
107
+ # from the fused version to unfused + check to make sure that
108
+ # each shard of the fused layer has the same scheme.
109
+ if proj_name in _FUSED_LAYER_NAME_MAPPING:
110
+ shard_proj_names = _FUSED_LAYER_NAME_MAPPING[proj_name]
111
+
112
+ # Convert fused_name --> [shard_names]
113
+ shard_names = [
114
+ layer_name.replace(proj_name, shard_proj_name)
115
+ for shard_proj_name in shard_proj_names
116
+ ]
117
+
118
+ # Layer should be ignored if shards are ignored.
119
+ should_ignore_layer = None
120
+ for shard_name in shard_names:
121
+ should_ignore_shard = check_equal_or_regex_match(
122
+ layer_name=shard_name, targets=ignore)
123
+
124
+ # If shard_idx=0, set layer ignore to match shard.
125
+ if should_ignore_layer is None:
126
+ should_ignore_layer = should_ignore_shard
127
+
128
+ # If shard_idx=1+ confirm scheme matches prior shards.
129
+ elif should_ignore_shard != should_ignore_layer:
130
+ raise ValueError(f"Found a different quantization schemes for "
131
+ f"{shard_proj_names} in {layer_name}. vLLM "
132
+ "requires all to use the same scheme.")
133
+
134
+ # Unfused layers like down_proj and o_proj will match
135
+ # the safetensors checkpoint already.
136
+ else:
137
+ should_ignore_layer = check_equal_or_regex_match(layer_name=layer_name,
138
+ targets=ignore)
139
+
140
+ assert should_ignore_layer is not None
141
+ return should_ignore_layer
142
+
143
+
144
+def check_equal_or_regex_match(layer_name: str,
145
+ targets: Iterable[str]) -> bool:
146
"""
95
- Helper function to map the quantization details listed in the config
96
- for a given list of targets against each model layer. First uses the
97
- layer name to try and find a match. If no name match is found, uses
98
- the layer class name. Returns None otherwise.
147
+ Checks whether a layer_name is exactly equal or a regex match for
148
+ if target starts with 're:' to any target in list.
149
+ """
150
+ for target in targets:
151
+ if _is_equal_or_regex_match(layer_name, target):
152
+ return True
153
+ return False
154
+
155
+
156
+def find_matched_target(layer_name: Optional[str], module: Module,
157
+ targets: Iterable[str]) -> str:
158
+ """
159
+ Helper function to look up which "target" in the compressed-tensors
160
+ config that a layer corresponds to.
161
100
- :param name: layer name
162
+ Recall that a compressed-tensors configs has a concept of
163
+ config_groups, where each layer can be quantized with with a different
164
+ scheme.
165
+
166
+ targets in each config_group will be a list of either layer names
167
+ (or regexes corresponding to layer names) or names of torch Modules.
168
+
169
+ First, we try to match the layer_name with a target
170
+ Second, we try to match the module's name with a target
171
+
172
+ :param layer_name: layer name
173
:param module: torch.nn.Module
174
:param targets: list of targets to match the layer against
103
- :param check_contains: whether or not to do a substring match
175
"""
176
106
- return _find_first_match(name, targets) or _find_first_match(
107
- module.__class__.__name__, targets, check_contains)
177
+ if layer_name is None:
178
+ layer_name = ""
179
+
180
+ matched_target = (_find_first_match(layer_name, targets)
181
+ or _find_first_match(module.__class__.__name__, targets,
182
+ True))
183
+
184
+ if matched_target is None:
185
+ raise ValueError(f"Unable to find matching target for {module} in the "
186
+ "compressed-tensors config.")
187
+
188
+ return matched_target
189
190
191
def _find_first_match(value: str,
@@ -121,13 +202,29 @@ def _find_first_match(value: str,
202
"""
203
204
for target in targets:
124
- if target.startswith("re:"):
125
- pattern = target[3:]
126
- if re.match(pattern, value):
127
- return target
128
- elif check_contains:
129
- if target.lower() in value.lower():
130
- return target
131
- elif target == value:
205
+ if _is_equal_or_regex_match(value,
206
+ target,
207
+ check_contains=check_contains):
208
return target
209
return None
210
+
211
+
212
+def _is_equal_or_regex_match(value: str,
213
+ target: str,
214
+ check_contains: bool = False) -> bool:
215
+ """
216
+ Checks whether a value is exactly equal or a regex match for target
217
+ if target starts with 're:'. If check_contains is set to True,
218
+ additionally checks if the target string is contained within the value.
219
+ """
220
+
221
+ if target.startswith("re:"):
222
+ pattern = target[3:]
223
+ if re.match(pattern, value):
224
+ return True
225
+ elif check_contains:
226
+ if target.lower() in value.lower():
227
+ return True
228
+ elif target == value:
229
+ return True
230
+ return False
vllm/model_executor/models/gpt2.py
+23
-4
@@ -51,6 +51,7 @@ class GPT2Attention(nn.Module):
51
config: GPT2Config,
52
cache_config: Optional[CacheConfig] = None,
53
quant_config: Optional[QuantizationConfig] = None,
54
+ prefix: str = "",
55
):
56
super().__init__()
57
self.hidden_size = config.hidden_size
@@ -68,12 +69,14 @@ class GPT2Attention(nn.Module):
69
total_num_heads,
70
bias=True,
71
quant_config=quant_config,
72
+ prefix=f"{prefix}.c_attn",
73
)
74
self.c_proj = RowParallelLinear(
75
self.hidden_size,
76
self.hidden_size,
77
bias=True,
78
quant_config=quant_config,
79
+ prefix=f"{prefix}.c_proj",
80
)
81
self.attn = Attention(self.num_heads,
82
self.head_dim,
@@ -101,6 +104,7 @@ class GPT2MLP(nn.Module):
104
intermediate_size: int,
105
config: GPT2Config,
106
quant_config: Optional[QuantizationConfig] = None,
107
+ prefix: str = "",
108
):
109
super().__init__()
110
hidden_size = config.hidden_size
@@ -109,12 +113,14 @@ class GPT2MLP(nn.Module):
113
intermediate_size,
114
bias=True,
115
quant_config=quant_config,
116
+ prefix=f"{prefix}.c_fc",
117
)
118
self.c_proj = RowParallelLinear(
119
intermediate_size,
120
hidden_size,
121
bias=True,
122
quant_config=quant_config,
123
+ prefix=f"{prefix}.c_proj",
124
)
125
self.act = get_act_fn(config.activation_function, quant_config,
126
intermediate_size)
@@ -133,6 +139,7 @@ class GPT2Block(nn.Module):
139
config: GPT2Config,
140
cache_config: Optional[CacheConfig] = None,
141
quant_config: Optional[QuantizationConfig] = None,
142
+ prefix: str = "",
143
):
144
super().__init__()
145
hidden_size = config.hidden_size
@@ -140,9 +147,15 @@ class GPT2Block(nn.Module):
147
hidden_size)
148
149
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
143
- self.attn = GPT2Attention(config, cache_config, quant_config)
150
+ self.attn = GPT2Attention(config,
151
+ cache_config,
152
+ quant_config,
153
+ prefix=f"{prefix}.attn")
154
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
145
- self.mlp = GPT2MLP(inner_dim, config, quant_config)
155
+ self.mlp = GPT2MLP(inner_dim,
156
+ config,
157
+ quant_config,
158
+ prefix=f"{prefix}.mlp")
159
160
def forward(
161
self,
@@ -175,6 +188,7 @@ class GPT2Model(nn.Module):
188
config: GPT2Config,
189
cache_config: Optional[CacheConfig] = None,
190
quant_config: Optional[QuantizationConfig] = None,
191
+ prefix: str = "",
192
):
193
super().__init__()
194
self.config = config
@@ -186,7 +200,9 @@ class GPT2Model(nn.Module):
200
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
201
self.start_layer, self.end_layer, self.h = make_layers(
202
config.num_hidden_layers,
189
- lambda: GPT2Block(config, cache_config, quant_config))
203
+ lambda prefix: GPT2Block(
204
+ config, cache_config, quant_config, prefix=prefix),
205
+ prefix=f"{prefix}.h")
206
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
207
208
def forward(
@@ -229,7 +245,10 @@ class GPT2LMHeadModel(nn.Module):
245
super().__init__()
246
self.config = config
247
self.quant_config = quant_config
232
- self.transformer = GPT2Model(config, cache_config, quant_config)
248
+ self.transformer = GPT2Model(config,
249
+ cache_config,
250
+ quant_config,
251
+ prefix="transformer")
252
self.lm_head = self.transformer.wte
253
self.logits_processor = LogitsProcessor(config.vocab_size)
254
self.sampler = Sampler()
vllm/model_executor/models/llama.py
+19
-6
@@ -62,17 +62,20 @@ class LlamaMLP(nn.Module):
62
hidden_act: str,
63
quant_config: Optional[QuantizationConfig] = None,
64
bias: bool = False,
65
+ prefix: str = "",
66
) -> None:
67
super().__init__()
68
self.gate_up_proj = MergedColumnParallelLinear(
69
input_size=hidden_size,
70
output_sizes=[intermediate_size] * 2,
71
bias=bias,
71
- quant_config=quant_config)
72
+ quant_config=quant_config,
73
+ prefix=f"{prefix}.gate_up_proj")
74
self.down_proj = RowParallelLinear(input_size=intermediate_size,
75
output_size=hidden_size,
76
bias=bias,
75
- quant_config=quant_config)
77
+ quant_config=quant_config,
78
+ prefix=f"{prefix}.down_proj")
79
if hidden_act != "silu":
80
raise ValueError(f"Unsupported activation: {hidden_act}. "
81
"Only silu is supported for now.")
@@ -99,6 +102,7 @@ class LlamaAttention(nn.Module):
102
quant_config: Optional[QuantizationConfig] = None,
103
bias: bool = False,
104
cache_config: Optional[CacheConfig] = None,
105
+ prefix: str = "",
106
) -> None:
107
super().__init__()
108
self.hidden_size = hidden_size
@@ -132,12 +136,14 @@ class LlamaAttention(nn.Module):
136
total_num_kv_heads=self.total_num_kv_heads,
137
bias=bias,
138
quant_config=quant_config,
139
+ prefix=f"{prefix}.qkv_proj",
140
)
141
self.o_proj = RowParallelLinear(
142
input_size=self.total_num_heads * self.head_dim,
143
output_size=hidden_size,
144
bias=bias,
145
quant_config=quant_config,
146
+ prefix=f"{prefix}.o_proj",
147
)
148
149
self.rotary_emb = get_rope(
@@ -176,6 +182,7 @@ class LlamaDecoderLayer(nn.Module):
182
config: LlamaConfig,
183
cache_config: Optional[CacheConfig] = None,
184
quant_config: Optional[QuantizationConfig] = None,
185
+ prefix: str = "",
186
) -> None:
187
super().__init__()
188
self.hidden_size = config.hidden_size
@@ -203,6 +210,7 @@ class LlamaDecoderLayer(nn.Module):
210
quant_config=quant_config,
211
bias=attention_bias,
212
cache_config=cache_config,
213
+ prefix=f"{prefix}.self_attn",
214
)
215
self.mlp = LlamaMLP(
216
hidden_size=self.hidden_size,
@@ -210,6 +218,7 @@ class LlamaDecoderLayer(nn.Module):
218
hidden_act=config.hidden_act,
219
quant_config=quant_config,
220
bias=getattr(config, "mlp_bias", False),
221
+ prefix=f"{prefix}.mlp",
222
)
223
self.input_layernorm = RMSNorm(config.hidden_size,
224
eps=config.rms_norm_eps)
@@ -253,6 +262,7 @@ class LlamaModel(nn.Module):
262
cache_config: Optional[CacheConfig] = None,
263
quant_config: Optional[QuantizationConfig] = None,
264
lora_config: Optional[LoRAConfig] = None,
265
+ prefix: str = "",
266
) -> None:
267
super().__init__()
268
self.config = config
@@ -272,9 +282,11 @@ class LlamaModel(nn.Module):
282
self.embed_tokens = PPMissingLayer()
283
self.start_layer, self.end_layer, self.layers = make_layers(
284
config.num_hidden_layers,
275
- lambda: LlamaDecoderLayer(config=config,
276
- cache_config=cache_config,
277
- quant_config=quant_config))
285
+ lambda prefix: LlamaDecoderLayer(config=config,
286
+ cache_config=cache_config,
287
+ quant_config=quant_config,
288
+ prefix=prefix),
289
+ prefix=f"{prefix}.layers")
290
if get_pp_group().is_last_rank:
291
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
292
else:
@@ -370,7 +382,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
382
self.model = LlamaModel(config,
383
cache_config,
384
quant_config,
373
- lora_config=lora_config)
385
+ lora_config=lora_config,
386
+ prefix="model")
387
if get_pp_group().is_last_rank:
388
self.unpadded_vocab_size = config.vocab_size
389
if lora_config:
vllm/model_executor/models/mixtral.py
+22
-8
@@ -67,7 +67,8 @@ class MixtralMoE(nn.Module):
67
intermediate_size: int,
68
params_dtype: Optional[torch.dtype] = None,
69
quant_config: Optional[QuantizationConfig] = None,
70
- tp_size: Optional[int] = None):
70
+ tp_size: Optional[int] = None,
71
+ prefix: str = ""):
72
super().__init__()
73
self.hidden_size = hidden_size
74
@@ -76,7 +77,8 @@ class MixtralMoE(nn.Module):
77
num_experts,
78
bias=False,
79
params_dtype=params_dtype,
79
- quant_config=None)
80
+ quant_config=None,
81
+ prefix=f"{prefix}.gate")
82
83
self.experts = FusedMoE(num_experts=num_experts,
84
top_k=top_k,
@@ -86,7 +88,8 @@ class MixtralMoE(nn.Module):
88
reduce_results=True,
89
renormalize=True,
90
quant_config=quant_config,
89
- tp_size=tp_size)
91
+ tp_size=tp_size,
92
+ prefix=f"{prefix}.experts")
93
94
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
95
# NOTE: hidden_states can have either 1D or 2D shape.
@@ -109,6 +112,7 @@ class MixtralAttention(nn.Module):
112
rope_theta: float = 10000,
113
cache_config: Optional[CacheConfig] = None,
114
quant_config: Optional[QuantizationConfig] = None,
115
+ prefix: str = "",
116
) -> None:
117
super().__init__()
118
self.hidden_size = hidden_size
@@ -139,12 +143,14 @@ class MixtralAttention(nn.Module):
143
self.total_num_kv_heads,
144
bias=False,
145
quant_config=quant_config,
146
+ prefix=f"{prefix}.qkv_proj",
147
)
148
self.o_proj = RowParallelLinear(
149
self.total_num_heads * self.head_dim,
150
hidden_size,
151
bias=False,
152
quant_config=quant_config,
153
+ prefix=f"{prefix}.o_proj",
154
)
155
self.rotary_emb = get_rope(
156
self.head_dim,
@@ -182,6 +188,7 @@ class MixtralDecoderLayer(nn.Module):
188
config: MixtralConfig,
189
cache_config: Optional[CacheConfig] = None,
190
quant_config: Optional[QuantizationConfig] = None,
191
+ prefix: str = "",
192
) -> None:
193
super().__init__()
194
self.hidden_size = config.hidden_size
@@ -194,13 +201,15 @@ class MixtralDecoderLayer(nn.Module):
201
num_kv_heads=config.num_key_value_heads,
202
rope_theta=rope_theta,
203
cache_config=cache_config,
197
- quant_config=quant_config)
204
+ quant_config=quant_config,
205
+ prefix=f"{prefix}.self_attn")
206
self.block_sparse_moe = MixtralMoE(
207
num_experts=config.num_local_experts,
208
top_k=config.num_experts_per_tok,
209
hidden_size=config.hidden_size,
210
intermediate_size=config.intermediate_size,
203
- quant_config=quant_config)
211
+ quant_config=quant_config,
212
+ prefix=f"{prefix}.block_sparse_moe")
213
self.input_layernorm = RMSNorm(config.hidden_size,
214
eps=config.rms_norm_eps)
215
self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -243,6 +252,7 @@ class MixtralModel(nn.Module):
252
cache_config: Optional[CacheConfig] = None,
253
quant_config: Optional[QuantizationConfig] = None,
254
lora_config: Optional[LoRAConfig] = None,
255
+ prefix: str = "",
256
) -> None:
257
super().__init__()
258
self.padding_idx = config.pad_token_id
@@ -258,8 +268,11 @@ class MixtralModel(nn.Module):
268
)
269
270
self.start_layer, self.end_layer, self.layers = make_layers(
261
- config.num_hidden_layers, lambda: MixtralDecoderLayer(
262
- config, cache_config, quant_config=quant_config))
271
+ config.num_hidden_layers,
272
+ lambda prefix: MixtralDecoderLayer(
273
+ config, cache_config, quant_config=quant_config, prefix=prefix
274
+ ),
275
+ prefix=f"{prefix}.layers")
276
277
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
278
@@ -331,7 +344,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
344
self.model = MixtralModel(config,
345
cache_config,
346
quant_config,
334
- lora_config=lora_config)
347
+ lora_config=lora_config,
348
+ prefix="model")
349
self.unpadded_vocab_size = config.vocab_size
350
if lora_config:
351
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/utils.py
+15
-4
@@ -1,4 +1,4 @@
1
-from typing import Callable, Dict, List, Tuple
1
+from typing import Dict, List, Protocol, Tuple
2
3
import torch
4
from torch.func import functional_call
@@ -45,6 +45,15 @@ def merge_vision_embeddings(input_ids: torch.Tensor,
45
return inputs_embeds
46
47
48
+class LayerFn(Protocol):
49
+
50
+ def __call__(
51
+ self,
52
+ prefix="",
53
+ ) -> torch.nn.Module:
54
+ ...
55
+
56
+
57
class PPMissingLayer(torch.nn.Identity):
58
"""
59
A placeholder layer for missing layers in a pipeline parallel model.
@@ -119,7 +128,9 @@ def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module:
128
129
130
def make_layers(
122
- num_hidden_layers: int, layer_fn: Callable[[], torch.nn.Module]
131
+ num_hidden_layers: int,
132
+ layer_fn: LayerFn,
133
+ prefix: str,
134
) -> Tuple[int, int, torch.nn.ModuleList]:
135
"""Make a list of layers with the given layer function, taking
136
pipeline parallelism into account.
@@ -131,8 +142,8 @@ def make_layers(
142
get_pp_group().world_size)
143
modules = torch.nn.ModuleList(
144
[PPMissingLayer() for _ in range(start_layer)] + [
134
- maybe_offload_to_cpu(layer_fn())
135
- for _ in range(start_layer, end_layer)
145
+ maybe_offload_to_cpu(layer_fn(prefix=f"{prefix}.{idx}"))
146
+ for idx in range(start_layer, end_layer)
147
] + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
148
return start_layer, end_layer, modules
149