@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dbe5588554
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Misc ] non-uniform quantization via `compressed-tensors` for `Llama` (#6515)

Robert Shaw committed Jul 18, 2024 at 22:39 UTC dbe55885541fcf27c0bc229df50b3ea3c53e38ce
11 files changed +300 -90
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test -b auto -l 1000 -f 5 -t 1
2 +model_name: "nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.758
8 + - name: "exact_match,flexible-extract"
9 + value: 0.759
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -2,4 +2,5 @@ Meta-Llama-3-8B-Instruct.yaml
2 Meta-Llama-3-8B-Instruct-FP8.yaml
3 Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5 +Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6 Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
vllm/model_executor/layers/fused_moe/layer.py
+1
@@ -158,6 +158,7 @@ class FusedMoE(torch.nn.Module):
158 topk_group: Optional[int] = None,
159 quant_config: Optional[QuantizationConfig] = None,
160 tp_size: Optional[int] = None,
161 + prefix: str = "",
162 ):
163 super().__init__()
164
vllm/model_executor/layers/linear.py
+32 -12
@@ -171,6 +171,8 @@ class ReplicatedLinear(LinearBase):
171 skip_bias_add: If true, skip adding bias but instead return it.
172 params_dtype: Data type for the parameters.
173 quant_config: Quantization configure.
174 + prefix: The name of the layer in the state dict, including all parents
175 + (e.g. model.layers.0.qkv_proj)
176 """
177
178 def __init__(self,
@@ -179,15 +181,19 @@ class ReplicatedLinear(LinearBase):
181 bias: bool = True,
182 skip_bias_add: bool = False,
183 params_dtype: Optional[torch.dtype] = None,
182 - quant_config: Optional[QuantizationConfig] = None):
184 + quant_config: Optional[QuantizationConfig] = None,
185 + prefix: Optional[str] = None):
186 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
187 quant_config)
188
189 # All the linear layer supports quant method.
190 assert self.quant_method is not None
188 - self.quant_method.create_weights(self, self.input_size,
189 - [self.output_size], self.input_size,
190 - self.output_size, self.params_dtype)
191 + self.quant_method.create_weights(self,
192 + self.input_size, [self.output_size],
193 + self.input_size,
194 + self.output_size,
195 + self.params_dtype,
196 + prefix=prefix)
197
198 if bias:
199 self.bias = Parameter(
@@ -239,6 +245,8 @@ class ColumnParallelLinear(LinearBase):
245 quant_config: Quantization configure.
246 output_sizes: list of output sizes packed into one output, like for QKV
247 the list would be size 3.
248 + prefix: The name of the layer in the state dict, including all parents
249 + (e.g. model.layers.0.qkv_proj)
250 """
251
252 def __init__(self,
@@ -249,7 +257,8 @@ class ColumnParallelLinear(LinearBase):
257 skip_bias_add: bool = False,
258 params_dtype: Optional[torch.dtype] = None,
259 quant_config: Optional[QuantizationConfig] = None,
252 - output_sizes: Optional[List[int]] = None):
260 + output_sizes: Optional[List[int]] = None,
261 + prefix: Optional[str] = None):
262 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
263 quant_config)
264
@@ -276,7 +285,8 @@ class ColumnParallelLinear(LinearBase):
285 input_size=self.input_size,
286 output_size=self.output_size,
287 params_dtype=self.params_dtype,
279 - weight_loader=self.weight_loader)
288 + weight_loader=self.weight_loader,
289 + prefix=prefix)
290 if bias:
291 self.bias = Parameter(
292 torch.empty(self.output_size_per_partition,
@@ -348,6 +358,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
358 skip adding bias but instead return it.
359 params_dtype: Data type for the parameters.
360 quant_config: Quantization configure.
361 + prefix: The name of the layer in the state dict, including all parents
362 + (e.g. model.layers.0.qkv_proj)
363 """
364
365 def __init__(self,
@@ -357,7 +369,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
369 gather_output: bool = False,
370 skip_bias_add: bool = False,
371 params_dtype: Optional[torch.dtype] = None,
360 - quant_config: Optional[QuantizationConfig] = None):
372 + quant_config: Optional[QuantizationConfig] = None,
373 + prefix: Optional[str] = None):
374 self.output_sizes = output_sizes
375 tp_size = get_tensor_model_parallel_world_size()
376 assert all(output_size % tp_size == 0 for output_size in output_sizes)
@@ -367,7 +380,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
380 gather_output=gather_output,
381 skip_bias_add=skip_bias_add,
382 params_dtype=params_dtype,
370 - quant_config=quant_config)
383 + quant_config=quant_config,
384 + prefix=prefix)
385
386 def weight_loader(self,
387 param: Parameter,
@@ -487,6 +501,8 @@ class QKVParallelLinear(ColumnParallelLinear):
501 skip adding bias but instead return it.
502 params_dtype: Data type for the parameters.
503 quant_config: Quantization configure.
504 + prefix: The name of the layer in the state dict, including all parents
505 + (e.g. model.layers.0.qkv_proj)
506 """
507
508 def __init__(self,
@@ -497,7 +513,8 @@ class QKVParallelLinear(ColumnParallelLinear):
513 bias: bool = True,
514 skip_bias_add: bool = False,
515 params_dtype: Optional[torch.dtype] = None,
500 - quant_config: Optional[QuantizationConfig] = None):
516 + quant_config: Optional[QuantizationConfig] = None,
517 + prefix: Optional[str] = None):
518 self.hidden_size = hidden_size
519 self.head_size = head_size
520 self.total_num_heads = total_num_heads
@@ -529,7 +546,8 @@ class QKVParallelLinear(ColumnParallelLinear):
546 gather_output=False,
547 skip_bias_add=skip_bias_add,
548 params_dtype=params_dtype,
532 - quant_config=quant_config)
549 + quant_config=quant_config,
550 + prefix=prefix)
551
552 def weight_loader(self,
553 param: Parameter,
@@ -688,7 +706,8 @@ class RowParallelLinear(LinearBase):
706 skip_bias_add: bool = False,
707 params_dtype: Optional[torch.dtype] = None,
708 reduce_results: bool = True,
691 - quant_config: Optional[QuantizationConfig] = None):
709 + quant_config: Optional[QuantizationConfig] = None,
710 + prefix: Optional[str] = None):
711 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
712 quant_config)
713
@@ -706,7 +725,8 @@ class RowParallelLinear(LinearBase):
725 input_size=self.input_size,
726 output_size=self.output_size,
727 params_dtype=self.params_dtype,
709 - weight_loader=self.weight_loader)
728 + weight_loader=self.weight_loader,
729 + prefix=prefix)
730 if not reduce_results and (bias and not skip_bias_add):
731 raise ValueError("When not reduce the results, adding bias to the "
732 "results can lead to incorrect results")
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+58 -34
@@ -8,23 +8,25 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8 QuantizationConfig)
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 - CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
12 - CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13 - CompressedTensorsWNA16)
11 + CompressedTensorsScheme, CompressedTensorsUnquantized,
12 + CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
13 + CompressedTensorsW8A8Int8, CompressedTensorsWNA16)
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
16 - QuantizationType, find_first_name_or_class_match,
17 - is_activation_quantization_format)
16 + QuantizationType, find_matched_target, is_activation_quantization_format,
17 + should_ignore_layer)
18 from vllm.platforms import current_platform
19
20
21 class CompressedTensorsConfig(QuantizationConfig):
22
23 - def __init__(self, layer_quant_details: Dict[str, Any], ignore: List[str],
23 + def __init__(self, target_scheme_map: Dict[str, Any], ignore: List[str],
24 quant_format: str):
25 +
26 self.ignore = ignore
26 - self.layer_quant_details = layer_quant_details
27 self.quant_format = quant_format
28 + # Map from [target -> scheme]
29 + self.target_scheme_map = target_scheme_map
30
31 def get_linear_method(self) -> "CompressedTensorsLinearMethod":
32 return CompressedTensorsLinearMethod(self)
@@ -51,7 +53,7 @@ class CompressedTensorsConfig(QuantizationConfig):
53
54 @classmethod
55 def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
54 - layer_quant_details: Dict[str, Any] = dict()
56 + target_scheme_map: Dict[str, Any] = dict()
57 ignore: List[str] = config.get("ignore", None)
58 quant_format: str = config.get("format", None)
59
@@ -63,21 +65,21 @@ class CompressedTensorsConfig(QuantizationConfig):
65 # details follow the structure defined by the QuantizationArgs
66 # pydantic model, which is used to verify the structure of the
67 # quant_config and also store the details for later use.
66 - for key, quant_config in config["config_groups"].items():
68 + for _, quant_config in config["config_groups"].items():
69 targets = quant_config.get("targets")
70 for target in targets:
69 - layer_quant_details[target] = {}
70 - layer_quant_details[target][
71 + target_scheme_map[target] = {}
72 + target_scheme_map[target][
73 "weights"] = QuantizationArgs.parse_obj(
74 quant_config.get("weights"))
75 try:
74 - layer_quant_details[target][
76 + target_scheme_map[target][
77 "input_activations"] = QuantizationArgs.parse_obj(
78 quant_config.get("input_activations"))
79 except Exception:
78 - layer_quant_details[target]["input_activations"] = None
80 + target_scheme_map[target]["input_activations"] = None
81
80 - return cls(layer_quant_details=layer_quant_details,
82 + return cls(target_scheme_map=target_scheme_map,
83 ignore=ignore,
84 quant_format=quant_format)
85
@@ -167,8 +169,9 @@ class CompressedTensorsConfig(QuantizationConfig):
169 return (is_channel_group and input_quant_none and is_symmetric
170 and is_static)
171
170 - def _get_schema(self, weight_quant: BaseModel,
171 - input_quant: BaseModel) -> "CompressedTensorsScheme":
172 + def _get_scheme_from_parts(
173 + self, weight_quant: BaseModel,
174 + input_quant: BaseModel) -> "CompressedTensorsScheme":
175
176 # Detect If Mixed Precision
177 if self._is_wNa16_group_channel(weight_quant, input_quant):
@@ -205,26 +208,47 @@ class CompressedTensorsConfig(QuantizationConfig):
208 raise NotImplementedError(
209 "No compressed-tensors compatible scheme was found.")
210
208 - def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
211 + def get_scheme(
212 + self,
213 + layer: torch.nn.Module,
214 + layer_name: Optional[str] = None) -> "CompressedTensorsScheme":
215 + """
216 + compressed-tensors supports non uniform in the following way:
217 +
218 + ignore: List of layer_names or nn.Module names to be ignored.
219 + targets of config_groups: There can be N config_groups which each
220 + have a quantization scheme. Each config_group has a list of targets
221 + which can be a full layer_name, a regex for a layer_name, or
222 + an nn.Module name.
223
210 - layer_type_name = find_first_name_or_class_match(
211 - name="",
212 - module=layer,
213 - targets=self.layer_quant_details.keys(),
214 - check_contains=True)
224 + We first check whether a layer is in the ignore group and use
225 + CompressedTensorsUnquantized (i.e. fp16/bf16) scheme for the layer
226
216 - if layer_type_name is None:
217 - raise ValueError(f"Could not matching target for layer {layer}")
227 + We then detect whether a layer_name is found in any target and
228 + use the quantization scheme corresponding to the matched target
229 + to select the CompressedTensorsScheme used for infernece.
230 + """
231 +
232 + # Check if the layer is skipped for quantization.
233 + # TODO (@robertgshaw2): support module names
234 + if should_ignore_layer(layer_name, ignore=self.ignore):
235 + return CompressedTensorsUnquantized()
236 +
237 + # Find the "target" in the compressed-tensors config
238 + # that our layer conforms to.
239 + # TODO (@robertgshaw): add compressed-tensors as dep
240 + # so we do not have to re-write these functions
241 + matched_target = find_matched_target(
242 + layer_name=layer_name,
243 + module=layer,
244 + targets=self.target_scheme_map.keys())
245
219 - layer_quant_details: Dict[str, Any] = self.layer_quant_details.get(
220 - layer_type_name, None)
221 - if layer_quant_details is None:
222 - raise ValueError(
223 - f"Could not find quantization details for {layer}.")
246 + # Find the quant_scheme
247 + scheme = self.target_scheme_map[matched_target]
248
225 - scheme = self._get_schema(
226 - weight_quant=layer_quant_details["weights"],
227 - input_quant=layer_quant_details["input_activations"])
249 + return self._get_scheme_from_parts(
250 + weight_quant=scheme["weights"],
251 + input_quant=scheme["input_activations"])
252
253 # Raise error if device does not support the scheme
254 # (e.g. fp8 needs ada lovelace)
@@ -250,11 +274,11 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
274 Use the CompressedTensorsScheme associated with each layer to create
275 the necessary parameters for the layer. See LinearMethodBase for param
276 details
253 -
277 """
278 weight_loader = extra_weight_attrs.get("weight_loader")
279 + layer_name = extra_weight_attrs.get("prefix")
280
257 - scheme = self.quantization_config.get_scheme(layer=layer)
281 + scheme = self.quantization_config.get_scheme(layer, layer_name)
282 scheme.create_weights(
283 layer=layer,
284 input_size=input_size,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
-1
@@ -33,7 +33,6 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
33
34 weight = Parameter(torch.empty(sum(output_partition_sizes),
35 input_size_per_partition,
36 - device="cuda",
36 dtype=params_dtype),
37 requires_grad=False)
38
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+118 -21
@@ -86,25 +86,106 @@ def is_activation_quantization_format(format: str) -> bool:
86 return format in _ACTIVATION_QUANTIZATION_FORMATS
87
88
89 -def find_first_name_or_class_match(
90 - name: str,
91 - module: Module,
92 - targets: Iterable[str],
93 - check_contains: bool = False) -> Optional[str]:
89 +# fused_name: List[shard_name]
90 +_FUSED_LAYER_NAME_MAPPING = {
91 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
92 + "gate_up_proj": ["gate_proj", "up_proj"]
93 +}
94 +
95 +
96 +def should_ignore_layer(layer_name: Optional[str],
97 + ignore: Iterable[str]) -> bool:
98 + if layer_name is None:
99 + return False
100 +
101 + # layer_name = model.layers.0.self_attn.qkv_proj
102 + # proj_name = qkv_proj
103 + proj_name = layer_name.split(".")[-1]
104 +
105 + # Fused layers like gate_up_proj or qkv_proj will not be fused
106 + # in the safetensors checkpoint. So, we convert the name
107 + # from the fused version to unfused + check to make sure that
108 + # each shard of the fused layer has the same scheme.
109 + if proj_name in _FUSED_LAYER_NAME_MAPPING:
110 + shard_proj_names = _FUSED_LAYER_NAME_MAPPING[proj_name]
111 +
112 + # Convert fused_name --> [shard_names]
113 + shard_names = [
114 + layer_name.replace(proj_name, shard_proj_name)
115 + for shard_proj_name in shard_proj_names
116 + ]
117 +
118 + # Layer should be ignored if shards are ignored.
119 + should_ignore_layer = None
120 + for shard_name in shard_names:
121 + should_ignore_shard = check_equal_or_regex_match(
122 + layer_name=shard_name, targets=ignore)
123 +
124 + # If shard_idx=0, set layer ignore to match shard.
125 + if should_ignore_layer is None:
126 + should_ignore_layer = should_ignore_shard
127 +
128 + # If shard_idx=1+ confirm scheme matches prior shards.
129 + elif should_ignore_shard != should_ignore_layer:
130 + raise ValueError(f"Found a different quantization schemes for "
131 + f"{shard_proj_names} in {layer_name}. vLLM "
132 + "requires all to use the same scheme.")
133 +
134 + # Unfused layers like down_proj and o_proj will match
135 + # the safetensors checkpoint already.
136 + else:
137 + should_ignore_layer = check_equal_or_regex_match(layer_name=layer_name,
138 + targets=ignore)
139 +
140 + assert should_ignore_layer is not None
141 + return should_ignore_layer
142 +
143 +
144 +def check_equal_or_regex_match(layer_name: str,
145 + targets: Iterable[str]) -> bool:
146 """
95 - Helper function to map the quantization details listed in the config
96 - for a given list of targets against each model layer. First uses the
97 - layer name to try and find a match. If no name match is found, uses
98 - the layer class name. Returns None otherwise.
147 + Checks whether a layer_name is exactly equal or a regex match for
148 + if target starts with 're:' to any target in list.
149 + """
150 + for target in targets:
151 + if _is_equal_or_regex_match(layer_name, target):
152 + return True
153 + return False
154 +
155 +
156 +def find_matched_target(layer_name: Optional[str], module: Module,
157 + targets: Iterable[str]) -> str:
158 + """
159 + Helper function to look up which "target" in the compressed-tensors
160 + config that a layer corresponds to.
161
100 - :param name: layer name
162 + Recall that a compressed-tensors configs has a concept of
163 + config_groups, where each layer can be quantized with with a different
164 + scheme.
165 +
166 + targets in each config_group will be a list of either layer names
167 + (or regexes corresponding to layer names) or names of torch Modules.
168 +
169 + First, we try to match the layer_name with a target
170 + Second, we try to match the module's name with a target
171 +
172 + :param layer_name: layer name
173 :param module: torch.nn.Module
174 :param targets: list of targets to match the layer against
103 - :param check_contains: whether or not to do a substring match
175 """
176
106 - return _find_first_match(name, targets) or _find_first_match(
107 - module.__class__.__name__, targets, check_contains)
177 + if layer_name is None:
178 + layer_name = ""
179 +
180 + matched_target = (_find_first_match(layer_name, targets)
181 + or _find_first_match(module.__class__.__name__, targets,
182 + True))
183 +
184 + if matched_target is None:
185 + raise ValueError(f"Unable to find matching target for {module} in the "
186 + "compressed-tensors config.")
187 +
188 + return matched_target
189
190
191 def _find_first_match(value: str,
@@ -121,13 +202,29 @@ def _find_first_match(value: str,
202 """
203
204 for target in targets:
124 - if target.startswith("re:"):
125 - pattern = target[3:]
126 - if re.match(pattern, value):
127 - return target
128 - elif check_contains:
129 - if target.lower() in value.lower():
130 - return target
131 - elif target == value:
205 + if _is_equal_or_regex_match(value,
206 + target,
207 + check_contains=check_contains):
208 return target
209 return None
210 +
211 +
212 +def _is_equal_or_regex_match(value: str,
213 + target: str,
214 + check_contains: bool = False) -> bool:
215 + """
216 + Checks whether a value is exactly equal or a regex match for target
217 + if target starts with 're:'. If check_contains is set to True,
218 + additionally checks if the target string is contained within the value.
219 + """
220 +
221 + if target.startswith("re:"):
222 + pattern = target[3:]
223 + if re.match(pattern, value):
224 + return True
225 + elif check_contains:
226 + if target.lower() in value.lower():
227 + return True
228 + elif target == value:
229 + return True
230 + return False
vllm/model_executor/models/gpt2.py
+23 -4
@@ -51,6 +51,7 @@ class GPT2Attention(nn.Module):
51 config: GPT2Config,
52 cache_config: Optional[CacheConfig] = None,
53 quant_config: Optional[QuantizationConfig] = None,
54 + prefix: str = "",
55 ):
56 super().__init__()
57 self.hidden_size = config.hidden_size
@@ -68,12 +69,14 @@ class GPT2Attention(nn.Module):
69 total_num_heads,
70 bias=True,
71 quant_config=quant_config,
72 + prefix=f"{prefix}.c_attn",
73 )
74 self.c_proj = RowParallelLinear(
75 self.hidden_size,
76 self.hidden_size,
77 bias=True,
78 quant_config=quant_config,
79 + prefix=f"{prefix}.c_proj",
80 )
81 self.attn = Attention(self.num_heads,
82 self.head_dim,
@@ -101,6 +104,7 @@ class GPT2MLP(nn.Module):
104 intermediate_size: int,
105 config: GPT2Config,
106 quant_config: Optional[QuantizationConfig] = None,
107 + prefix: str = "",
108 ):
109 super().__init__()
110 hidden_size = config.hidden_size
@@ -109,12 +113,14 @@ class GPT2MLP(nn.Module):
113 intermediate_size,
114 bias=True,
115 quant_config=quant_config,
116 + prefix=f"{prefix}.c_fc",
117 )
118 self.c_proj = RowParallelLinear(
119 intermediate_size,
120 hidden_size,
121 bias=True,
122 quant_config=quant_config,
123 + prefix=f"{prefix}.c_proj",
124 )
125 self.act = get_act_fn(config.activation_function, quant_config,
126 intermediate_size)
@@ -133,6 +139,7 @@ class GPT2Block(nn.Module):
139 config: GPT2Config,
140 cache_config: Optional[CacheConfig] = None,
141 quant_config: Optional[QuantizationConfig] = None,
142 + prefix: str = "",
143 ):
144 super().__init__()
145 hidden_size = config.hidden_size
@@ -140,9 +147,15 @@ class GPT2Block(nn.Module):
147 hidden_size)
148
149 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
143 - self.attn = GPT2Attention(config, cache_config, quant_config)
150 + self.attn = GPT2Attention(config,
151 + cache_config,
152 + quant_config,
153 + prefix=f"{prefix}.attn")
154 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
145 - self.mlp = GPT2MLP(inner_dim, config, quant_config)
155 + self.mlp = GPT2MLP(inner_dim,
156 + config,
157 + quant_config,
158 + prefix=f"{prefix}.mlp")
159
160 def forward(
161 self,
@@ -175,6 +188,7 @@ class GPT2Model(nn.Module):
188 config: GPT2Config,
189 cache_config: Optional[CacheConfig] = None,
190 quant_config: Optional[QuantizationConfig] = None,
191 + prefix: str = "",
192 ):
193 super().__init__()
194 self.config = config
@@ -186,7 +200,9 @@ class GPT2Model(nn.Module):
200 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
201 self.start_layer, self.end_layer, self.h = make_layers(
202 config.num_hidden_layers,
189 - lambda: GPT2Block(config, cache_config, quant_config))
203 + lambda prefix: GPT2Block(
204 + config, cache_config, quant_config, prefix=prefix),
205 + prefix=f"{prefix}.h")
206 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
207
208 def forward(
@@ -229,7 +245,10 @@ class GPT2LMHeadModel(nn.Module):
245 super().__init__()
246 self.config = config
247 self.quant_config = quant_config
232 - self.transformer = GPT2Model(config, cache_config, quant_config)
248 + self.transformer = GPT2Model(config,
249 + cache_config,
250 + quant_config,
251 + prefix="transformer")
252 self.lm_head = self.transformer.wte
253 self.logits_processor = LogitsProcessor(config.vocab_size)
254 self.sampler = Sampler()
vllm/model_executor/models/llama.py
+19 -6
@@ -62,17 +62,20 @@ class LlamaMLP(nn.Module):
62 hidden_act: str,
63 quant_config: Optional[QuantizationConfig] = None,
64 bias: bool = False,
65 + prefix: str = "",
66 ) -> None:
67 super().__init__()
68 self.gate_up_proj = MergedColumnParallelLinear(
69 input_size=hidden_size,
70 output_sizes=[intermediate_size] * 2,
71 bias=bias,
71 - quant_config=quant_config)
72 + quant_config=quant_config,
73 + prefix=f"{prefix}.gate_up_proj")
74 self.down_proj = RowParallelLinear(input_size=intermediate_size,
75 output_size=hidden_size,
76 bias=bias,
75 - quant_config=quant_config)
77 + quant_config=quant_config,
78 + prefix=f"{prefix}.down_proj")
79 if hidden_act != "silu":
80 raise ValueError(f"Unsupported activation: {hidden_act}. "
81 "Only silu is supported for now.")
@@ -99,6 +102,7 @@ class LlamaAttention(nn.Module):
102 quant_config: Optional[QuantizationConfig] = None,
103 bias: bool = False,
104 cache_config: Optional[CacheConfig] = None,
105 + prefix: str = "",
106 ) -> None:
107 super().__init__()
108 self.hidden_size = hidden_size
@@ -132,12 +136,14 @@ class LlamaAttention(nn.Module):
136 total_num_kv_heads=self.total_num_kv_heads,
137 bias=bias,
138 quant_config=quant_config,
139 + prefix=f"{prefix}.qkv_proj",
140 )
141 self.o_proj = RowParallelLinear(
142 input_size=self.total_num_heads * self.head_dim,
143 output_size=hidden_size,
144 bias=bias,
145 quant_config=quant_config,
146 + prefix=f"{prefix}.o_proj",
147 )
148
149 self.rotary_emb = get_rope(
@@ -176,6 +182,7 @@ class LlamaDecoderLayer(nn.Module):
182 config: LlamaConfig,
183 cache_config: Optional[CacheConfig] = None,
184 quant_config: Optional[QuantizationConfig] = None,
185 + prefix: str = "",
186 ) -> None:
187 super().__init__()
188 self.hidden_size = config.hidden_size
@@ -203,6 +210,7 @@ class LlamaDecoderLayer(nn.Module):
210 quant_config=quant_config,
211 bias=attention_bias,
212 cache_config=cache_config,
213 + prefix=f"{prefix}.self_attn",
214 )
215 self.mlp = LlamaMLP(
216 hidden_size=self.hidden_size,
@@ -210,6 +218,7 @@ class LlamaDecoderLayer(nn.Module):
218 hidden_act=config.hidden_act,
219 quant_config=quant_config,
220 bias=getattr(config, "mlp_bias", False),
221 + prefix=f"{prefix}.mlp",
222 )
223 self.input_layernorm = RMSNorm(config.hidden_size,
224 eps=config.rms_norm_eps)
@@ -253,6 +262,7 @@ class LlamaModel(nn.Module):
262 cache_config: Optional[CacheConfig] = None,
263 quant_config: Optional[QuantizationConfig] = None,
264 lora_config: Optional[LoRAConfig] = None,
265 + prefix: str = "",
266 ) -> None:
267 super().__init__()
268 self.config = config
@@ -272,9 +282,11 @@ class LlamaModel(nn.Module):
282 self.embed_tokens = PPMissingLayer()
283 self.start_layer, self.end_layer, self.layers = make_layers(
284 config.num_hidden_layers,
275 - lambda: LlamaDecoderLayer(config=config,
276 - cache_config=cache_config,
277 - quant_config=quant_config))
285 + lambda prefix: LlamaDecoderLayer(config=config,
286 + cache_config=cache_config,
287 + quant_config=quant_config,
288 + prefix=prefix),
289 + prefix=f"{prefix}.layers")
290 if get_pp_group().is_last_rank:
291 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
292 else:
@@ -370,7 +382,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
382 self.model = LlamaModel(config,
383 cache_config,
384 quant_config,
373 - lora_config=lora_config)
385 + lora_config=lora_config,
386 + prefix="model")
387 if get_pp_group().is_last_rank:
388 self.unpadded_vocab_size = config.vocab_size
389 if lora_config:
vllm/model_executor/models/mixtral.py
+22 -8
@@ -67,7 +67,8 @@ class MixtralMoE(nn.Module):
67 intermediate_size: int,
68 params_dtype: Optional[torch.dtype] = None,
69 quant_config: Optional[QuantizationConfig] = None,
70 - tp_size: Optional[int] = None):
70 + tp_size: Optional[int] = None,
71 + prefix: str = ""):
72 super().__init__()
73 self.hidden_size = hidden_size
74
@@ -76,7 +77,8 @@ class MixtralMoE(nn.Module):
77 num_experts,
78 bias=False,
79 params_dtype=params_dtype,
79 - quant_config=None)
80 + quant_config=None,
81 + prefix=f"{prefix}.gate")
82
83 self.experts = FusedMoE(num_experts=num_experts,
84 top_k=top_k,
@@ -86,7 +88,8 @@ class MixtralMoE(nn.Module):
88 reduce_results=True,
89 renormalize=True,
90 quant_config=quant_config,
89 - tp_size=tp_size)
91 + tp_size=tp_size,
92 + prefix=f"{prefix}.experts")
93
94 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
95 # NOTE: hidden_states can have either 1D or 2D shape.
@@ -109,6 +112,7 @@ class MixtralAttention(nn.Module):
112 rope_theta: float = 10000,
113 cache_config: Optional[CacheConfig] = None,
114 quant_config: Optional[QuantizationConfig] = None,
115 + prefix: str = "",
116 ) -> None:
117 super().__init__()
118 self.hidden_size = hidden_size
@@ -139,12 +143,14 @@ class MixtralAttention(nn.Module):
143 self.total_num_kv_heads,
144 bias=False,
145 quant_config=quant_config,
146 + prefix=f"{prefix}.qkv_proj",
147 )
148 self.o_proj = RowParallelLinear(
149 self.total_num_heads * self.head_dim,
150 hidden_size,
151 bias=False,
152 quant_config=quant_config,
153 + prefix=f"{prefix}.o_proj",
154 )
155 self.rotary_emb = get_rope(
156 self.head_dim,
@@ -182,6 +188,7 @@ class MixtralDecoderLayer(nn.Module):
188 config: MixtralConfig,
189 cache_config: Optional[CacheConfig] = None,
190 quant_config: Optional[QuantizationConfig] = None,
191 + prefix: str = "",
192 ) -> None:
193 super().__init__()
194 self.hidden_size = config.hidden_size
@@ -194,13 +201,15 @@ class MixtralDecoderLayer(nn.Module):
201 num_kv_heads=config.num_key_value_heads,
202 rope_theta=rope_theta,
203 cache_config=cache_config,
197 - quant_config=quant_config)
204 + quant_config=quant_config,
205 + prefix=f"{prefix}.self_attn")
206 self.block_sparse_moe = MixtralMoE(
207 num_experts=config.num_local_experts,
208 top_k=config.num_experts_per_tok,
209 hidden_size=config.hidden_size,
210 intermediate_size=config.intermediate_size,
203 - quant_config=quant_config)
211 + quant_config=quant_config,
212 + prefix=f"{prefix}.block_sparse_moe")
213 self.input_layernorm = RMSNorm(config.hidden_size,
214 eps=config.rms_norm_eps)
215 self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -243,6 +252,7 @@ class MixtralModel(nn.Module):
252 cache_config: Optional[CacheConfig] = None,
253 quant_config: Optional[QuantizationConfig] = None,
254 lora_config: Optional[LoRAConfig] = None,
255 + prefix: str = "",
256 ) -> None:
257 super().__init__()
258 self.padding_idx = config.pad_token_id
@@ -258,8 +268,11 @@ class MixtralModel(nn.Module):
268 )
269
270 self.start_layer, self.end_layer, self.layers = make_layers(
261 - config.num_hidden_layers, lambda: MixtralDecoderLayer(
262 - config, cache_config, quant_config=quant_config))
271 + config.num_hidden_layers,
272 + lambda prefix: MixtralDecoderLayer(
273 + config, cache_config, quant_config=quant_config, prefix=prefix
274 + ),
275 + prefix=f"{prefix}.layers")
276
277 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
278
@@ -331,7 +344,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
344 self.model = MixtralModel(config,
345 cache_config,
346 quant_config,
334 - lora_config=lora_config)
347 + lora_config=lora_config,
348 + prefix="model")
349 self.unpadded_vocab_size = config.vocab_size
350 if lora_config:
351 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/utils.py
+15 -4
@@ -1,4 +1,4 @@
1 -from typing import Callable, Dict, List, Tuple
1 +from typing import Dict, List, Protocol, Tuple
2
3 import torch
4 from torch.func import functional_call
@@ -45,6 +45,15 @@ def merge_vision_embeddings(input_ids: torch.Tensor,
45 return inputs_embeds
46
47
48 +class LayerFn(Protocol):
49 +
50 + def __call__(
51 + self,
52 + prefix="",
53 + ) -> torch.nn.Module:
54 + ...
55 +
56 +
57 class PPMissingLayer(torch.nn.Identity):
58 """
59 A placeholder layer for missing layers in a pipeline parallel model.
@@ -119,7 +128,9 @@ def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module:
128
129
130 def make_layers(
122 - num_hidden_layers: int, layer_fn: Callable[[], torch.nn.Module]
131 + num_hidden_layers: int,
132 + layer_fn: LayerFn,
133 + prefix: str,
134 ) -> Tuple[int, int, torch.nn.ModuleList]:
135 """Make a list of layers with the given layer function, taking
136 pipeline parallelism into account.
@@ -131,8 +142,8 @@ def make_layers(
142 get_pp_group().world_size)
143 modules = torch.nn.ModuleList(
144 [PPMissingLayer() for _ in range(start_layer)] + [
134 - maybe_offload_to_cpu(layer_fn())
135 - for _ in range(start_layer, end_layer)
145 + maybe_offload_to_cpu(layer_fn(prefix=f"{prefix}.{idx}"))
146 + for idx in range(start_layer, end_layer)
147 ] + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
148 return start_layer, end_layer, modules
149