@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c055747867
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[model][utils] add extract_layer_index utility function (#10599)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 23, 2024 at 22:22 UTC c055747867e771dbc791c9aa3c394c4d4489cd82
6 files changed +59 -51
vllm/model_executor/models/arctic.py
+18 -23
@@ -33,7 +33,7 @@ from vllm.sequence import IntermediateTensors
33 from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35 from .interfaces import SupportsPP
36 -from .utils import (is_pp_missing_parameter,
36 +from .utils import (extract_layer_index, is_pp_missing_parameter,
37 make_empty_intermediate_tensors_factory, make_layers,
38 maybe_prefix)
39
@@ -44,15 +44,14 @@ class ArcticMLP(nn.Module):
44
45 def __init__(self,
46 config: ArcticConfig,
47 - layer_id: int,
47 expert_id: int = -1,
48 is_residual_mlp: bool = False,
49 quant_config: Optional[QuantizationConfig] = None,
51 - reduce_results: bool = True):
50 + reduce_results: bool = True,
51 + prefix: str = ""):
52 super().__init__()
53 self.hidden_size = config.hidden_size
54 self.expert_id = expert_id
55 - self.layer_id = layer_id
55
56 self.ffn_dim = config.intermediate_size if not is_residual_mlp \
57 else self.hidden_size
@@ -85,13 +84,14 @@ class ArcticMoE(nn.Module):
84
85 def __init__(self,
86 config: ArcticConfig,
88 - layer_id: int,
87 tp_size: Optional[int] = None,
88 params_dtype: Optional[torch.dtype] = None,
89 quant_config: Optional[QuantizationConfig] = None,
92 - reduce_results: bool = True):
90 + reduce_results: bool = True,
91 + prefix: str = ""):
92 super().__init__()
93
94 + layer_id = extract_layer_index(prefix)
95 self.tp_size = tp_size or get_tensor_model_parallel_world_size()
96 self.hidden_size = config.hidden_size
97 self.num_experts = config.num_local_experts
@@ -109,15 +109,16 @@ class ArcticMoE(nn.Module):
109
110 if not self.is_moe_layer:
111 self.mlp = ArcticMLP(config,
112 - layer_id=layer_id,
112 quant_config=quant_config,
114 - reduce_results=reduce_results)
113 + reduce_results=reduce_results,
114 + prefix=f"{prefix}.mlp")
115 else:
116 self.gate = ReplicatedLinear(self.hidden_size,
117 self.num_experts,
118 bias=False,
119 params_dtype=self.params_dtype,
120 - quant_config=quant_config)
120 + quant_config=quant_config,
121 + prefix=f"{prefix}.gate")
122 if self.is_quant:
123 self.ws = DeepSpeedFPParameter(
124 torch.Size((self.num_experts, 2 * self.intermediate_size,
@@ -220,14 +221,12 @@ class ArcticAttention(nn.Module):
221 def __init__(
222 self,
223 config: ArcticConfig,
223 - layer_idx: Optional[int] = None,
224 cache_config: Optional[CacheConfig] = None,
225 quant_config: Optional[QuantizationConfig] = None,
226 prefix: str = "",
227 ):
228 super().__init__()
229 self.config = config
230 - self.layer_idx = layer_idx
230 self.hidden_size = config.hidden_size
231
232 tp_size = get_tensor_model_parallel_world_size()
@@ -298,26 +297,25 @@ class ArcticDecoderLayer(nn.Module):
297 def __init__(
298 self,
299 config: ArcticConfig,
301 - layer_idx: int,
300 cache_config: Optional[CacheConfig] = None,
301 quant_config: Optional[QuantizationConfig] = None,
302 prefix: str = "",
303 ) -> None:
304 super().__init__()
307 - self.layer_idx = layer_idx
305 self.hidden_size = config.hidden_size
306 + layer_idx = extract_layer_index(prefix)
307 is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
308 self.use_residual = config.use_residual and is_moe_layer
309 self.self_attn = ArcticAttention(config,
312 - layer_idx,
310 cache_config,
311 quant_config=quant_config,
312 prefix=f"{prefix}.self_attn")
313 self.block_sparse_moe = ArcticMoE(
314 config,
318 - layer_id=layer_idx,
315 quant_config=quant_config,
320 - reduce_results=(not self.use_residual))
316 + reduce_results=(not self.use_residual),
317 + prefix=f"{prefix}.block_sparse_moe",
318 + )
319
320 self.input_layernorm = RMSNorm(config.hidden_size,
321 eps=config.rms_norm_eps)
@@ -328,9 +326,9 @@ class ArcticDecoderLayer(nn.Module):
326 self.residual_layernorm = RMSNorm(config.hidden_size,
327 eps=config.rms_norm_eps)
328 self.residual_mlp = ArcticMLP(config,
331 - layer_id=layer_idx,
329 is_residual_mlp=True,
333 - reduce_results=False)
330 + reduce_results=False,
331 + prefix=f"{prefix}.residual_mlp")
332
333 def forward(
334 self,
@@ -384,11 +382,8 @@ class ArcticModel(nn.Module):
382 org_num_embeddings=self.vocab_size)
383 self.start_layer, self.end_layer, self.layers = make_layers(
384 config.num_hidden_layers,
387 - lambda prefix: ArcticDecoderLayer(config,
388 - int(prefix.split(".")[-1]),
389 - cache_config,
390 - quant_config,
391 - prefix=prefix),
385 + lambda prefix: ArcticDecoderLayer(
386 + config, cache_config, quant_config, prefix=prefix),
387 prefix=f"{prefix}.layers")
388 self._attn_implementation = config._attn_implementation
389 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/deepseek.py
+11 -8
@@ -49,7 +49,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import IntermediateTensors
50
51 from .interfaces import SupportsPP
52 -from .utils import (is_pp_missing_parameter,
52 +from .utils import (extract_layer_index, is_pp_missing_parameter,
53 make_empty_intermediate_tensors_factory, make_layers,
54 maybe_prefix)
55
@@ -63,6 +63,7 @@ class DeepseekMLP(nn.Module):
63 hidden_act: str,
64 quant_config: Optional[QuantizationConfig] = None,
65 reduce_results: bool = True,
66 + prefix: str = "",
67 ) -> None:
68 super().__init__()
69 self.gate_up_proj = MergedColumnParallelLinear(
@@ -92,6 +93,7 @@ class DeepseekMoE(nn.Module):
93 self,
94 config: PretrainedConfig,
95 quant_config: Optional[QuantizationConfig] = None,
96 + prefix: str = "",
97 ):
98 super().__init__()
99 self.config = config
@@ -260,12 +262,12 @@ class DeepseekDecoderLayer(nn.Module):
262 def __init__(
263 self,
264 config: PretrainedConfig,
263 - layer_idx: int,
265 cache_config: Optional[CacheConfig] = None,
266 quant_config: Optional[QuantizationConfig] = None,
267 prefix: str = "",
268 ) -> None:
269 super().__init__()
270 + layer_idx = extract_layer_index(prefix)
271 self.hidden_size = config.hidden_size
272 rope_theta = getattr(config, "rope_theta", 10000)
273 rope_scaling = getattr(config, "rope_scaling", None)
@@ -285,13 +287,16 @@ class DeepseekDecoderLayer(nn.Module):
287 if (config.n_routed_experts is not None
288 and layer_idx >= config.first_k_dense_replace
289 and layer_idx % config.moe_layer_freq == 0):
288 - self.mlp = DeepseekMoE(config=config, quant_config=quant_config)
290 + self.mlp = DeepseekMoE(config=config,
291 + quant_config=quant_config,
292 + prefix=f"{prefix}.mlp")
293 else:
294 self.mlp = DeepseekMLP(
295 hidden_size=config.hidden_size,
296 intermediate_size=config.intermediate_size,
297 hidden_act=config.hidden_act,
298 quant_config=quant_config,
299 + prefix=f"{prefix}.mlp",
300 )
301 self.input_layernorm = RMSNorm(config.hidden_size,
302 eps=config.rms_norm_eps)
@@ -347,11 +352,9 @@ class DeepseekModel(nn.Module):
352 )
353 self.start_layer, self.end_layer, self.layers = make_layers(
354 config.num_hidden_layers,
350 - lambda prefix: DeepseekDecoderLayer(config,
351 - int(prefix.split(".")[-1]),
352 - cache_config,
353 - quant_config=quant_config,
354 - prefix=prefix),
355 + lambda prefix: DeepseekDecoderLayer(
356 + config, cache_config, quant_config=quant_config, prefix=prefix
357 + ),
358 prefix=f"{prefix}.layers")
359 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
360 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/gemma2.py
+5 -10
@@ -42,7 +42,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, PoolerOutput
43
44 from .interfaces import SupportsLoRA, SupportsPP
45 -from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
45 +from .utils import (AutoWeightsLoader, extract_layer_index,
46 + is_pp_missing_parameter,
47 make_empty_intermediate_tensors_factory, make_layers,
48 maybe_prefix)
49
@@ -85,7 +86,6 @@ class Gemma2MLP(nn.Module):
86 class Gemma2Attention(nn.Module):
87
88 def __init__(self,
88 - layer_idx: int,
89 config: Gemma2Config,
90 hidden_size: int,
91 num_heads: int,
@@ -98,7 +98,6 @@ class Gemma2Attention(nn.Module):
98 attn_logits_soft_cap: Optional[float] = None,
99 prefix: str = "") -> None:
100 super().__init__()
101 - self.layer_idx = layer_idx
101 self.config = config
102 self.hidden_size = hidden_size
103 tp_size = get_tensor_model_parallel_world_size()
@@ -145,6 +144,7 @@ class Gemma2Attention(nn.Module):
144
145 # reference:
146 # https://github.com/huggingface/transformers/blob/54be2d7ae87e873482b984cc956e165ca4dc0ba3/src/transformers/models/gemma2/modeling_gemma2.py#L312 # noqa
147 + layer_idx = extract_layer_index(prefix)
148 use_sliding_window = (layer_idx % 2 == 0 and
149 config.interleaved_sliding_window is not None)
150 sliding_window = config.interleaved_sliding_window if \
@@ -178,7 +178,6 @@ class Gemma2DecoderLayer(nn.Module):
178
179 def __init__(
180 self,
181 - layer_idx: int,
181 config: Gemma2Config,
182 cache_config: Optional[CacheConfig] = None,
183 quant_config: Optional[QuantizationConfig] = None,
@@ -187,7 +186,6 @@ class Gemma2DecoderLayer(nn.Module):
186 super().__init__()
187 self.hidden_size = config.hidden_size
188 self.self_attn = Gemma2Attention(
190 - layer_idx=layer_idx,
189 config=config,
190 hidden_size=self.hidden_size,
191 num_heads=config.num_attention_heads,
@@ -262,11 +260,8 @@ class Gemma2Model(nn.Module):
260 )
261 self.start_layer, self.end_layer, self.layers = make_layers(
262 config.num_hidden_layers,
265 - lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[-1]),
266 - config,
267 - cache_config,
268 - quant_config,
269 - prefix=prefix),
263 + lambda prefix: Gemma2DecoderLayer(
264 + config, cache_config, quant_config, prefix=prefix),
265 prefix=f"{prefix}.layers")
266 self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
267
vllm/model_executor/models/olmoe.py
+2 -6
@@ -181,7 +181,6 @@ class OlmoeDecoderLayer(nn.Module):
181 def __init__(
182 self,
183 config: PretrainedConfig,
184 - layer_idx: int,
184 cache_config: Optional[CacheConfig] = None,
185 quant_config: Optional[QuantizationConfig] = None,
186 prefix: str = "",
@@ -264,11 +263,8 @@ class OlmoeModel(nn.Module):
263 )
264 self.start_layer, self.end_layer, self.layers = make_layers(
265 config.num_hidden_layers,
267 - lambda prefix: OlmoeDecoderLayer(config,
268 - int(prefix.split(".")[-1]),
269 - cache_config,
270 - quant_config,
271 - prefix=prefix),
266 + lambda prefix: OlmoeDecoderLayer(
267 + config, cache_config, quant_config, prefix=prefix),
268 prefix=f"{prefix}.layers")
269 self.norm = RMSNorm(config.hidden_size, eps=1e-5)
270
vllm/model_executor/models/qwen2_moe.py
+2 -4
@@ -53,7 +53,7 @@ from vllm.sequence import IntermediateTensors
53 from vllm.utils import print_warning_once
54
55 from .interfaces import SupportsPP
56 -from .utils import (is_pp_missing_parameter,
56 +from .utils import (extract_layer_index, is_pp_missing_parameter,
57 make_empty_intermediate_tensors_factory, make_layers,
58 maybe_prefix)
59
@@ -244,7 +244,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
244 def __init__(
245 self,
246 config: PretrainedConfig,
247 - layer_idx: int,
247 cache_config: Optional[CacheConfig] = None,
248 quant_config: Optional[QuantizationConfig] = None,
249 prefix: str = "",
@@ -269,6 +268,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
268
269 # Note: Qwen/Qwen2-57B-A14B-Instruct does not have
270 # `mlp_only_layers` in the config.
271 + layer_idx = extract_layer_index(prefix)
272 mlp_only_layers = ([] if not hasattr(config, "mlp_only_layers") else
273 config.mlp_only_layers)
274 if (layer_idx not in mlp_only_layers) and (
@@ -337,8 +337,6 @@ class Qwen2MoeModel(nn.Module):
337 self.start_layer, self.end_layer, self.layers = make_layers(
338 config.num_hidden_layers,
339 lambda prefix: Qwen2MoeDecoderLayer(config=config,
340 - layer_idx=int(
341 - prefix.split(".")[-1]),
340 cache_config=cache_config,
341 quant_config=quant_config,
342 prefix=prefix),
vllm/model_executor/models/utils.py
+21
@@ -629,3 +629,24 @@ def maybe_prefix(prefix: str, name: str) -> str:
629 The string "prefix.name" if prefix was non-empty, otherwise just "name".
630 """
631 return name if not prefix else f"{prefix}.{name}"
632 +
633 +
634 +def extract_layer_index(layer_name: str) -> int:
635 + """
636 + Extract the layer index from the module name.
637 + Examples:
638 + - "encoder.layers.0" -> 0
639 + - "encoder.layers.1.self_attn" -> 1
640 + - "2.self_attn" -> 2
641 + - "model.encoder.layers.0.sub.1" -> ValueError
642 + """
643 + subnames = layer_name.split(".")
644 + int_vals: List[int] = []
645 + for subname in subnames:
646 + try:
647 + int_vals.append(int(subname))
648 + except ValueError:
649 + continue
650 + assert len(int_vals) == 1, (f"layer name {layer_name} should"
651 + " only contain one integer")
652 + return int_vals[0]