A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[model][utils] add extract_layer_index utility function (#10599)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 23, 2024 at 22:22 UTC
c055747867e771dbc791c9aa3c394c4d4489cd82
6 files changed
+59
-51
vllm/model_executor/models/arctic.py
+18
-23
@@ -33,7 +33,7 @@ from vllm.sequence import IntermediateTensors
33
from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35
from .interfaces import SupportsPP
36
-from .utils import (is_pp_missing_parameter,
36
+from .utils import (extract_layer_index, is_pp_missing_parameter,
37
make_empty_intermediate_tensors_factory, make_layers,
38
maybe_prefix)
39
@@ -44,15 +44,14 @@ class ArcticMLP(nn.Module):
44
45
def __init__(self,
46
config: ArcticConfig,
47
- layer_id: int,
47
expert_id: int = -1,
48
is_residual_mlp: bool = False,
49
quant_config: Optional[QuantizationConfig] = None,
51
- reduce_results: bool = True):
50
+ reduce_results: bool = True,
51
+ prefix: str = ""):
52
super().__init__()
53
self.hidden_size = config.hidden_size
54
self.expert_id = expert_id
55
- self.layer_id = layer_id
55
56
self.ffn_dim = config.intermediate_size if not is_residual_mlp \
57
else self.hidden_size
@@ -85,13 +84,14 @@ class ArcticMoE(nn.Module):
84
85
def __init__(self,
86
config: ArcticConfig,
88
- layer_id: int,
87
tp_size: Optional[int] = None,
88
params_dtype: Optional[torch.dtype] = None,
89
quant_config: Optional[QuantizationConfig] = None,
92
- reduce_results: bool = True):
90
+ reduce_results: bool = True,
91
+ prefix: str = ""):
92
super().__init__()
93
94
+ layer_id = extract_layer_index(prefix)
95
self.tp_size = tp_size or get_tensor_model_parallel_world_size()
96
self.hidden_size = config.hidden_size
97
self.num_experts = config.num_local_experts
@@ -109,15 +109,16 @@ class ArcticMoE(nn.Module):
109
110
if not self.is_moe_layer:
111
self.mlp = ArcticMLP(config,
112
- layer_id=layer_id,
112
quant_config=quant_config,
114
- reduce_results=reduce_results)
113
+ reduce_results=reduce_results,
114
+ prefix=f"{prefix}.mlp")
115
else:
116
self.gate = ReplicatedLinear(self.hidden_size,
117
self.num_experts,
118
bias=False,
119
params_dtype=self.params_dtype,
120
- quant_config=quant_config)
120
+ quant_config=quant_config,
121
+ prefix=f"{prefix}.gate")
122
if self.is_quant:
123
self.ws = DeepSpeedFPParameter(
124
torch.Size((self.num_experts, 2 * self.intermediate_size,
@@ -220,14 +221,12 @@ class ArcticAttention(nn.Module):
221
def __init__(
222
self,
223
config: ArcticConfig,
223
- layer_idx: Optional[int] = None,
224
cache_config: Optional[CacheConfig] = None,
225
quant_config: Optional[QuantizationConfig] = None,
226
prefix: str = "",
227
):
228
super().__init__()
229
self.config = config
230
- self.layer_idx = layer_idx
230
self.hidden_size = config.hidden_size
231
232
tp_size = get_tensor_model_parallel_world_size()
@@ -298,26 +297,25 @@ class ArcticDecoderLayer(nn.Module):
297
def __init__(
298
self,
299
config: ArcticConfig,
301
- layer_idx: int,
300
cache_config: Optional[CacheConfig] = None,
301
quant_config: Optional[QuantizationConfig] = None,
302
prefix: str = "",
303
) -> None:
304
super().__init__()
307
- self.layer_idx = layer_idx
305
self.hidden_size = config.hidden_size
306
+ layer_idx = extract_layer_index(prefix)
307
is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
308
self.use_residual = config.use_residual and is_moe_layer
309
self.self_attn = ArcticAttention(config,
312
- layer_idx,
310
cache_config,
311
quant_config=quant_config,
312
prefix=f"{prefix}.self_attn")
313
self.block_sparse_moe = ArcticMoE(
314
config,
318
- layer_id=layer_idx,
315
quant_config=quant_config,
320
- reduce_results=(not self.use_residual))
316
+ reduce_results=(not self.use_residual),
317
+ prefix=f"{prefix}.block_sparse_moe",
318
+ )
319
320
self.input_layernorm = RMSNorm(config.hidden_size,
321
eps=config.rms_norm_eps)
@@ -328,9 +326,9 @@ class ArcticDecoderLayer(nn.Module):
326
self.residual_layernorm = RMSNorm(config.hidden_size,
327
eps=config.rms_norm_eps)
328
self.residual_mlp = ArcticMLP(config,
331
- layer_id=layer_idx,
329
is_residual_mlp=True,
333
- reduce_results=False)
330
+ reduce_results=False,
331
+ prefix=f"{prefix}.residual_mlp")
332
333
def forward(
334
self,
@@ -384,11 +382,8 @@ class ArcticModel(nn.Module):
382
org_num_embeddings=self.vocab_size)
383
self.start_layer, self.end_layer, self.layers = make_layers(
384
config.num_hidden_layers,
387
- lambda prefix: ArcticDecoderLayer(config,
388
- int(prefix.split(".")[-1]),
389
- cache_config,
390
- quant_config,
391
- prefix=prefix),
385
+ lambda prefix: ArcticDecoderLayer(
386
+ config, cache_config, quant_config, prefix=prefix),
387
prefix=f"{prefix}.layers")
388
self._attn_implementation = config._attn_implementation
389
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/deepseek.py
+11
-8
@@ -49,7 +49,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import IntermediateTensors
50
51
from .interfaces import SupportsPP
52
-from .utils import (is_pp_missing_parameter,
52
+from .utils import (extract_layer_index, is_pp_missing_parameter,
53
make_empty_intermediate_tensors_factory, make_layers,
54
maybe_prefix)
55
@@ -63,6 +63,7 @@ class DeepseekMLP(nn.Module):
63
hidden_act: str,
64
quant_config: Optional[QuantizationConfig] = None,
65
reduce_results: bool = True,
66
+ prefix: str = "",
67
) -> None:
68
super().__init__()
69
self.gate_up_proj = MergedColumnParallelLinear(
@@ -92,6 +93,7 @@ class DeepseekMoE(nn.Module):
93
self,
94
config: PretrainedConfig,
95
quant_config: Optional[QuantizationConfig] = None,
96
+ prefix: str = "",
97
):
98
super().__init__()
99
self.config = config
@@ -260,12 +262,12 @@ class DeepseekDecoderLayer(nn.Module):
262
def __init__(
263
self,
264
config: PretrainedConfig,
263
- layer_idx: int,
265
cache_config: Optional[CacheConfig] = None,
266
quant_config: Optional[QuantizationConfig] = None,
267
prefix: str = "",
268
) -> None:
269
super().__init__()
270
+ layer_idx = extract_layer_index(prefix)
271
self.hidden_size = config.hidden_size
272
rope_theta = getattr(config, "rope_theta", 10000)
273
rope_scaling = getattr(config, "rope_scaling", None)
@@ -285,13 +287,16 @@ class DeepseekDecoderLayer(nn.Module):
287
if (config.n_routed_experts is not None
288
and layer_idx >= config.first_k_dense_replace
289
and layer_idx % config.moe_layer_freq == 0):
288
- self.mlp = DeepseekMoE(config=config, quant_config=quant_config)
290
+ self.mlp = DeepseekMoE(config=config,
291
+ quant_config=quant_config,
292
+ prefix=f"{prefix}.mlp")
293
else:
294
self.mlp = DeepseekMLP(
295
hidden_size=config.hidden_size,
296
intermediate_size=config.intermediate_size,
297
hidden_act=config.hidden_act,
298
quant_config=quant_config,
299
+ prefix=f"{prefix}.mlp",
300
)
301
self.input_layernorm = RMSNorm(config.hidden_size,
302
eps=config.rms_norm_eps)
@@ -347,11 +352,9 @@ class DeepseekModel(nn.Module):
352
)
353
self.start_layer, self.end_layer, self.layers = make_layers(
354
config.num_hidden_layers,
350
- lambda prefix: DeepseekDecoderLayer(config,
351
- int(prefix.split(".")[-1]),
352
- cache_config,
353
- quant_config=quant_config,
354
- prefix=prefix),
355
+ lambda prefix: DeepseekDecoderLayer(
356
+ config, cache_config, quant_config=quant_config, prefix=prefix
357
+ ),
358
prefix=f"{prefix}.layers")
359
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
360
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/gemma2.py
+5
-10
@@ -42,7 +42,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import IntermediateTensors, PoolerOutput
43
44
from .interfaces import SupportsLoRA, SupportsPP
45
-from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
45
+from .utils import (AutoWeightsLoader, extract_layer_index,
46
+ is_pp_missing_parameter,
47
make_empty_intermediate_tensors_factory, make_layers,
48
maybe_prefix)
49
@@ -85,7 +86,6 @@ class Gemma2MLP(nn.Module):
86
class Gemma2Attention(nn.Module):
87
88
def __init__(self,
88
- layer_idx: int,
89
config: Gemma2Config,
90
hidden_size: int,
91
num_heads: int,
@@ -98,7 +98,6 @@ class Gemma2Attention(nn.Module):
98
attn_logits_soft_cap: Optional[float] = None,
99
prefix: str = "") -> None:
100
super().__init__()
101
- self.layer_idx = layer_idx
101
self.config = config
102
self.hidden_size = hidden_size
103
tp_size = get_tensor_model_parallel_world_size()
@@ -145,6 +144,7 @@ class Gemma2Attention(nn.Module):
144
145
# reference:
146
# https://github.com/huggingface/transformers/blob/54be2d7ae87e873482b984cc956e165ca4dc0ba3/src/transformers/models/gemma2/modeling_gemma2.py#L312 # noqa
147
+ layer_idx = extract_layer_index(prefix)
148
use_sliding_window = (layer_idx % 2 == 0 and
149
config.interleaved_sliding_window is not None)
150
sliding_window = config.interleaved_sliding_window if \
@@ -178,7 +178,6 @@ class Gemma2DecoderLayer(nn.Module):
178
179
def __init__(
180
self,
181
- layer_idx: int,
181
config: Gemma2Config,
182
cache_config: Optional[CacheConfig] = None,
183
quant_config: Optional[QuantizationConfig] = None,
@@ -187,7 +186,6 @@ class Gemma2DecoderLayer(nn.Module):
186
super().__init__()
187
self.hidden_size = config.hidden_size
188
self.self_attn = Gemma2Attention(
190
- layer_idx=layer_idx,
189
config=config,
190
hidden_size=self.hidden_size,
191
num_heads=config.num_attention_heads,
@@ -262,11 +260,8 @@ class Gemma2Model(nn.Module):
260
)
261
self.start_layer, self.end_layer, self.layers = make_layers(
262
config.num_hidden_layers,
265
- lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[-1]),
266
- config,
267
- cache_config,
268
- quant_config,
269
- prefix=prefix),
263
+ lambda prefix: Gemma2DecoderLayer(
264
+ config, cache_config, quant_config, prefix=prefix),
265
prefix=f"{prefix}.layers")
266
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
267
vllm/model_executor/models/olmoe.py
+2
-6
@@ -181,7 +181,6 @@ class OlmoeDecoderLayer(nn.Module):
181
def __init__(
182
self,
183
config: PretrainedConfig,
184
- layer_idx: int,
184
cache_config: Optional[CacheConfig] = None,
185
quant_config: Optional[QuantizationConfig] = None,
186
prefix: str = "",
@@ -264,11 +263,8 @@ class OlmoeModel(nn.Module):
263
)
264
self.start_layer, self.end_layer, self.layers = make_layers(
265
config.num_hidden_layers,
267
- lambda prefix: OlmoeDecoderLayer(config,
268
- int(prefix.split(".")[-1]),
269
- cache_config,
270
- quant_config,
271
- prefix=prefix),
266
+ lambda prefix: OlmoeDecoderLayer(
267
+ config, cache_config, quant_config, prefix=prefix),
268
prefix=f"{prefix}.layers")
269
self.norm = RMSNorm(config.hidden_size, eps=1e-5)
270
vllm/model_executor/models/qwen2_moe.py
+2
-4
@@ -53,7 +53,7 @@ from vllm.sequence import IntermediateTensors
53
from vllm.utils import print_warning_once
54
55
from .interfaces import SupportsPP
56
-from .utils import (is_pp_missing_parameter,
56
+from .utils import (extract_layer_index, is_pp_missing_parameter,
57
make_empty_intermediate_tensors_factory, make_layers,
58
maybe_prefix)
59
@@ -244,7 +244,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
244
def __init__(
245
self,
246
config: PretrainedConfig,
247
- layer_idx: int,
247
cache_config: Optional[CacheConfig] = None,
248
quant_config: Optional[QuantizationConfig] = None,
249
prefix: str = "",
@@ -269,6 +268,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
268
269
# Note: Qwen/Qwen2-57B-A14B-Instruct does not have
270
# `mlp_only_layers` in the config.
271
+ layer_idx = extract_layer_index(prefix)
272
mlp_only_layers = ([] if not hasattr(config, "mlp_only_layers") else
273
config.mlp_only_layers)
274
if (layer_idx not in mlp_only_layers) and (
@@ -337,8 +337,6 @@ class Qwen2MoeModel(nn.Module):
337
self.start_layer, self.end_layer, self.layers = make_layers(
338
config.num_hidden_layers,
339
lambda prefix: Qwen2MoeDecoderLayer(config=config,
340
- layer_idx=int(
341
- prefix.split(".")[-1]),
340
cache_config=cache_config,
341
quant_config=quant_config,
342
prefix=prefix),
vllm/model_executor/models/utils.py
+21
@@ -629,3 +629,24 @@ def maybe_prefix(prefix: str, name: str) -> str:
629
The string "prefix.name" if prefix was non-empty, otherwise just "name".
630
"""
631
return name if not prefix else f"{prefix}.{name}"
632
+
633
+
634
+def extract_layer_index(layer_name: str) -> int:
635
+ """
636
+ Extract the layer index from the module name.
637
+ Examples:
638
+ - "encoder.layers.0" -> 0
639
+ - "encoder.layers.1.self_attn" -> 1
640
+ - "2.self_attn" -> 2
641
+ - "model.encoder.layers.0.sub.1" -> ValueError
642
+ """
643
+ subnames = layer_name.split(".")
644
+ int_vals: List[int] = []
645
+ for subname in subnames:
646
+ try:
647
+ int_vals.append(int(subname))
648
+ except ValueError:
649
+ continue
650
+ assert len(int_vals) == 1, (f"layer name {layer_name} should"
651
+ " only contain one integer")
652
+ return int_vals[0]