A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix prefix strings for quantized VLMs (#9772)
Michael Goin committed
Oct 29, 2024 at 19:02 UTC
bc73e9821cb4f90a88c04e7d550f132d8911266b
20 files changed
+288
-97
vllm/model_executor/model_loader/loader.py
+8
-3
@@ -147,15 +147,20 @@ def _get_model_initialization_kwargs(
147
return extra_kwargs
148
149
150
-def build_model(model_class: Type[nn.Module], hf_config: PretrainedConfig,
150
+def build_model(model_class: Type[nn.Module],
151
+ hf_config: PretrainedConfig,
152
cache_config: Optional[CacheConfig],
152
- quant_config: Optional[QuantizationConfig], *,
153
+ quant_config: Optional[QuantizationConfig],
154
+ *,
155
lora_config: Optional[LoRAConfig],
156
multimodal_config: Optional[MultiModalConfig],
155
- scheduler_config: Optional[SchedulerConfig]) -> nn.Module:
157
+ scheduler_config: Optional[SchedulerConfig],
158
+ prefix: Optional[str] = None) -> nn.Module:
159
extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
160
multimodal_config,
161
scheduler_config)
162
+ if prefix:
163
+ extra_kwargs["prefix"] = prefix
164
165
return model_class(config=hf_config,
166
cache_config=cache_config,
vllm/model_executor/models/blip2.py
+4
-1
@@ -507,7 +507,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
507
)
508
509
self.language_model = init_vllm_registered_model(
510
- config.text_config, cache_config, quant_config)
510
+ config.text_config,
511
+ cache_config,
512
+ quant_config,
513
+ prefix="language_model")
514
515
self.make_empty_intermediate_tensors = (
516
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/gemma.py
+39
-19
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44
from .interfaces import SupportsLoRA, SupportsPP
45
from .utils import (is_pp_missing_parameter,
46
- make_empty_intermediate_tensors_factory, make_layers)
46
+ make_empty_intermediate_tensors_factory, make_layers,
47
+ maybe_prefix)
48
49
logger = init_logger(__name__)
50
@@ -83,16 +84,23 @@ class GemmaMLP(nn.Module):
84
hidden_act: Optional[str] = None,
85
hidden_activation: Optional[str] = None,
86
quant_config: Optional[QuantizationConfig] = None,
87
+ prefix: str = "",
88
) -> None:
89
super().__init__()
90
self.gate_up_proj = MergedColumnParallelLinear(
89
- hidden_size, [intermediate_size] * 2,
91
+ hidden_size,
92
+ [intermediate_size] * 2,
93
bias=False,
91
- quant_config=quant_config)
92
- self.down_proj = RowParallelLinear(intermediate_size,
93
- hidden_size,
94
- bias=False,
95
- quant_config=quant_config)
94
+ quant_config=quant_config,
95
+ prefix=f"{prefix}.gate_up_proj",
96
+ )
97
+ self.down_proj = RowParallelLinear(
98
+ intermediate_size,
99
+ hidden_size,
100
+ bias=False,
101
+ quant_config=quant_config,
102
+ prefix=f"{prefix}.down_proj",
103
+ )
104
self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation)
105
106
def forward(self, x):
@@ -104,15 +112,18 @@ class GemmaMLP(nn.Module):
112
113
class GemmaAttention(nn.Module):
114
107
- def __init__(self,
108
- hidden_size: int,
109
- num_heads: int,
110
- num_kv_heads: int,
111
- head_dim: int,
112
- max_position_embeddings: int = 8192,
113
- rope_theta: float = 10000,
114
- cache_config: Optional[CacheConfig] = None,
115
- quant_config: Optional[QuantizationConfig] = None) -> None:
115
+ def __init__(
116
+ self,
117
+ hidden_size: int,
118
+ num_heads: int,
119
+ num_kv_heads: int,
120
+ head_dim: int,
121
+ max_position_embeddings: int = 8192,
122
+ rope_theta: float = 10000,
123
+ cache_config: Optional[CacheConfig] = None,
124
+ quant_config: Optional[QuantizationConfig] = None,
125
+ prefix: str = "",
126
+ ) -> None:
127
super().__init__()
128
self.hidden_size = hidden_size
129
tp_size = get_tensor_model_parallel_world_size()
@@ -142,12 +153,14 @@ class GemmaAttention(nn.Module):
153
self.total_num_kv_heads,
154
bias=False,
155
quant_config=quant_config,
156
+ prefix=f"{prefix}.qkv_proj",
157
)
158
self.o_proj = RowParallelLinear(
159
self.total_num_heads * self.head_dim,
160
hidden_size,
161
bias=False,
162
quant_config=quant_config,
163
+ prefix=f"{prefix}.o_proj",
164
)
165
166
self.rotary_emb = get_rope(
@@ -186,6 +199,7 @@ class GemmaDecoderLayer(nn.Module):
199
config: GemmaConfig,
200
cache_config: Optional[CacheConfig] = None,
201
quant_config: Optional[QuantizationConfig] = None,
202
+ prefix: str = "",
203
) -> None:
204
super().__init__()
205
self.hidden_size = config.hidden_size
@@ -198,6 +212,7 @@ class GemmaDecoderLayer(nn.Module):
212
rope_theta=config.rope_theta,
213
cache_config=cache_config,
214
quant_config=quant_config,
215
+ prefix=f"{prefix}.self_attn",
216
)
217
self.mlp = GemmaMLP(
218
hidden_size=self.hidden_size,
@@ -205,6 +220,7 @@ class GemmaDecoderLayer(nn.Module):
220
hidden_act=config.hidden_act,
221
hidden_activation=getattr(config, "hidden_activation", None),
222
quant_config=quant_config,
223
+ prefix=f"{prefix}.mlp",
224
)
225
self.input_layernorm = GemmaRMSNorm(config.hidden_size,
226
eps=config.rms_norm_eps)
@@ -259,8 +275,8 @@ class GemmaModel(nn.Module):
275
)
276
self.start_layer, self.end_layer, self.layers = make_layers(
277
config.num_hidden_layers,
262
- lambda prefix: GemmaDecoderLayer(config, cache_config, quant_config
263
- ),
278
+ lambda prefix: GemmaDecoderLayer(
279
+ config, cache_config, quant_config, prefix=prefix),
280
prefix=f"{prefix}.layers")
281
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
282
@@ -366,6 +382,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
382
cache_config: Optional[CacheConfig] = None,
383
quant_config: Optional[QuantizationConfig] = None,
384
lora_config: Optional[LoRAConfig] = None,
385
+ prefix: str = "",
386
) -> None:
387
super().__init__()
388
@@ -375,7 +392,10 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
392
self.lora_config = lora_config
393
394
self.quant_config = quant_config
378
- self.model = GemmaModel(config, cache_config, quant_config)
395
+ self.model = GemmaModel(config,
396
+ cache_config,
397
+ quant_config,
398
+ prefix=maybe_prefix(prefix, "model"))
399
self.logits_processor = LogitsProcessor(config.vocab_size)
400
self.sampler = Sampler()
401
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/internlm2.py
+39
-17
@@ -30,7 +30,8 @@ from vllm.sequence import IntermediateTensors
30
31
from .interfaces import SupportsPP
32
from .utils import (is_pp_missing_parameter,
33
- make_empty_intermediate_tensors_factory, make_layers)
33
+ make_empty_intermediate_tensors_factory, make_layers,
34
+ maybe_prefix)
35
36
37
class InternLM2MLP(nn.Module):
@@ -41,16 +42,23 @@ class InternLM2MLP(nn.Module):
42
intermediate_size: int,
43
hidden_act: str,
44
quant_config: Optional[QuantizationConfig] = None,
45
+ prefix: str = "",
46
) -> None:
47
super().__init__()
48
self.gate_up_proj = MergedColumnParallelLinear(
47
- hidden_size, [intermediate_size] * 2,
49
+ hidden_size,
50
+ [intermediate_size] * 2,
51
+ bias=False,
52
+ quant_config=quant_config,
53
+ prefix=f"{prefix}.gate_up_proj",
54
+ )
55
+ self.w2 = RowParallelLinear(
56
+ intermediate_size,
57
+ hidden_size,
58
bias=False,
49
- quant_config=quant_config)
50
- self.w2 = RowParallelLinear(intermediate_size,
51
- hidden_size,
52
- bias=False,
53
- quant_config=quant_config)
59
+ quant_config=quant_config,
60
+ prefix=f"{prefix}.w2",
61
+ )
62
if hidden_act != "silu":
63
raise ValueError(f"Unsupported activation: {hidden_act}. "
64
"Only silu is supported for now.")
@@ -75,6 +83,7 @@ class InternLM2Attention(nn.Module):
83
max_position_embeddings: int = 8192,
84
cache_config: Optional[CacheConfig] = None,
85
quant_config: Optional[QuantizationConfig] = None,
86
+ prefix: str = "",
87
) -> None:
88
super().__init__()
89
self.hidden_size = hidden_size
@@ -108,12 +117,14 @@ class InternLM2Attention(nn.Module):
117
self.total_num_kv_heads,
118
bias=False,
119
quant_config=quant_config,
120
+ prefix=f"{prefix}.wqkv",
121
)
122
self.wo = RowParallelLinear(
123
self.total_num_heads * self.head_dim,
124
hidden_size,
125
bias=False,
126
quant_config=quant_config,
127
+ prefix=f"{prefix}.wo",
128
)
129
130
self.rotary_emb = get_rope(
@@ -123,12 +134,15 @@ class InternLM2Attention(nn.Module):
134
base=rope_theta,
135
rope_scaling=rope_scaling,
136
)
126
- self.attn = Attention(self.num_heads,
127
- self.head_dim,
128
- self.scaling,
129
- num_kv_heads=self.num_kv_heads,
130
- cache_config=cache_config,
131
- quant_config=quant_config)
137
+ self.attn = Attention(
138
+ self.num_heads,
139
+ self.head_dim,
140
+ self.scaling,
141
+ num_kv_heads=self.num_kv_heads,
142
+ cache_config=cache_config,
143
+ quant_config=quant_config,
144
+ prefix=f"{prefix}.attn",
145
+ )
146
147
def split_qkv(self, qkv: torch.Tensor):
148
seq_len = qkv.shape[0]
@@ -176,6 +190,7 @@ class InternLMDecoderLayer(nn.Module):
190
config: PretrainedConfig,
191
cache_config: Optional[CacheConfig] = None,
192
quant_config: Optional[QuantizationConfig] = None,
193
+ prefix: str = "",
194
) -> None:
195
super().__init__()
196
self.hidden_size = config.hidden_size
@@ -192,12 +207,14 @@ class InternLMDecoderLayer(nn.Module):
207
max_position_embeddings=max_position_embeddings,
208
cache_config=cache_config,
209
quant_config=quant_config,
210
+ prefix=f"{prefix}.attention",
211
)
212
self.feed_forward = InternLM2MLP(
213
hidden_size=self.hidden_size,
214
intermediate_size=config.intermediate_size,
215
hidden_act=config.hidden_act,
216
quant_config=quant_config,
217
+ prefix=f"{prefix}.feed_forward",
218
)
219
self.attention_norm = RMSNorm(config.hidden_size,
220
eps=config.rms_norm_eps)
@@ -251,8 +268,8 @@ class InternLM2Model(nn.Module):
268
)
269
self.start_layer, self.end_layer, self.layers = make_layers(
270
config.num_hidden_layers,
254
- lambda prefix: InternLMDecoderLayer(config, cache_config,
255
- quant_config),
271
+ lambda prefix: InternLMDecoderLayer(
272
+ config, cache_config, quant_config, prefix=prefix),
273
prefix=f"{prefix}.layers")
274
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
275
self.make_empty_intermediate_tensors = (
@@ -306,14 +323,19 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
323
config: PretrainedConfig,
324
cache_config: Optional[CacheConfig] = None,
325
quant_config: Optional[QuantizationConfig] = None,
326
+ prefix: str = "",
327
) -> None:
328
super().__init__()
329
self.config = config
330
self.quant_config = quant_config
313
- self.model = InternLM2Model(config, cache_config, quant_config)
331
+ self.model = InternLM2Model(config,
332
+ cache_config,
333
+ quant_config,
334
+ prefix=maybe_prefix(prefix, "model"))
335
self.output = ParallelLMHead(config.vocab_size,
336
config.hidden_size,
316
- quant_config=quant_config)
337
+ quant_config=quant_config,
338
+ prefix=maybe_prefix(prefix, "output"))
339
if self.config.tie_word_embeddings:
340
self.output.weight = self.model.tok_embeddings.weight
341
self.logits_processor = LogitsProcessor(config.vocab_size)
vllm/model_executor/models/internlm2_ve.py
+12
-4
@@ -15,7 +15,7 @@ from vllm.model_executor.models.internlm2 import (InternLM2Attention,
15
InternLM2MLP, InternLM2Model)
16
from vllm.sequence import IntermediateTensors
17
18
-from .utils import make_layers
18
+from .utils import make_layers, maybe_prefix
19
20
21
class InternLM2VEDecoderLayer(nn.Module):
@@ -25,6 +25,7 @@ class InternLM2VEDecoderLayer(nn.Module):
25
config: PretrainedConfig,
26
cache_config: Optional[CacheConfig] = None,
27
quant_config: Optional[QuantizationConfig] = None,
28
+ prefix: str = "",
29
) -> None:
30
super().__init__()
31
self.hidden_size = config.hidden_size
@@ -41,18 +42,21 @@ class InternLM2VEDecoderLayer(nn.Module):
42
max_position_embeddings=max_position_embeddings,
43
cache_config=cache_config,
44
quant_config=quant_config,
45
+ prefix=f"{prefix}.attention",
46
)
47
self.feed_forward = InternLM2MLP(
48
hidden_size=self.hidden_size,
49
intermediate_size=config.intermediate_size,
50
hidden_act=config.hidden_act,
51
quant_config=quant_config,
52
+ prefix=f"{prefix}.feed_forward",
53
)
54
self.feed_forward_ve = InternLM2MLP(
55
hidden_size=self.hidden_size,
56
intermediate_size=config.intermediate_size,
57
hidden_act=config.hidden_act,
58
quant_config=quant_config,
59
+ prefix=f"{prefix}.feed_forward_ve",
60
)
61
self.attention_norm = RMSNorm(config.hidden_size,
62
eps=config.rms_norm_eps)
@@ -111,8 +115,8 @@ class InternLM2VEModel(InternLM2Model):
115
super().__init__(config, cache_config, quant_config)
116
self.start_layer, self.end_layer, self.layers = make_layers(
117
config.num_hidden_layers,
114
- lambda prefix: InternLM2VEDecoderLayer(config, cache_config,
115
- quant_config),
118
+ lambda prefix: InternLM2VEDecoderLayer(
119
+ config, cache_config, quant_config, prefix=prefix),
120
prefix=f"{prefix}.layers")
121
122
def forward(
@@ -161,6 +165,10 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
165
config: PretrainedConfig,
166
cache_config: Optional[CacheConfig] = None,
167
quant_config: Optional[QuantizationConfig] = None,
168
+ prefix: str = "",
169
) -> None:
170
super().__init__(config, cache_config, quant_config)
166
- self.model = InternLM2VEModel(config, cache_config, quant_config)
171
+ self.model = InternLM2VEModel(config,
172
+ cache_config,
173
+ quant_config,
174
+ prefix=maybe_prefix(prefix, "model"))
vllm/model_executor/models/internvl.py
+4
-1
@@ -439,7 +439,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
439
)
440
441
self.language_model = init_vllm_registered_model(
442
- config.text_config, cache_config, quant_config)
442
+ config.text_config,
443
+ cache_config,
444
+ quant_config,
445
+ prefix="language_model")
446
447
self.mlp1 = self._init_mlp1(config)
448
vllm/model_executor/models/llama.py
+5
-2
@@ -55,7 +55,8 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
55
56
from .interfaces import SupportsLoRA, SupportsPP
57
from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
58
- make_empty_intermediate_tensors_factory, make_layers)
58
+ make_empty_intermediate_tensors_factory, make_layers,
59
+ maybe_prefix)
60
61
62
class LlamaMLP(nn.Module):
@@ -500,6 +501,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
501
cache_config: Optional[CacheConfig] = None,
502
quant_config: Optional[QuantizationConfig] = None,
503
lora_config: Optional[LoRAConfig] = None,
504
+ prefix: str = "",
505
) -> None:
506
super().__init__()
507
@@ -510,7 +512,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
512
cache_config,
513
quant_config,
514
lora_config=lora_config,
513
- prefix="model")
515
+ prefix=maybe_prefix(prefix, "model"))
516
if get_pp_group().is_last_rank:
517
self.unpadded_vocab_size = config.vocab_size
518
if lora_config:
@@ -526,6 +528,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
528
if not lora_config else
529
lora_config.lora_vocab_padding_size),
530
quant_config=quant_config,
531
+ prefix=maybe_prefix(prefix, "lm_head"),
532
)
533
if config.tie_word_embeddings:
534
self.lm_head = self.lm_head.tie_weights(
vllm/model_executor/models/llava.py
+15
-5
@@ -210,6 +210,7 @@ def init_vision_tower_for_llava(
210
quant_config: Optional[QuantizationConfig],
211
*,
212
require_post_norm: Optional[bool] = None,
213
+ prefix: str = "",
214
):
215
vision_config = hf_config.vision_config
216
@@ -224,23 +225,26 @@ def init_vision_tower_for_llava(
225
if isinstance(vision_config, CLIPVisionConfig):
226
return CLIPVisionModel(
227
vision_config,
227
- quant_config,
228
+ quant_config=quant_config,
229
num_hidden_layers_override=num_hidden_layers,
230
require_post_norm=require_post_norm,
231
+ prefix=prefix,
232
)
233
elif isinstance(vision_config, SiglipVisionConfig):
234
return SiglipVisionModel(
235
vision_config,
234
- quant_config,
236
+ quant_config=quant_config,
237
num_hidden_layers_override=num_hidden_layers,
238
require_post_norm=require_post_norm,
239
+ prefix=prefix,
240
)
241
elif isinstance(vision_config, PixtralVisionConfig):
242
return PixtralHFVisionModel(
243
vision_config,
241
- quant_config,
244
+ quant_config=quant_config,
245
num_hidden_layers_override=num_hidden_layers,
246
require_post_norm=require_post_norm,
247
+ prefix=prefix,
248
)
249
250
msg = f"Unsupported vision config: {type(vision_config)}"
@@ -274,14 +278,20 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
278
279
# TODO: Optionally initializes this for supporting embeddings.
280
self.vision_tower = init_vision_tower_for_llava(
277
- config, quant_config, require_post_norm=False)
281
+ config,
282
+ quant_config,
283
+ require_post_norm=False,
284
+ prefix="vision_tower")
285
self.multi_modal_projector = LlavaMultiModalProjector(
286
vision_hidden_size=config.vision_config.hidden_size,
287
text_hidden_size=config.text_config.hidden_size,
288
projector_hidden_act=config.projector_hidden_act)
289
290
self.language_model = init_vllm_registered_model(
284
- config.text_config, cache_config, quant_config)
291
+ config.text_config,
292
+ cache_config,
293
+ quant_config,
294
+ prefix="language_model")
295
296
self.make_empty_intermediate_tensors = (
297
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_next.py
+8
-2
@@ -293,7 +293,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
293
294
# TODO: Optionally initializes this for supporting embeddings.
295
self.vision_tower = init_vision_tower_for_llava(
296
- config, quant_config, require_post_norm=False)
296
+ config,
297
+ quant_config,
298
+ require_post_norm=False,
299
+ prefix="vision_tower")
300
self.image_newline = nn.Parameter(
301
torch.empty(config.text_config.hidden_size))
302
self.multi_modal_projector = LlavaMultiModalProjector(
@@ -302,7 +305,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
305
projector_hidden_act=config.projector_hidden_act)
306
307
self.language_model = init_vllm_registered_model(
305
- config.text_config, cache_config, quant_config)
308
+ config.text_config,
309
+ cache_config,
310
+ quant_config,
311
+ prefix="language_model")
312
313
# The same model class supports both language generation and embedding
314
# because the architecture name is the same
vllm/model_executor/models/llava_next_video.py
+8
-2
@@ -257,14 +257,20 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
257
258
# Initialize the vision tower only up to the required feature layer
259
self.vision_tower = init_vision_tower_for_llava(
260
- config, quant_config, require_post_norm=False)
260
+ config,
261
+ quant_config,
262
+ require_post_norm=False,
263
+ prefix="vision_tower")
264
self.vision_resampler = LlavaNextVideoPooler(config)
265
self.multi_modal_projector = LlavaNextMultiModalProjector(
266
vision_hidden_size=config.vision_config.hidden_size,
267
text_hidden_size=config.text_config.hidden_size,
268
projector_hidden_act=config.projector_hidden_act)
269
self.language_model = init_vllm_registered_model(
267
- config.text_config, cache_config, quant_config)
270
+ config.text_config,
271
+ cache_config,
272
+ quant_config,
273
+ prefix="language_model")
274
275
self.make_empty_intermediate_tensors = (
276
self.language_model.model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_onevision.py
+8
-2
@@ -415,10 +415,16 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
415
416
# Initialize the vision tower only up to the required feature layer
417
self.vision_tower = init_vision_tower_for_llava(
418
- config, quant_config, require_post_norm=False)
418
+ config,
419
+ quant_config,
420
+ require_post_norm=False,
421
+ prefix="vision_tower")
422
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
423
self.language_model = init_vllm_registered_model(
421
- config.text_config, cache_config, quant_config)
424
+ config.text_config,
425
+ cache_config,
426
+ quant_config,
427
+ prefix="language_model")
428
self.image_newline = nn.Parameter(
429
torch.empty(config.text_config.hidden_size))
430
vllm/model_executor/models/minicpmv.py
+26
-8
@@ -394,8 +394,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
394
self.multimodal_config = multimodal_config
395
396
self.version = get_version_by_config(self.config)
397
- self.llm = self.init_llm(config, cache_config, quant_config)
398
- self.vpm = self.init_vision_module(config, quant_config)
397
+ self.llm = self.init_llm(config,
398
+ cache_config,
399
+ quant_config,
400
+ prefix="llm")
401
+ self.vpm = self.init_vision_module(config, quant_config, prefix="vpm")
402
param_dtype = torch.get_default_dtype()
403
self.vpm.to(dtype=param_dtype)
404
self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
@@ -403,9 +406,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
406
self.embed_dim = self.config.hidden_size
407
self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
408
self.resampler.to(device="cuda", dtype=param_dtype)
409
+ # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
410
self.lm_head = ParallelLMHead(config.vocab_size,
411
config.hidden_size,
408
- quant_config=quant_config)
412
+ quant_config=quant_config,
413
+ prefix="llm.lm_head")
414
self.logits_processor = LogitsProcessor(config.vocab_size)
415
self.sampler = Sampler()
416
@@ -644,6 +649,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
649
config: PretrainedConfig,
650
cache_config: Optional[CacheConfig] = None,
651
quant_config: Optional[QuantizationConfig] = None,
652
+ prefix: str = "",
653
) -> nn.Module:
654
raise NotImplementedError
655
@@ -651,6 +657,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
657
self,
658
config: PretrainedConfig,
659
quant_config: Optional[QuantizationConfig],
660
+ prefix: str = "",
661
) -> nn.Module:
662
raise NotImplementedError
663
@@ -690,17 +697,20 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
697
config: PretrainedConfig,
698
cache_config: Optional[CacheConfig] = None,
699
quant_config: Optional[QuantizationConfig] = None,
700
+ prefix: str = "",
701
) -> nn.Module:
702
703
return LLMWrapper(MiniCPMModel(config,
704
cache_config=cache_config,
697
- quant_config=quant_config),
705
+ quant_config=quant_config,
706
+ prefix=prefix),
707
name="model")
708
709
def init_vision_module(
710
self,
711
config: PretrainedConfig,
712
quant_config: Optional[QuantizationConfig],
713
+ prefix: str = "",
714
) -> nn.Module:
715
# TODO :refactor this vision model
716
try:
@@ -819,19 +829,23 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
829
config: PretrainedConfig,
830
cache_config: Optional[CacheConfig] = None,
831
quant_config: Optional[QuantizationConfig] = None,
832
+ prefix: str = "",
833
) -> nn.Module:
834
return LLMWrapper(LlamaModel(config,
835
cache_config=cache_config,
825
- quant_config=quant_config),
836
+ quant_config=quant_config,
837
+ prefix=prefix),
838
name="model")
839
840
def init_vision_module(
841
self,
842
config: PretrainedConfig,
843
quant_config: Optional[QuantizationConfig],
844
+ prefix: str = "",
845
) -> nn.Module:
846
model = Idefics2VisionTransformer(config.vision_config,
834
- quant_config=quant_config)
847
+ quant_config=quant_config,
848
+ prefix=prefix)
849
if self.config.drop_vision_last_layer:
850
model.encoder.layers = model.encoder.layers[:-1]
851
return model
@@ -935,20 +949,24 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
949
config: PretrainedConfig,
950
cache_config: Optional[CacheConfig] = None,
951
quant_config: Optional[QuantizationConfig] = None,
952
+ prefix: str = "",
953
) -> nn.Module:
954
955
return LLMWrapper(Qwen2Model(config,
956
cache_config=cache_config,
942
- quant_config=quant_config),
957
+ quant_config=quant_config,
958
+ prefix=prefix),
959
name="model")
960
961
def init_vision_module(
962
self,
963
config: PretrainedConfig,
964
quant_config: Optional[QuantizationConfig],
965
+ prefix: str = "",
966
) -> nn.Module:
967
model = Idefics2VisionTransformer(config.vision_config,
951
- quant_config=quant_config)
968
+ quant_config=quant_config,
969
+ prefix=prefix)
970
if self.config.drop_vision_last_layer:
971
model.encoder.layers = model.encoder.layers[:-1]
972
return model
vllm/model_executor/models/opt.py
+27
-7
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44
from .interfaces import SupportsPP
45
from .utils import (is_pp_missing_parameter,
46
- make_empty_intermediate_tensors_factory, make_layers)
46
+ make_empty_intermediate_tensors_factory, make_layers,
47
+ maybe_prefix)
48
49
50
class OPTLearnedPositionalEmbedding(nn.Embedding):
@@ -68,6 +69,7 @@ class OPTAttention(nn.Module):
69
bias: bool = True,
70
cache_config: Optional[CacheConfig] = None,
71
quant_config: Optional[QuantizationConfig] = None,
72
+ prefix: str = "",
73
) -> None:
74
super().__init__()
75
self.embed_dim = embed_dim
@@ -85,18 +87,21 @@ class OPTAttention(nn.Module):
87
total_num_heads,
88
bias=bias,
89
quant_config=quant_config,
90
+ prefix=f"{prefix}.qkv_proj",
91
)
92
self.out_proj = RowParallelLinear(
93
embed_dim,
94
embed_dim,
95
bias=bias,
96
quant_config=quant_config,
97
+ prefix=f"{prefix}.out_proj",
98
)
99
self.attn = Attention(self.num_heads,
100
self.head_dim,
101
scale=self.scaling,
102
cache_config=cache_config,
99
- quant_config=quant_config)
103
+ quant_config=quant_config,
104
+ prefix=f"{prefix}.attn")
105
106
def forward(
107
self,
@@ -118,6 +123,7 @@ class OPTDecoderLayer(nn.Module):
123
config: OPTConfig,
124
cache_config: Optional[CacheConfig] = None,
125
quant_config: Optional[QuantizationConfig] = None,
126
+ prefix: str = "",
127
):
128
super().__init__()
129
self.config = config
@@ -128,6 +134,7 @@ class OPTDecoderLayer(nn.Module):
134
bias=config.enable_bias,
135
cache_config=cache_config,
136
quant_config=quant_config,
137
+ prefix=f"{prefix}.self_attn",
138
)
139
self.do_layer_norm_before = config.do_layer_norm_before
140
@@ -139,6 +146,7 @@ class OPTDecoderLayer(nn.Module):
146
config.ffn_dim,
147
bias=config.enable_bias,
148
quant_config=quant_config,
149
+ prefix=f"{prefix}.fc1",
150
)
151
self.activation_fn = get_act_fn(config.activation_function,
152
quant_config, config.ffn_dim)
@@ -147,6 +155,7 @@ class OPTDecoderLayer(nn.Module):
155
self.embed_dim,
156
bias=config.enable_bias,
157
quant_config=quant_config,
158
+ prefix=f"{prefix}.fc2",
159
)
160
self.final_layer_norm = nn.LayerNorm(
161
self.embed_dim,
@@ -214,7 +223,8 @@ class OPTDecoder(nn.Module):
223
self.project_out = ReplicatedLinear(config.hidden_size,
224
config.word_embed_proj_dim,
225
bias=False,
217
- quant_config=quant_config)
226
+ quant_config=quant_config,
227
+ prefix=f"{prefix}.project_out")
228
else:
229
self.project_out = None
230
@@ -222,7 +232,8 @@ class OPTDecoder(nn.Module):
232
self.project_in = ReplicatedLinear(config.word_embed_proj_dim,
233
config.hidden_size,
234
bias=False,
225
- quant_config=quant_config)
235
+ quant_config=quant_config,
236
+ prefix=f"{prefix}.project_in")
237
else:
238
self.project_in = None
239
@@ -239,7 +250,8 @@ class OPTDecoder(nn.Module):
250
251
self.start_layer, self.end_layer, self.layers = make_layers(
252
config.num_hidden_layers,
242
- lambda prefix: OPTDecoderLayer(config, cache_config, quant_config),
253
+ lambda prefix: OPTDecoderLayer(
254
+ config, cache_config, quant_config, prefix=prefix),
255
prefix=f"{prefix}.layers")
256
257
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -288,9 +300,13 @@ class OPTModel(nn.Module):
300
config: OPTConfig,
301
cache_config: Optional[CacheConfig] = None,
302
quant_config: Optional[QuantizationConfig] = None,
303
+ prefix: str = "",
304
):
305
super().__init__()
293
- self.decoder = OPTDecoder(config, cache_config, quant_config)
306
+ self.decoder = OPTDecoder(config,
307
+ cache_config,
308
+ quant_config,
309
+ prefix=f"{prefix}.decoder")
310
self.make_empty_intermediate_tensors = (
311
make_empty_intermediate_tensors_factory(["hidden_states"],
312
config.hidden_size))
@@ -335,11 +351,15 @@ class OPTForCausalLM(nn.Module, SupportsPP):
351
config: OPTConfig,
352
cache_config: Optional[CacheConfig] = None,
353
quant_config: Optional[QuantizationConfig] = None,
354
+ prefix: str = "",
355
):
356
super().__init__()
357
self.config = config
358
self.quant_config = quant_config
342
- self.model = OPTModel(config, cache_config, quant_config)
359
+ self.model = OPTModel(config,
360
+ cache_config,
361
+ quant_config,
362
+ prefix=maybe_prefix(prefix, "model"))
363
if self.config.tie_word_embeddings:
364
self.lm_head = self.model.decoder.embed_tokens
365
else:
vllm/model_executor/models/paligemma.py
+5
-2
@@ -143,14 +143,17 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
143
self.multimodal_config = multimodal_config
144
145
self.vision_tower = SiglipVisionModel(config.vision_config,
146
- quant_config)
146
+ quant_config,
147
+ prefix="vision_tower")
148
self.multi_modal_projector = PaliGemmaMultiModalProjector(
149
vision_hidden_size=config.vision_config.hidden_size,
150
projection_dim=config.vision_config.projection_dim)
151
152
self.quant_config = quant_config
153
self.language_model = GemmaForCausalLM(config.text_config,
153
- cache_config, quant_config)
154
+ cache_config,
155
+ quant_config,
156
+ prefix="language_model")
157
logit_scale = getattr(config, "logit_scale", 1.0)
158
self.language_model.logits_processor.scale *= logit_scale
159
vllm/model_executor/models/phi3v.py
+14
-5
@@ -71,7 +71,8 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
71
72
73
def _init_img_processor(hf_config: PretrainedConfig,
74
- quant_config: Optional[QuantizationConfig]):
74
+ quant_config: Optional[QuantizationConfig],
75
+ prefix: str = "") -> CLIPVisionModel:
76
clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
77
layer_idx = hf_config.img_processor.get('layer_idx', -2)
78
@@ -86,6 +87,7 @@ def _init_img_processor(hf_config: PretrainedConfig,
87
clip_config,
88
quant_config,
89
num_hidden_layers_override=num_hidden_layers,
90
+ prefix=prefix,
91
)
92
93
return img_processor
@@ -152,15 +154,18 @@ class Phi3ImageEmbeddingBase(nn.Module):
154
class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
155
"""Phi3 Image embedding with HD transform."""
156
155
- def __init__(self, config: PretrainedConfig,
156
- quant_config: Optional[QuantizationConfig]) -> None:
157
+ def __init__(self,
158
+ config: PretrainedConfig,
159
+ quant_config: Optional[QuantizationConfig],
160
+ prefix: str = "") -> None:
161
super().__init__()
162
163
# n_embed or hidden_size
164
hidden_size = config.n_embd if hasattr(
165
config, 'n_embd') else config.hidden_size
166
163
- self.img_processor = _init_img_processor(config, quant_config)
167
+ self.img_processor = _init_img_processor(
168
+ config, quant_config, prefix=f"{prefix}.img_processor")
169
170
image_dim_out = config.img_processor['image_dim_out']
171
self.num_img_tokens = config.img_processor['num_img_tokens']
@@ -537,11 +542,15 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
542
config.hidden_size,
543
org_num_embeddings=config.vocab_size,
544
quant_config=quant_config,
545
+ prefix="model.embed_tokens",
546
)
547
548
# TODO: Optionally initializes this for supporting input embeddings.
543
- self.vision_embed_tokens = Phi3HDImageEmbedding(config, quant_config)
549
+ self.vision_embed_tokens = Phi3HDImageEmbedding(
550
+ config, quant_config, prefix="model.vision_embed_tokens")
551
552
+ # The prefix is empty intentionally because default prefix of
553
+ # LlamaForCausalLM is "model"
554
self.language_model = LlamaForCausalLM(config, cache_config,
555
quant_config)
556
vllm/model_executor/models/pixtral.py
+4
-1
@@ -164,7 +164,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
164
165
# init MistralForCausalLM
166
self.language_model = init_vllm_registered_model(
167
- config.text_config, cache_config, quant_config)
167
+ config.text_config,
168
+ cache_config,
169
+ quant_config,
170
+ prefix="language_model")
171
172
self.vision_encoder = VisionTransformer(self.vision_args)
173
self.vision_language_adapter = VisionLanguageAdapter(
vllm/model_executor/models/qwen2.py
+37
-13
@@ -49,7 +49,8 @@ from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA, SupportsPP
51
from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
52
- make_empty_intermediate_tensors_factory, make_layers)
52
+ make_empty_intermediate_tensors_factory, make_layers,
53
+ maybe_prefix)
54
55
56
class Qwen2MLP(nn.Module):
@@ -60,16 +61,23 @@ class Qwen2MLP(nn.Module):
61
intermediate_size: int,
62
hidden_act: str,
63
quant_config: Optional[QuantizationConfig] = None,
64
+ prefix: str = "",
65
) -> None:
66
super().__init__()
67
self.gate_up_proj = MergedColumnParallelLinear(
66
- hidden_size, [intermediate_size] * 2,
68
+ hidden_size,
69
+ [intermediate_size] * 2,
70
+ bias=False,
71
+ quant_config=quant_config,
72
+ prefix=f"{prefix}.gate_up_proj",
73
+ )
74
+ self.down_proj = RowParallelLinear(
75
+ intermediate_size,
76
+ hidden_size,
77
bias=False,
68
- quant_config=quant_config)
69
- self.down_proj = RowParallelLinear(intermediate_size,
70
- hidden_size,
71
- bias=False,
72
- quant_config=quant_config)
78
+ quant_config=quant_config,
79
+ prefix=f"{prefix}.down_proj",
80
+ )
81
if hidden_act != "silu":
82
raise ValueError(f"Unsupported activation: {hidden_act}. "
83
"Only silu is supported for now.")
@@ -92,7 +100,8 @@ class Qwen2Attention(nn.Module):
100
rope_theta: float = 10000,
101
cache_config: Optional[CacheConfig] = None,
102
quant_config: Optional[QuantizationConfig] = None,
95
- rope_scaling: Optional[Tuple] = None) -> None:
103
+ rope_scaling: Optional[Tuple] = None,
104
+ prefix: str = "") -> None:
105
super().__init__()
106
self.hidden_size = hidden_size
107
tp_size = get_tensor_model_parallel_world_size()
@@ -122,12 +131,14 @@ class Qwen2Attention(nn.Module):
131
self.total_num_kv_heads,
132
bias=True,
133
quant_config=quant_config,
134
+ prefix=f"{prefix}.qkv_proj",
135
)
136
self.o_proj = RowParallelLinear(
137
self.total_num_heads * self.head_dim,
138
hidden_size,
139
bias=False,
140
quant_config=quant_config,
141
+ prefix=f"{prefix}.o_proj",
142
)
143
144
self.rotary_emb = get_rope(
@@ -142,7 +153,8 @@ class Qwen2Attention(nn.Module):
153
self.scaling,
154
num_kv_heads=self.num_kv_heads,
155
cache_config=cache_config,
145
- quant_config=quant_config)
156
+ quant_config=quant_config,
157
+ prefix=f"{prefix}.attn")
158
159
def forward(
160
self,
@@ -166,6 +178,7 @@ class Qwen2DecoderLayer(nn.Module):
178
config: Qwen2Config,
179
cache_config: Optional[CacheConfig] = None,
180
quant_config: Optional[QuantizationConfig] = None,
181
+ prefix: str = "",
182
) -> None:
183
super().__init__()
184
self.hidden_size = config.hidden_size
@@ -180,12 +193,15 @@ class Qwen2DecoderLayer(nn.Module):
193
rope_theta=rope_theta,
194
cache_config=cache_config,
195
quant_config=quant_config,
183
- rope_scaling=rope_scaling)
196
+ rope_scaling=rope_scaling,
197
+ prefix=f"{prefix}.self_attn",
198
+ )
199
self.mlp = Qwen2MLP(
200
hidden_size=self.hidden_size,
201
intermediate_size=config.intermediate_size,
202
hidden_act=config.hidden_act,
203
quant_config=quant_config,
204
+ prefix=f"{prefix}.mlp",
205
)
206
self.input_layernorm = RMSNorm(config.hidden_size,
207
eps=config.rms_norm_eps)
@@ -241,6 +257,7 @@ class Qwen2Model(nn.Module):
257
config.vocab_size,
258
config.hidden_size,
259
quant_config=quant_config,
260
+ prefix=f"{prefix}.embed_tokens",
261
)
262
else:
263
self.embed_tokens = PPMissingLayer()
@@ -249,7 +266,8 @@ class Qwen2Model(nn.Module):
266
config.num_hidden_layers,
267
lambda prefix: Qwen2DecoderLayer(config=config,
268
cache_config=cache_config,
252
- quant_config=quant_config),
269
+ quant_config=quant_config,
270
+ prefix=f"{prefix}.layers"),
271
prefix=f"{prefix}.layers",
272
)
273
@@ -393,6 +411,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411
cache_config: Optional[CacheConfig] = None,
412
quant_config: Optional[QuantizationConfig] = None,
413
lora_config: Optional[LoRAConfig] = None,
414
+ prefix: str = "",
415
) -> None:
416
# TODO (@robertgshaw2): see if this can be moved out
417
if (cache_config.sliding_window is not None
@@ -412,14 +431,19 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
431
self.lora_config = lora_config
432
433
self.quant_config = quant_config
415
- self.model = Qwen2Model(config, cache_config, quant_config)
434
+ self.model = Qwen2Model(config,
435
+ cache_config,
436
+ quant_config,
437
+ prefix=maybe_prefix(prefix, "model"))
438
439
if config.tie_word_embeddings:
440
self.lm_head = self.model.embed_tokens
441
else:
442
self.lm_head = ParallelLMHead(config.vocab_size,
443
config.hidden_size,
422
- quant_config=quant_config)
444
+ quant_config=quant_config,
445
+ prefix=maybe_prefix(
446
+ prefix, "lm_head"))
447
448
self.logits_processor = LogitsProcessor(config.vocab_size)
449
self.sampler = Sampler()
vllm/model_executor/models/qwen2_vl.py
+6
-2
@@ -938,7 +938,10 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
938
quant_config=None,
939
)
940
941
- self.model = Qwen2Model(config, cache_config, quant_config)
941
+ self.model = Qwen2Model(config,
942
+ cache_config,
943
+ quant_config,
944
+ prefix="model")
945
946
if get_pp_group().is_last_rank:
947
if config.tie_word_embeddings:
@@ -946,7 +949,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
949
else:
950
self.lm_head = ParallelLMHead(config.vocab_size,
951
config.hidden_size,
949
- quant_config=quant_config)
952
+ quant_config=quant_config,
953
+ prefix="lm_head")
954
else:
955
self.lm_head = PPMissingLayer()
956
vllm/model_executor/models/ultravox.py
+4
-1
@@ -357,7 +357,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
357
))
358
self.multi_modal_projector = UltravoxProjector(config)
359
self.language_model = init_vllm_registered_model(
360
- config.text_config, cache_config, quant_config)
360
+ config.text_config,
361
+ cache_config,
362
+ quant_config,
363
+ prefix="language_model")
364
if config.text_model_id is not None:
365
self.secondary_weights.append(
366
DefaultModelLoader.Source(model_or_path=config.text_model_id,
vllm/model_executor/models/utils.py
+15
@@ -242,6 +242,7 @@ def init_vllm_registered_model(
242
lora_config: Optional[LoRAConfig] = None,
243
multimodal_config: Optional[MultiModalConfig] = None,
244
scheduler_config: Optional[SchedulerConfig] = None,
245
+ prefix: str = "",
246
) -> nn.Module:
247
"""
248
Helper function to initialize an inner model registered to vLLM,
@@ -257,6 +258,7 @@ def init_vllm_registered_model(
258
lora_config=lora_config,
259
multimodal_config=multimodal_config,
260
scheduler_config=scheduler_config,
261
+ prefix=prefix,
262
)
263
264
@@ -610,3 +612,16 @@ def get_vit_attn_backend() -> _Backend:
612
else:
613
selected_backend = _Backend.XFORMERS
614
return selected_backend
615
+
616
+
617
+def maybe_prefix(prefix: str, name: str) -> str:
618
+ """Add a prefix to a name if the prefix is non-empty.
619
+
620
+ Args:
621
+ prefix: The prefix to add. If empty, no prefix will be added.
622
+ name: The name to potentially prefix.
623
+
624
+ Returns:
625
+ The string "prefix.name" if prefix was non-empty, otherwise just "name".
626
+ """
627
+ return name if not prefix else f"{prefix}.{name}"