@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc73e9821c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix prefix strings for quantized VLMs (#9772)

Michael Goin committed Oct 29, 2024 at 19:02 UTC bc73e9821cb4f90a88c04e7d550f132d8911266b
20 files changed +288 -97
vllm/model_executor/model_loader/loader.py
+8 -3
@@ -147,15 +147,20 @@ def _get_model_initialization_kwargs(
147 return extra_kwargs
148
149
150 -def build_model(model_class: Type[nn.Module], hf_config: PretrainedConfig,
150 +def build_model(model_class: Type[nn.Module],
151 + hf_config: PretrainedConfig,
152 cache_config: Optional[CacheConfig],
152 - quant_config: Optional[QuantizationConfig], *,
153 + quant_config: Optional[QuantizationConfig],
154 + *,
155 lora_config: Optional[LoRAConfig],
156 multimodal_config: Optional[MultiModalConfig],
155 - scheduler_config: Optional[SchedulerConfig]) -> nn.Module:
157 + scheduler_config: Optional[SchedulerConfig],
158 + prefix: Optional[str] = None) -> nn.Module:
159 extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
160 multimodal_config,
161 scheduler_config)
162 + if prefix:
163 + extra_kwargs["prefix"] = prefix
164
165 return model_class(config=hf_config,
166 cache_config=cache_config,
vllm/model_executor/models/blip2.py
+4 -1
@@ -507,7 +507,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
507 )
508
509 self.language_model = init_vllm_registered_model(
510 - config.text_config, cache_config, quant_config)
510 + config.text_config,
511 + cache_config,
512 + quant_config,
513 + prefix="language_model")
514
515 self.make_empty_intermediate_tensors = (
516 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/gemma.py
+39 -19
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44 from .interfaces import SupportsLoRA, SupportsPP
45 from .utils import (is_pp_missing_parameter,
46 - make_empty_intermediate_tensors_factory, make_layers)
46 + make_empty_intermediate_tensors_factory, make_layers,
47 + maybe_prefix)
48
49 logger = init_logger(__name__)
50
@@ -83,16 +84,23 @@ class GemmaMLP(nn.Module):
84 hidden_act: Optional[str] = None,
85 hidden_activation: Optional[str] = None,
86 quant_config: Optional[QuantizationConfig] = None,
87 + prefix: str = "",
88 ) -> None:
89 super().__init__()
90 self.gate_up_proj = MergedColumnParallelLinear(
89 - hidden_size, [intermediate_size] * 2,
91 + hidden_size,
92 + [intermediate_size] * 2,
93 bias=False,
91 - quant_config=quant_config)
92 - self.down_proj = RowParallelLinear(intermediate_size,
93 - hidden_size,
94 - bias=False,
95 - quant_config=quant_config)
94 + quant_config=quant_config,
95 + prefix=f"{prefix}.gate_up_proj",
96 + )
97 + self.down_proj = RowParallelLinear(
98 + intermediate_size,
99 + hidden_size,
100 + bias=False,
101 + quant_config=quant_config,
102 + prefix=f"{prefix}.down_proj",
103 + )
104 self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation)
105
106 def forward(self, x):
@@ -104,15 +112,18 @@ class GemmaMLP(nn.Module):
112
113 class GemmaAttention(nn.Module):
114
107 - def __init__(self,
108 - hidden_size: int,
109 - num_heads: int,
110 - num_kv_heads: int,
111 - head_dim: int,
112 - max_position_embeddings: int = 8192,
113 - rope_theta: float = 10000,
114 - cache_config: Optional[CacheConfig] = None,
115 - quant_config: Optional[QuantizationConfig] = None) -> None:
115 + def __init__(
116 + self,
117 + hidden_size: int,
118 + num_heads: int,
119 + num_kv_heads: int,
120 + head_dim: int,
121 + max_position_embeddings: int = 8192,
122 + rope_theta: float = 10000,
123 + cache_config: Optional[CacheConfig] = None,
124 + quant_config: Optional[QuantizationConfig] = None,
125 + prefix: str = "",
126 + ) -> None:
127 super().__init__()
128 self.hidden_size = hidden_size
129 tp_size = get_tensor_model_parallel_world_size()
@@ -142,12 +153,14 @@ class GemmaAttention(nn.Module):
153 self.total_num_kv_heads,
154 bias=False,
155 quant_config=quant_config,
156 + prefix=f"{prefix}.qkv_proj",
157 )
158 self.o_proj = RowParallelLinear(
159 self.total_num_heads * self.head_dim,
160 hidden_size,
161 bias=False,
162 quant_config=quant_config,
163 + prefix=f"{prefix}.o_proj",
164 )
165
166 self.rotary_emb = get_rope(
@@ -186,6 +199,7 @@ class GemmaDecoderLayer(nn.Module):
199 config: GemmaConfig,
200 cache_config: Optional[CacheConfig] = None,
201 quant_config: Optional[QuantizationConfig] = None,
202 + prefix: str = "",
203 ) -> None:
204 super().__init__()
205 self.hidden_size = config.hidden_size
@@ -198,6 +212,7 @@ class GemmaDecoderLayer(nn.Module):
212 rope_theta=config.rope_theta,
213 cache_config=cache_config,
214 quant_config=quant_config,
215 + prefix=f"{prefix}.self_attn",
216 )
217 self.mlp = GemmaMLP(
218 hidden_size=self.hidden_size,
@@ -205,6 +220,7 @@ class GemmaDecoderLayer(nn.Module):
220 hidden_act=config.hidden_act,
221 hidden_activation=getattr(config, "hidden_activation", None),
222 quant_config=quant_config,
223 + prefix=f"{prefix}.mlp",
224 )
225 self.input_layernorm = GemmaRMSNorm(config.hidden_size,
226 eps=config.rms_norm_eps)
@@ -259,8 +275,8 @@ class GemmaModel(nn.Module):
275 )
276 self.start_layer, self.end_layer, self.layers = make_layers(
277 config.num_hidden_layers,
262 - lambda prefix: GemmaDecoderLayer(config, cache_config, quant_config
263 - ),
278 + lambda prefix: GemmaDecoderLayer(
279 + config, cache_config, quant_config, prefix=prefix),
280 prefix=f"{prefix}.layers")
281 self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
282
@@ -366,6 +382,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
382 cache_config: Optional[CacheConfig] = None,
383 quant_config: Optional[QuantizationConfig] = None,
384 lora_config: Optional[LoRAConfig] = None,
385 + prefix: str = "",
386 ) -> None:
387 super().__init__()
388
@@ -375,7 +392,10 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
392 self.lora_config = lora_config
393
394 self.quant_config = quant_config
378 - self.model = GemmaModel(config, cache_config, quant_config)
395 + self.model = GemmaModel(config,
396 + cache_config,
397 + quant_config,
398 + prefix=maybe_prefix(prefix, "model"))
399 self.logits_processor = LogitsProcessor(config.vocab_size)
400 self.sampler = Sampler()
401 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/internlm2.py
+39 -17
@@ -30,7 +30,8 @@ from vllm.sequence import IntermediateTensors
30
31 from .interfaces import SupportsPP
32 from .utils import (is_pp_missing_parameter,
33 - make_empty_intermediate_tensors_factory, make_layers)
33 + make_empty_intermediate_tensors_factory, make_layers,
34 + maybe_prefix)
35
36
37 class InternLM2MLP(nn.Module):
@@ -41,16 +42,23 @@ class InternLM2MLP(nn.Module):
42 intermediate_size: int,
43 hidden_act: str,
44 quant_config: Optional[QuantizationConfig] = None,
45 + prefix: str = "",
46 ) -> None:
47 super().__init__()
48 self.gate_up_proj = MergedColumnParallelLinear(
47 - hidden_size, [intermediate_size] * 2,
49 + hidden_size,
50 + [intermediate_size] * 2,
51 + bias=False,
52 + quant_config=quant_config,
53 + prefix=f"{prefix}.gate_up_proj",
54 + )
55 + self.w2 = RowParallelLinear(
56 + intermediate_size,
57 + hidden_size,
58 bias=False,
49 - quant_config=quant_config)
50 - self.w2 = RowParallelLinear(intermediate_size,
51 - hidden_size,
52 - bias=False,
53 - quant_config=quant_config)
59 + quant_config=quant_config,
60 + prefix=f"{prefix}.w2",
61 + )
62 if hidden_act != "silu":
63 raise ValueError(f"Unsupported activation: {hidden_act}. "
64 "Only silu is supported for now.")
@@ -75,6 +83,7 @@ class InternLM2Attention(nn.Module):
83 max_position_embeddings: int = 8192,
84 cache_config: Optional[CacheConfig] = None,
85 quant_config: Optional[QuantizationConfig] = None,
86 + prefix: str = "",
87 ) -> None:
88 super().__init__()
89 self.hidden_size = hidden_size
@@ -108,12 +117,14 @@ class InternLM2Attention(nn.Module):
117 self.total_num_kv_heads,
118 bias=False,
119 quant_config=quant_config,
120 + prefix=f"{prefix}.wqkv",
121 )
122 self.wo = RowParallelLinear(
123 self.total_num_heads * self.head_dim,
124 hidden_size,
125 bias=False,
126 quant_config=quant_config,
127 + prefix=f"{prefix}.wo",
128 )
129
130 self.rotary_emb = get_rope(
@@ -123,12 +134,15 @@ class InternLM2Attention(nn.Module):
134 base=rope_theta,
135 rope_scaling=rope_scaling,
136 )
126 - self.attn = Attention(self.num_heads,
127 - self.head_dim,
128 - self.scaling,
129 - num_kv_heads=self.num_kv_heads,
130 - cache_config=cache_config,
131 - quant_config=quant_config)
137 + self.attn = Attention(
138 + self.num_heads,
139 + self.head_dim,
140 + self.scaling,
141 + num_kv_heads=self.num_kv_heads,
142 + cache_config=cache_config,
143 + quant_config=quant_config,
144 + prefix=f"{prefix}.attn",
145 + )
146
147 def split_qkv(self, qkv: torch.Tensor):
148 seq_len = qkv.shape[0]
@@ -176,6 +190,7 @@ class InternLMDecoderLayer(nn.Module):
190 config: PretrainedConfig,
191 cache_config: Optional[CacheConfig] = None,
192 quant_config: Optional[QuantizationConfig] = None,
193 + prefix: str = "",
194 ) -> None:
195 super().__init__()
196 self.hidden_size = config.hidden_size
@@ -192,12 +207,14 @@ class InternLMDecoderLayer(nn.Module):
207 max_position_embeddings=max_position_embeddings,
208 cache_config=cache_config,
209 quant_config=quant_config,
210 + prefix=f"{prefix}.attention",
211 )
212 self.feed_forward = InternLM2MLP(
213 hidden_size=self.hidden_size,
214 intermediate_size=config.intermediate_size,
215 hidden_act=config.hidden_act,
216 quant_config=quant_config,
217 + prefix=f"{prefix}.feed_forward",
218 )
219 self.attention_norm = RMSNorm(config.hidden_size,
220 eps=config.rms_norm_eps)
@@ -251,8 +268,8 @@ class InternLM2Model(nn.Module):
268 )
269 self.start_layer, self.end_layer, self.layers = make_layers(
270 config.num_hidden_layers,
254 - lambda prefix: InternLMDecoderLayer(config, cache_config,
255 - quant_config),
271 + lambda prefix: InternLMDecoderLayer(
272 + config, cache_config, quant_config, prefix=prefix),
273 prefix=f"{prefix}.layers")
274 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
275 self.make_empty_intermediate_tensors = (
@@ -306,14 +323,19 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
323 config: PretrainedConfig,
324 cache_config: Optional[CacheConfig] = None,
325 quant_config: Optional[QuantizationConfig] = None,
326 + prefix: str = "",
327 ) -> None:
328 super().__init__()
329 self.config = config
330 self.quant_config = quant_config
313 - self.model = InternLM2Model(config, cache_config, quant_config)
331 + self.model = InternLM2Model(config,
332 + cache_config,
333 + quant_config,
334 + prefix=maybe_prefix(prefix, "model"))
335 self.output = ParallelLMHead(config.vocab_size,
336 config.hidden_size,
316 - quant_config=quant_config)
337 + quant_config=quant_config,
338 + prefix=maybe_prefix(prefix, "output"))
339 if self.config.tie_word_embeddings:
340 self.output.weight = self.model.tok_embeddings.weight
341 self.logits_processor = LogitsProcessor(config.vocab_size)
vllm/model_executor/models/internlm2_ve.py
+12 -4
@@ -15,7 +15,7 @@ from vllm.model_executor.models.internlm2 import (InternLM2Attention,
15 InternLM2MLP, InternLM2Model)
16 from vllm.sequence import IntermediateTensors
17
18 -from .utils import make_layers
18 +from .utils import make_layers, maybe_prefix
19
20
21 class InternLM2VEDecoderLayer(nn.Module):
@@ -25,6 +25,7 @@ class InternLM2VEDecoderLayer(nn.Module):
25 config: PretrainedConfig,
26 cache_config: Optional[CacheConfig] = None,
27 quant_config: Optional[QuantizationConfig] = None,
28 + prefix: str = "",
29 ) -> None:
30 super().__init__()
31 self.hidden_size = config.hidden_size
@@ -41,18 +42,21 @@ class InternLM2VEDecoderLayer(nn.Module):
42 max_position_embeddings=max_position_embeddings,
43 cache_config=cache_config,
44 quant_config=quant_config,
45 + prefix=f"{prefix}.attention",
46 )
47 self.feed_forward = InternLM2MLP(
48 hidden_size=self.hidden_size,
49 intermediate_size=config.intermediate_size,
50 hidden_act=config.hidden_act,
51 quant_config=quant_config,
52 + prefix=f"{prefix}.feed_forward",
53 )
54 self.feed_forward_ve = InternLM2MLP(
55 hidden_size=self.hidden_size,
56 intermediate_size=config.intermediate_size,
57 hidden_act=config.hidden_act,
58 quant_config=quant_config,
59 + prefix=f"{prefix}.feed_forward_ve",
60 )
61 self.attention_norm = RMSNorm(config.hidden_size,
62 eps=config.rms_norm_eps)
@@ -111,8 +115,8 @@ class InternLM2VEModel(InternLM2Model):
115 super().__init__(config, cache_config, quant_config)
116 self.start_layer, self.end_layer, self.layers = make_layers(
117 config.num_hidden_layers,
114 - lambda prefix: InternLM2VEDecoderLayer(config, cache_config,
115 - quant_config),
118 + lambda prefix: InternLM2VEDecoderLayer(
119 + config, cache_config, quant_config, prefix=prefix),
120 prefix=f"{prefix}.layers")
121
122 def forward(
@@ -161,6 +165,10 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
165 config: PretrainedConfig,
166 cache_config: Optional[CacheConfig] = None,
167 quant_config: Optional[QuantizationConfig] = None,
168 + prefix: str = "",
169 ) -> None:
170 super().__init__(config, cache_config, quant_config)
166 - self.model = InternLM2VEModel(config, cache_config, quant_config)
171 + self.model = InternLM2VEModel(config,
172 + cache_config,
173 + quant_config,
174 + prefix=maybe_prefix(prefix, "model"))
vllm/model_executor/models/internvl.py
+4 -1
@@ -439,7 +439,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
439 )
440
441 self.language_model = init_vllm_registered_model(
442 - config.text_config, cache_config, quant_config)
442 + config.text_config,
443 + cache_config,
444 + quant_config,
445 + prefix="language_model")
446
447 self.mlp1 = self._init_mlp1(config)
448
vllm/model_executor/models/llama.py
+5 -2
@@ -55,7 +55,8 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
55
56 from .interfaces import SupportsLoRA, SupportsPP
57 from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
58 - make_empty_intermediate_tensors_factory, make_layers)
58 + make_empty_intermediate_tensors_factory, make_layers,
59 + maybe_prefix)
60
61
62 class LlamaMLP(nn.Module):
@@ -500,6 +501,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
501 cache_config: Optional[CacheConfig] = None,
502 quant_config: Optional[QuantizationConfig] = None,
503 lora_config: Optional[LoRAConfig] = None,
504 + prefix: str = "",
505 ) -> None:
506 super().__init__()
507
@@ -510,7 +512,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
512 cache_config,
513 quant_config,
514 lora_config=lora_config,
513 - prefix="model")
515 + prefix=maybe_prefix(prefix, "model"))
516 if get_pp_group().is_last_rank:
517 self.unpadded_vocab_size = config.vocab_size
518 if lora_config:
@@ -526,6 +528,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
528 if not lora_config else
529 lora_config.lora_vocab_padding_size),
530 quant_config=quant_config,
531 + prefix=maybe_prefix(prefix, "lm_head"),
532 )
533 if config.tie_word_embeddings:
534 self.lm_head = self.lm_head.tie_weights(
vllm/model_executor/models/llava.py
+15 -5
@@ -210,6 +210,7 @@ def init_vision_tower_for_llava(
210 quant_config: Optional[QuantizationConfig],
211 *,
212 require_post_norm: Optional[bool] = None,
213 + prefix: str = "",
214 ):
215 vision_config = hf_config.vision_config
216
@@ -224,23 +225,26 @@ def init_vision_tower_for_llava(
225 if isinstance(vision_config, CLIPVisionConfig):
226 return CLIPVisionModel(
227 vision_config,
227 - quant_config,
228 + quant_config=quant_config,
229 num_hidden_layers_override=num_hidden_layers,
230 require_post_norm=require_post_norm,
231 + prefix=prefix,
232 )
233 elif isinstance(vision_config, SiglipVisionConfig):
234 return SiglipVisionModel(
235 vision_config,
234 - quant_config,
236 + quant_config=quant_config,
237 num_hidden_layers_override=num_hidden_layers,
238 require_post_norm=require_post_norm,
239 + prefix=prefix,
240 )
241 elif isinstance(vision_config, PixtralVisionConfig):
242 return PixtralHFVisionModel(
243 vision_config,
241 - quant_config,
244 + quant_config=quant_config,
245 num_hidden_layers_override=num_hidden_layers,
246 require_post_norm=require_post_norm,
247 + prefix=prefix,
248 )
249
250 msg = f"Unsupported vision config: {type(vision_config)}"
@@ -274,14 +278,20 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
278
279 # TODO: Optionally initializes this for supporting embeddings.
280 self.vision_tower = init_vision_tower_for_llava(
277 - config, quant_config, require_post_norm=False)
281 + config,
282 + quant_config,
283 + require_post_norm=False,
284 + prefix="vision_tower")
285 self.multi_modal_projector = LlavaMultiModalProjector(
286 vision_hidden_size=config.vision_config.hidden_size,
287 text_hidden_size=config.text_config.hidden_size,
288 projector_hidden_act=config.projector_hidden_act)
289
290 self.language_model = init_vllm_registered_model(
284 - config.text_config, cache_config, quant_config)
291 + config.text_config,
292 + cache_config,
293 + quant_config,
294 + prefix="language_model")
295
296 self.make_empty_intermediate_tensors = (
297 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_next.py
+8 -2
@@ -293,7 +293,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
293
294 # TODO: Optionally initializes this for supporting embeddings.
295 self.vision_tower = init_vision_tower_for_llava(
296 - config, quant_config, require_post_norm=False)
296 + config,
297 + quant_config,
298 + require_post_norm=False,
299 + prefix="vision_tower")
300 self.image_newline = nn.Parameter(
301 torch.empty(config.text_config.hidden_size))
302 self.multi_modal_projector = LlavaMultiModalProjector(
@@ -302,7 +305,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
305 projector_hidden_act=config.projector_hidden_act)
306
307 self.language_model = init_vllm_registered_model(
305 - config.text_config, cache_config, quant_config)
308 + config.text_config,
309 + cache_config,
310 + quant_config,
311 + prefix="language_model")
312
313 # The same model class supports both language generation and embedding
314 # because the architecture name is the same
vllm/model_executor/models/llava_next_video.py
+8 -2
@@ -257,14 +257,20 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
257
258 # Initialize the vision tower only up to the required feature layer
259 self.vision_tower = init_vision_tower_for_llava(
260 - config, quant_config, require_post_norm=False)
260 + config,
261 + quant_config,
262 + require_post_norm=False,
263 + prefix="vision_tower")
264 self.vision_resampler = LlavaNextVideoPooler(config)
265 self.multi_modal_projector = LlavaNextMultiModalProjector(
266 vision_hidden_size=config.vision_config.hidden_size,
267 text_hidden_size=config.text_config.hidden_size,
268 projector_hidden_act=config.projector_hidden_act)
269 self.language_model = init_vllm_registered_model(
267 - config.text_config, cache_config, quant_config)
270 + config.text_config,
271 + cache_config,
272 + quant_config,
273 + prefix="language_model")
274
275 self.make_empty_intermediate_tensors = (
276 self.language_model.model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_onevision.py
+8 -2
@@ -415,10 +415,16 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
415
416 # Initialize the vision tower only up to the required feature layer
417 self.vision_tower = init_vision_tower_for_llava(
418 - config, quant_config, require_post_norm=False)
418 + config,
419 + quant_config,
420 + require_post_norm=False,
421 + prefix="vision_tower")
422 self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
423 self.language_model = init_vllm_registered_model(
421 - config.text_config, cache_config, quant_config)
424 + config.text_config,
425 + cache_config,
426 + quant_config,
427 + prefix="language_model")
428 self.image_newline = nn.Parameter(
429 torch.empty(config.text_config.hidden_size))
430
vllm/model_executor/models/minicpmv.py
+26 -8
@@ -394,8 +394,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
394 self.multimodal_config = multimodal_config
395
396 self.version = get_version_by_config(self.config)
397 - self.llm = self.init_llm(config, cache_config, quant_config)
398 - self.vpm = self.init_vision_module(config, quant_config)
397 + self.llm = self.init_llm(config,
398 + cache_config,
399 + quant_config,
400 + prefix="llm")
401 + self.vpm = self.init_vision_module(config, quant_config, prefix="vpm")
402 param_dtype = torch.get_default_dtype()
403 self.vpm.to(dtype=param_dtype)
404 self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
@@ -403,9 +406,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
406 self.embed_dim = self.config.hidden_size
407 self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
408 self.resampler.to(device="cuda", dtype=param_dtype)
409 + # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
410 self.lm_head = ParallelLMHead(config.vocab_size,
411 config.hidden_size,
408 - quant_config=quant_config)
412 + quant_config=quant_config,
413 + prefix="llm.lm_head")
414 self.logits_processor = LogitsProcessor(config.vocab_size)
415 self.sampler = Sampler()
416
@@ -644,6 +649,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
649 config: PretrainedConfig,
650 cache_config: Optional[CacheConfig] = None,
651 quant_config: Optional[QuantizationConfig] = None,
652 + prefix: str = "",
653 ) -> nn.Module:
654 raise NotImplementedError
655
@@ -651,6 +657,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
657 self,
658 config: PretrainedConfig,
659 quant_config: Optional[QuantizationConfig],
660 + prefix: str = "",
661 ) -> nn.Module:
662 raise NotImplementedError
663
@@ -690,17 +697,20 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
697 config: PretrainedConfig,
698 cache_config: Optional[CacheConfig] = None,
699 quant_config: Optional[QuantizationConfig] = None,
700 + prefix: str = "",
701 ) -> nn.Module:
702
703 return LLMWrapper(MiniCPMModel(config,
704 cache_config=cache_config,
697 - quant_config=quant_config),
705 + quant_config=quant_config,
706 + prefix=prefix),
707 name="model")
708
709 def init_vision_module(
710 self,
711 config: PretrainedConfig,
712 quant_config: Optional[QuantizationConfig],
713 + prefix: str = "",
714 ) -> nn.Module:
715 # TODO :refactor this vision model
716 try:
@@ -819,19 +829,23 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
829 config: PretrainedConfig,
830 cache_config: Optional[CacheConfig] = None,
831 quant_config: Optional[QuantizationConfig] = None,
832 + prefix: str = "",
833 ) -> nn.Module:
834 return LLMWrapper(LlamaModel(config,
835 cache_config=cache_config,
825 - quant_config=quant_config),
836 + quant_config=quant_config,
837 + prefix=prefix),
838 name="model")
839
840 def init_vision_module(
841 self,
842 config: PretrainedConfig,
843 quant_config: Optional[QuantizationConfig],
844 + prefix: str = "",
845 ) -> nn.Module:
846 model = Idefics2VisionTransformer(config.vision_config,
834 - quant_config=quant_config)
847 + quant_config=quant_config,
848 + prefix=prefix)
849 if self.config.drop_vision_last_layer:
850 model.encoder.layers = model.encoder.layers[:-1]
851 return model
@@ -935,20 +949,24 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
949 config: PretrainedConfig,
950 cache_config: Optional[CacheConfig] = None,
951 quant_config: Optional[QuantizationConfig] = None,
952 + prefix: str = "",
953 ) -> nn.Module:
954
955 return LLMWrapper(Qwen2Model(config,
956 cache_config=cache_config,
942 - quant_config=quant_config),
957 + quant_config=quant_config,
958 + prefix=prefix),
959 name="model")
960
961 def init_vision_module(
962 self,
963 config: PretrainedConfig,
964 quant_config: Optional[QuantizationConfig],
965 + prefix: str = "",
966 ) -> nn.Module:
967 model = Idefics2VisionTransformer(config.vision_config,
951 - quant_config=quant_config)
968 + quant_config=quant_config,
969 + prefix=prefix)
970 if self.config.drop_vision_last_layer:
971 model.encoder.layers = model.encoder.layers[:-1]
972 return model
vllm/model_executor/models/opt.py
+27 -7
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44 from .interfaces import SupportsPP
45 from .utils import (is_pp_missing_parameter,
46 - make_empty_intermediate_tensors_factory, make_layers)
46 + make_empty_intermediate_tensors_factory, make_layers,
47 + maybe_prefix)
48
49
50 class OPTLearnedPositionalEmbedding(nn.Embedding):
@@ -68,6 +69,7 @@ class OPTAttention(nn.Module):
69 bias: bool = True,
70 cache_config: Optional[CacheConfig] = None,
71 quant_config: Optional[QuantizationConfig] = None,
72 + prefix: str = "",
73 ) -> None:
74 super().__init__()
75 self.embed_dim = embed_dim
@@ -85,18 +87,21 @@ class OPTAttention(nn.Module):
87 total_num_heads,
88 bias=bias,
89 quant_config=quant_config,
90 + prefix=f"{prefix}.qkv_proj",
91 )
92 self.out_proj = RowParallelLinear(
93 embed_dim,
94 embed_dim,
95 bias=bias,
96 quant_config=quant_config,
97 + prefix=f"{prefix}.out_proj",
98 )
99 self.attn = Attention(self.num_heads,
100 self.head_dim,
101 scale=self.scaling,
102 cache_config=cache_config,
99 - quant_config=quant_config)
103 + quant_config=quant_config,
104 + prefix=f"{prefix}.attn")
105
106 def forward(
107 self,
@@ -118,6 +123,7 @@ class OPTDecoderLayer(nn.Module):
123 config: OPTConfig,
124 cache_config: Optional[CacheConfig] = None,
125 quant_config: Optional[QuantizationConfig] = None,
126 + prefix: str = "",
127 ):
128 super().__init__()
129 self.config = config
@@ -128,6 +134,7 @@ class OPTDecoderLayer(nn.Module):
134 bias=config.enable_bias,
135 cache_config=cache_config,
136 quant_config=quant_config,
137 + prefix=f"{prefix}.self_attn",
138 )
139 self.do_layer_norm_before = config.do_layer_norm_before
140
@@ -139,6 +146,7 @@ class OPTDecoderLayer(nn.Module):
146 config.ffn_dim,
147 bias=config.enable_bias,
148 quant_config=quant_config,
149 + prefix=f"{prefix}.fc1",
150 )
151 self.activation_fn = get_act_fn(config.activation_function,
152 quant_config, config.ffn_dim)
@@ -147,6 +155,7 @@ class OPTDecoderLayer(nn.Module):
155 self.embed_dim,
156 bias=config.enable_bias,
157 quant_config=quant_config,
158 + prefix=f"{prefix}.fc2",
159 )
160 self.final_layer_norm = nn.LayerNorm(
161 self.embed_dim,
@@ -214,7 +223,8 @@ class OPTDecoder(nn.Module):
223 self.project_out = ReplicatedLinear(config.hidden_size,
224 config.word_embed_proj_dim,
225 bias=False,
217 - quant_config=quant_config)
226 + quant_config=quant_config,
227 + prefix=f"{prefix}.project_out")
228 else:
229 self.project_out = None
230
@@ -222,7 +232,8 @@ class OPTDecoder(nn.Module):
232 self.project_in = ReplicatedLinear(config.word_embed_proj_dim,
233 config.hidden_size,
234 bias=False,
225 - quant_config=quant_config)
235 + quant_config=quant_config,
236 + prefix=f"{prefix}.project_in")
237 else:
238 self.project_in = None
239
@@ -239,7 +250,8 @@ class OPTDecoder(nn.Module):
250
251 self.start_layer, self.end_layer, self.layers = make_layers(
252 config.num_hidden_layers,
242 - lambda prefix: OPTDecoderLayer(config, cache_config, quant_config),
253 + lambda prefix: OPTDecoderLayer(
254 + config, cache_config, quant_config, prefix=prefix),
255 prefix=f"{prefix}.layers")
256
257 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -288,9 +300,13 @@ class OPTModel(nn.Module):
300 config: OPTConfig,
301 cache_config: Optional[CacheConfig] = None,
302 quant_config: Optional[QuantizationConfig] = None,
303 + prefix: str = "",
304 ):
305 super().__init__()
293 - self.decoder = OPTDecoder(config, cache_config, quant_config)
306 + self.decoder = OPTDecoder(config,
307 + cache_config,
308 + quant_config,
309 + prefix=f"{prefix}.decoder")
310 self.make_empty_intermediate_tensors = (
311 make_empty_intermediate_tensors_factory(["hidden_states"],
312 config.hidden_size))
@@ -335,11 +351,15 @@ class OPTForCausalLM(nn.Module, SupportsPP):
351 config: OPTConfig,
352 cache_config: Optional[CacheConfig] = None,
353 quant_config: Optional[QuantizationConfig] = None,
354 + prefix: str = "",
355 ):
356 super().__init__()
357 self.config = config
358 self.quant_config = quant_config
342 - self.model = OPTModel(config, cache_config, quant_config)
359 + self.model = OPTModel(config,
360 + cache_config,
361 + quant_config,
362 + prefix=maybe_prefix(prefix, "model"))
363 if self.config.tie_word_embeddings:
364 self.lm_head = self.model.decoder.embed_tokens
365 else:
vllm/model_executor/models/paligemma.py
+5 -2
@@ -143,14 +143,17 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
143 self.multimodal_config = multimodal_config
144
145 self.vision_tower = SiglipVisionModel(config.vision_config,
146 - quant_config)
146 + quant_config,
147 + prefix="vision_tower")
148 self.multi_modal_projector = PaliGemmaMultiModalProjector(
149 vision_hidden_size=config.vision_config.hidden_size,
150 projection_dim=config.vision_config.projection_dim)
151
152 self.quant_config = quant_config
153 self.language_model = GemmaForCausalLM(config.text_config,
153 - cache_config, quant_config)
154 + cache_config,
155 + quant_config,
156 + prefix="language_model")
157 logit_scale = getattr(config, "logit_scale", 1.0)
158 self.language_model.logits_processor.scale *= logit_scale
159
vllm/model_executor/models/phi3v.py
+14 -5
@@ -71,7 +71,8 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
71
72
73 def _init_img_processor(hf_config: PretrainedConfig,
74 - quant_config: Optional[QuantizationConfig]):
74 + quant_config: Optional[QuantizationConfig],
75 + prefix: str = "") -> CLIPVisionModel:
76 clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
77 layer_idx = hf_config.img_processor.get('layer_idx', -2)
78
@@ -86,6 +87,7 @@ def _init_img_processor(hf_config: PretrainedConfig,
87 clip_config,
88 quant_config,
89 num_hidden_layers_override=num_hidden_layers,
90 + prefix=prefix,
91 )
92
93 return img_processor
@@ -152,15 +154,18 @@ class Phi3ImageEmbeddingBase(nn.Module):
154 class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
155 """Phi3 Image embedding with HD transform."""
156
155 - def __init__(self, config: PretrainedConfig,
156 - quant_config: Optional[QuantizationConfig]) -> None:
157 + def __init__(self,
158 + config: PretrainedConfig,
159 + quant_config: Optional[QuantizationConfig],
160 + prefix: str = "") -> None:
161 super().__init__()
162
163 # n_embed or hidden_size
164 hidden_size = config.n_embd if hasattr(
165 config, 'n_embd') else config.hidden_size
166
163 - self.img_processor = _init_img_processor(config, quant_config)
167 + self.img_processor = _init_img_processor(
168 + config, quant_config, prefix=f"{prefix}.img_processor")
169
170 image_dim_out = config.img_processor['image_dim_out']
171 self.num_img_tokens = config.img_processor['num_img_tokens']
@@ -537,11 +542,15 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
542 config.hidden_size,
543 org_num_embeddings=config.vocab_size,
544 quant_config=quant_config,
545 + prefix="model.embed_tokens",
546 )
547
548 # TODO: Optionally initializes this for supporting input embeddings.
543 - self.vision_embed_tokens = Phi3HDImageEmbedding(config, quant_config)
549 + self.vision_embed_tokens = Phi3HDImageEmbedding(
550 + config, quant_config, prefix="model.vision_embed_tokens")
551
552 + # The prefix is empty intentionally because default prefix of
553 + # LlamaForCausalLM is "model"
554 self.language_model = LlamaForCausalLM(config, cache_config,
555 quant_config)
556
vllm/model_executor/models/pixtral.py
+4 -1
@@ -164,7 +164,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
164
165 # init MistralForCausalLM
166 self.language_model = init_vllm_registered_model(
167 - config.text_config, cache_config, quant_config)
167 + config.text_config,
168 + cache_config,
169 + quant_config,
170 + prefix="language_model")
171
172 self.vision_encoder = VisionTransformer(self.vision_args)
173 self.vision_language_adapter = VisionLanguageAdapter(
vllm/model_executor/models/qwen2.py
+37 -13
@@ -49,7 +49,8 @@ from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA, SupportsPP
51 from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
52 - make_empty_intermediate_tensors_factory, make_layers)
52 + make_empty_intermediate_tensors_factory, make_layers,
53 + maybe_prefix)
54
55
56 class Qwen2MLP(nn.Module):
@@ -60,16 +61,23 @@ class Qwen2MLP(nn.Module):
61 intermediate_size: int,
62 hidden_act: str,
63 quant_config: Optional[QuantizationConfig] = None,
64 + prefix: str = "",
65 ) -> None:
66 super().__init__()
67 self.gate_up_proj = MergedColumnParallelLinear(
66 - hidden_size, [intermediate_size] * 2,
68 + hidden_size,
69 + [intermediate_size] * 2,
70 + bias=False,
71 + quant_config=quant_config,
72 + prefix=f"{prefix}.gate_up_proj",
73 + )
74 + self.down_proj = RowParallelLinear(
75 + intermediate_size,
76 + hidden_size,
77 bias=False,
68 - quant_config=quant_config)
69 - self.down_proj = RowParallelLinear(intermediate_size,
70 - hidden_size,
71 - bias=False,
72 - quant_config=quant_config)
78 + quant_config=quant_config,
79 + prefix=f"{prefix}.down_proj",
80 + )
81 if hidden_act != "silu":
82 raise ValueError(f"Unsupported activation: {hidden_act}. "
83 "Only silu is supported for now.")
@@ -92,7 +100,8 @@ class Qwen2Attention(nn.Module):
100 rope_theta: float = 10000,
101 cache_config: Optional[CacheConfig] = None,
102 quant_config: Optional[QuantizationConfig] = None,
95 - rope_scaling: Optional[Tuple] = None) -> None:
103 + rope_scaling: Optional[Tuple] = None,
104 + prefix: str = "") -> None:
105 super().__init__()
106 self.hidden_size = hidden_size
107 tp_size = get_tensor_model_parallel_world_size()
@@ -122,12 +131,14 @@ class Qwen2Attention(nn.Module):
131 self.total_num_kv_heads,
132 bias=True,
133 quant_config=quant_config,
134 + prefix=f"{prefix}.qkv_proj",
135 )
136 self.o_proj = RowParallelLinear(
137 self.total_num_heads * self.head_dim,
138 hidden_size,
139 bias=False,
140 quant_config=quant_config,
141 + prefix=f"{prefix}.o_proj",
142 )
143
144 self.rotary_emb = get_rope(
@@ -142,7 +153,8 @@ class Qwen2Attention(nn.Module):
153 self.scaling,
154 num_kv_heads=self.num_kv_heads,
155 cache_config=cache_config,
145 - quant_config=quant_config)
156 + quant_config=quant_config,
157 + prefix=f"{prefix}.attn")
158
159 def forward(
160 self,
@@ -166,6 +178,7 @@ class Qwen2DecoderLayer(nn.Module):
178 config: Qwen2Config,
179 cache_config: Optional[CacheConfig] = None,
180 quant_config: Optional[QuantizationConfig] = None,
181 + prefix: str = "",
182 ) -> None:
183 super().__init__()
184 self.hidden_size = config.hidden_size
@@ -180,12 +193,15 @@ class Qwen2DecoderLayer(nn.Module):
193 rope_theta=rope_theta,
194 cache_config=cache_config,
195 quant_config=quant_config,
183 - rope_scaling=rope_scaling)
196 + rope_scaling=rope_scaling,
197 + prefix=f"{prefix}.self_attn",
198 + )
199 self.mlp = Qwen2MLP(
200 hidden_size=self.hidden_size,
201 intermediate_size=config.intermediate_size,
202 hidden_act=config.hidden_act,
203 quant_config=quant_config,
204 + prefix=f"{prefix}.mlp",
205 )
206 self.input_layernorm = RMSNorm(config.hidden_size,
207 eps=config.rms_norm_eps)
@@ -241,6 +257,7 @@ class Qwen2Model(nn.Module):
257 config.vocab_size,
258 config.hidden_size,
259 quant_config=quant_config,
260 + prefix=f"{prefix}.embed_tokens",
261 )
262 else:
263 self.embed_tokens = PPMissingLayer()
@@ -249,7 +266,8 @@ class Qwen2Model(nn.Module):
266 config.num_hidden_layers,
267 lambda prefix: Qwen2DecoderLayer(config=config,
268 cache_config=cache_config,
252 - quant_config=quant_config),
269 + quant_config=quant_config,
270 + prefix=f"{prefix}.layers"),
271 prefix=f"{prefix}.layers",
272 )
273
@@ -393,6 +411,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411 cache_config: Optional[CacheConfig] = None,
412 quant_config: Optional[QuantizationConfig] = None,
413 lora_config: Optional[LoRAConfig] = None,
414 + prefix: str = "",
415 ) -> None:
416 # TODO (@robertgshaw2): see if this can be moved out
417 if (cache_config.sliding_window is not None
@@ -412,14 +431,19 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
431 self.lora_config = lora_config
432
433 self.quant_config = quant_config
415 - self.model = Qwen2Model(config, cache_config, quant_config)
434 + self.model = Qwen2Model(config,
435 + cache_config,
436 + quant_config,
437 + prefix=maybe_prefix(prefix, "model"))
438
439 if config.tie_word_embeddings:
440 self.lm_head = self.model.embed_tokens
441 else:
442 self.lm_head = ParallelLMHead(config.vocab_size,
443 config.hidden_size,
422 - quant_config=quant_config)
444 + quant_config=quant_config,
445 + prefix=maybe_prefix(
446 + prefix, "lm_head"))
447
448 self.logits_processor = LogitsProcessor(config.vocab_size)
449 self.sampler = Sampler()
vllm/model_executor/models/qwen2_vl.py
+6 -2
@@ -938,7 +938,10 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
938 quant_config=None,
939 )
940
941 - self.model = Qwen2Model(config, cache_config, quant_config)
941 + self.model = Qwen2Model(config,
942 + cache_config,
943 + quant_config,
944 + prefix="model")
945
946 if get_pp_group().is_last_rank:
947 if config.tie_word_embeddings:
@@ -946,7 +949,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
949 else:
950 self.lm_head = ParallelLMHead(config.vocab_size,
951 config.hidden_size,
949 - quant_config=quant_config)
952 + quant_config=quant_config,
953 + prefix="lm_head")
954 else:
955 self.lm_head = PPMissingLayer()
956
vllm/model_executor/models/ultravox.py
+4 -1
@@ -357,7 +357,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
357 ))
358 self.multi_modal_projector = UltravoxProjector(config)
359 self.language_model = init_vllm_registered_model(
360 - config.text_config, cache_config, quant_config)
360 + config.text_config,
361 + cache_config,
362 + quant_config,
363 + prefix="language_model")
364 if config.text_model_id is not None:
365 self.secondary_weights.append(
366 DefaultModelLoader.Source(model_or_path=config.text_model_id,
vllm/model_executor/models/utils.py
+15
@@ -242,6 +242,7 @@ def init_vllm_registered_model(
242 lora_config: Optional[LoRAConfig] = None,
243 multimodal_config: Optional[MultiModalConfig] = None,
244 scheduler_config: Optional[SchedulerConfig] = None,
245 + prefix: str = "",
246 ) -> nn.Module:
247 """
248 Helper function to initialize an inner model registered to vLLM,
@@ -257,6 +258,7 @@ def init_vllm_registered_model(
258 lora_config=lora_config,
259 multimodal_config=multimodal_config,
260 scheduler_config=scheduler_config,
261 + prefix=prefix,
262 )
263
264
@@ -610,3 +612,16 @@ def get_vit_attn_backend() -> _Backend:
612 else:
613 selected_backend = _Backend.XFORMERS
614 return selected_backend
615 +
616 +
617 +def maybe_prefix(prefix: str, name: str) -> str:
618 + """Add a prefix to a name if the prefix is non-empty.
619 +
620 + Args:
621 + prefix: The prefix to add. If empty, no prefix will be added.
622 + name: The name to potentially prefix.
623 +
624 + Returns:
625 + The string "prefix.name" if prefix was non-empty, otherwise just "name".
626 + """
627 + return name if not prefix else f"{prefix}.{name}"