@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f89d18ff74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[6/N] pass whole config to inner model (#10205)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 10, 2024 at 22:41 UTC f89d18ff74e48f97c76afbab31956218d2486e36
69 files changed +681 -963
vllm/model_executor/models/arctic.py
+10 -13
@@ -34,7 +34,8 @@ from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35 from .interfaces import SupportsPP
36 from .utils import (is_pp_missing_parameter,
37 - make_empty_intermediate_tensors_factory, make_layers)
37 + make_empty_intermediate_tensors_factory, make_layers,
38 + maybe_prefix)
39
40 logger = init_logger(__name__)
41
@@ -364,14 +365,13 @@ class ArcticDecoderLayer(nn.Module):
365 @support_torch_compile
366 class ArcticModel(nn.Module):
367
367 - def __init__(
368 - self,
369 - config: ArcticConfig,
370 - cache_config: Optional[CacheConfig] = None,
371 - quant_config: Optional[QuantizationConfig] = None,
372 - prefix: str = "",
373 - ) -> None:
368 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
369 super().__init__()
370 +
371 + config = vllm_config.model_config.hf_config
372 + cache_config = vllm_config.cache_config
373 + quant_config = vllm_config.quant_config
374 +
375 self.padding_idx = config.pad_token_id
376 self.vocab_size = config.vocab_size
377 self.embed_tokens = VocabParallelEmbedding(
@@ -418,13 +418,10 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
418 def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
419 super().__init__()
420 config = vllm_config.model_config.hf_config
421 - cache_config = vllm_config.cache_config
421 quant_config = vllm_config.quant_config
422 self.config = config
424 - self.model = ArcticModel(config,
425 - cache_config,
426 - quant_config,
427 - prefix=prefix)
423 + self.model = ArcticModel(vllm_config=vllm_config,
424 + prefix=maybe_prefix(prefix, "model"))
425 self.vocab_size = config.vocab_size
426 self.lm_head = ParallelLMHead(
427 self.vocab_size,
vllm/model_executor/models/baichuan.py
+26 -22
@@ -253,13 +253,18 @@ class BaiChuanDecoderLayer(nn.Module):
253 @support_torch_compile
254 class BaiChuanModel(nn.Module):
255
256 - def __init__(self,
257 - config: PretrainedConfig,
258 - position_embedding: str,
259 - cache_config: Optional[CacheConfig] = None,
260 - quant_config: Optional[QuantizationConfig] = None,
261 - prefix: str = ""):
256 + def __init__(
257 + self,
258 + vllm_config: VllmConfig,
259 + prefix: str = "",
260 + position_embedding: str = "ROPE",
261 + ) -> None:
262 super().__init__()
263 +
264 + config = vllm_config.model_config.hf_config
265 + cache_config = vllm_config.cache_config
266 + quant_config = vllm_config.quant_config
267 +
268 self.config = config
269 self.padding_idx = config.pad_token_id
270 self.vocab_size = config.vocab_size
@@ -332,21 +337,22 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
337
338 def __init__(
339 self,
340 + *,
341 vllm_config: VllmConfig,
342 prefix: str = "",
343 position_embedding: str = "ROPE",
344 ):
345 super().__init__()
346 config = vllm_config.model_config.hf_config
341 - cache_config = vllm_config.cache_config
347 quant_config = vllm_config.quant_config
348 lora_config = vllm_config.lora_config
349 self.config = config
350 self.lora_config = lora_config
351
352 self.quant_config = quant_config
348 - self.model = BaiChuanModel(config, position_embedding, cache_config,
349 - quant_config)
353 + self.model = BaiChuanModel(vllm_config=vllm_config,
354 + prefix=prefix,
355 + position_embedding=position_embedding)
356 self.lm_head = ParallelLMHead(config.vocab_size,
357 config.hidden_size,
358 quant_config=quant_config)
@@ -438,16 +444,16 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
444 NOTE: the class name has a lower case 'c'.
445 """
446
441 - def __init__(
442 - self,
443 - vllm_config: VllmConfig,
444 - prefix: str = "",
445 - ):
447 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
448 config = vllm_config.model_config.hf_config
449 if config.hidden_size == 4096: # baichuan2 7b
448 - super().__init__(vllm_config, prefix, "ROPE")
450 + super().__init__(vllm_config=vllm_config,
451 + prefix=prefix,
452 + position_embedding="ROPE")
453 else: # baichuan 13b, baichuan2 13b
450 - super().__init__(vllm_config, prefix, "ALIBI")
454 + super().__init__(vllm_config=vllm_config,
455 + prefix=prefix,
456 + position_embedding="ALIBI")
457
458
459 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -455,9 +461,7 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
461 NOTE: the class name has an upper case 'C'.
462 """
463
458 - def __init__(
459 - self,
460 - vllm_config: VllmConfig,
461 - prefix: str = "",
462 - ):
463 - super().__init__(vllm_config, prefix, "ROPE")
464 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
465 + super().__init__(vllm_config=vllm_config,
466 + prefix=prefix,
467 + position_embedding="ROPE")
vllm/model_executor/models/bart.py
+11 -12
@@ -41,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors
43
44 +from .utils import maybe_prefix
45 +
46 logger = logging.get_logger(__name__)
47
48
@@ -739,13 +741,14 @@ class BartModel(nn.Module):
741 "encoder.embed_tokens.weight", "decoder.embed_tokens.weight"
742 ]
743
742 - def __init__(self,
743 - config: BartConfig,
744 - cache_config: Optional[CacheConfig] = None,
745 - quant_config: Optional[QuantizationConfig] = None,
746 - lora_config: Optional[LoRAConfig] = None):
744 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
745 super().__init__()
746
747 + config = vllm_config.model_config.hf_config
748 + cache_config = vllm_config.cache_config
749 + quant_config = vllm_config.quant_config
750 + lora_config = vllm_config.lora_config
751 +
752 self.config = config
753
754 self.padding_idx = config.pad_token_id
@@ -810,20 +813,16 @@ class BartModel(nn.Module):
813 class BartForConditionalGeneration(nn.Module):
814 base_model_prefix = "model"
815
813 - def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
816 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
817
818 super().__init__()
819 config = vllm_config.model_config.hf_config
817 - cache_config = vllm_config.cache_config
818 - quant_config = vllm_config.quant_config
820 lora_config = vllm_config.lora_config
821 # currently all existing BART models have `tie_word_embeddings` enabled
822 assert config.tie_word_embeddings
823 self.config = config
823 - self.model = BartModel(config,
824 - cache_config,
825 - quant_config,
826 - lora_config=lora_config)
824 + self.model = BartModel(vllm_config=vllm_config,
825 + prefix=maybe_prefix(prefix, "model"))
826
827 self.unpadded_vocab_size = config.vocab_size
828 if lora_config:
vllm/model_executor/models/bert.py
+11 -14
@@ -21,6 +21,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.model_executor.pooling_metadata import PoolingMetadata
22 from vllm.sequence import IntermediateTensors, PoolerOutput
23
24 +from .utils import maybe_prefix
25 +
26
27 class BertEmbedding(nn.Module):
28
@@ -309,12 +311,13 @@ class BertOutput(nn.Module):
311
312 class BertModel(nn.Module):
313
312 - def __init__(self,
313 - config: BertConfig,
314 - cache_config: Optional[CacheConfig] = None,
315 - quant_config: Optional[QuantizationConfig] = None,
316 - prefix: str = ""):
314 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
315 super().__init__()
316 +
317 + config = vllm_config.model_config.hf_config
318 + cache_config = vllm_config.cache_config
319 + quant_config = vllm_config.quant_config
320 +
321 self.embeddings = BertEmbedding(config)
322 self.encoder = BertEncoder(config,
323 cache_config,
@@ -382,17 +385,11 @@ class BertEmbeddingModel(nn.Module):
385 _pooler: An instance of Pooler used for pooling operations.
386 """
387
385 - def __init__(
386 - self,
387 - vllm_config: VllmConfig,
388 - prefix: str = "",
389 - ) -> None:
388 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
389 super().__init__()
391 - config = vllm_config.model_config.hf_config
392 - cache_config = vllm_config.cache_config
393 - quant_config = vllm_config.quant_config
390 pooler_config = vllm_config.model_config.pooler_config
395 - self.model = BertModel(config, cache_config, quant_config)
391 + self.model = BertModel(vllm_config=vllm_config,
392 + prefix=maybe_prefix(prefix, "model"))
393 self._pooler = Pooler.from_config_with_defaults(
394 pooler_config,
395 pooling_type=PoolingType.CLS,
vllm/model_executor/models/blip2.py
+3 -7
@@ -23,7 +23,7 @@ from .blip import (BlipVisionModel, dummy_image_for_blip,
23 get_max_blip_image_tokens)
24 from .interfaces import SupportsMultiModal, SupportsPP
25 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
26 - merge_multimodal_embeddings)
26 + maybe_prefix, merge_multimodal_embeddings)
27
28 # We use this internally as placeholders since there is no image token
29 # defined on the HuggingFace repo
@@ -483,11 +483,7 @@ def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
483 @INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
484 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
485
486 - def __init__(
487 - self,
488 - vllm_config: VllmConfig,
489 - prefix: str = "",
490 - ) -> None:
486 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
487
488 super().__init__()
489 config = vllm_config.model_config.hf_config
@@ -517,7 +513,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
513 self.language_model = init_vllm_registered_model(
514 config.text_config,
515 vllm_config=vllm_config,
520 - prefix="language_model")
516 + prefix=maybe_prefix(prefix, "language_model"))
517
518 self.make_empty_intermediate_tensors = (
519 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/bloom.py
+11 -10
@@ -42,7 +42,8 @@ from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsPP
44 from .utils import (is_pp_missing_parameter,
45 - make_empty_intermediate_tensors_factory, make_layers)
45 + make_empty_intermediate_tensors_factory, make_layers,
46 + maybe_prefix)
47
48
49 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
@@ -221,14 +222,13 @@ class BloomBlock(nn.Module):
222 @support_torch_compile
223 class BloomModel(nn.Module):
224
224 - def __init__(
225 - self,
226 - config: BloomConfig,
227 - cache_config: Optional[CacheConfig] = None,
228 - quant_config: Optional[QuantizationConfig] = None,
229 - prefix: str = "",
230 - ):
225 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
226 super().__init__()
227 +
228 + config = vllm_config.model_config.hf_config
229 + cache_config = vllm_config.cache_config
230 + quant_config = vllm_config.quant_config
231 +
232 self.embed_dim = config.hidden_size
233
234 # Embedding + LN Embedding
@@ -288,11 +288,12 @@ class BloomForCausalLM(nn.Module, SupportsPP):
288 ):
289 super().__init__()
290 config = vllm_config.model_config.hf_config
291 - cache_config = vllm_config.cache_config
291 quant_config = vllm_config.quant_config
292 self.config = config
293 self.quant_config = quant_config
295 - self.transformer = BloomModel(config, cache_config, quant_config)
294 + self.transformer = BloomModel(vllm_config=vllm_config,
295 + prefix=maybe_prefix(
296 + prefix, "transformer"))
297 if self.config.tie_word_embeddings:
298 self.lm_head = self.transformer.word_embeddings
299 else:
vllm/model_executor/models/chameleon.py
+11 -16
@@ -37,7 +37,8 @@ from vllm.utils import print_warning_once
37
38 from .interfaces import SupportsMultiModal, SupportsPP
39 from .utils import (is_pp_missing_parameter,
40 - make_empty_intermediate_tensors_factory, make_layers)
40 + make_empty_intermediate_tensors_factory, make_layers,
41 + maybe_prefix)
42
43 # These configs are not part of the model config but the preprocessor
44 # and processor files, so we hardcode them in the model file for now.
@@ -831,14 +832,13 @@ class ChameleonImageVocabularyMapping:
832
833 class ChameleonModel(nn.Module):
834
834 - def __init__(
835 - self,
836 - config: ChameleonConfig,
837 - cache_config: Optional[CacheConfig] = None,
838 - quant_config: Optional[QuantizationConfig] = None,
839 - prefix: str = "",
840 - ) -> None:
835 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
836 super().__init__()
837 +
838 + config = vllm_config.model_config.hf_config
839 + cache_config = vllm_config.cache_config
840 + quant_config = vllm_config.quant_config
841 +
842 self.config = config
843 self.padding_idx = config.pad_token_id
844 self.vocab_size = config.vocab_size
@@ -924,19 +924,14 @@ class ChameleonModel(nn.Module):
924 class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
925 SupportsPP):
926
927 - def __init__(
928 - self,
929 - vllm_config: VllmConfig,
930 - prefix: str = "",
931 - ) -> None:
927 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
928 super().__init__()
929 config = vllm_config.model_config.hf_config
934 - cache_config = vllm_config.cache_config
935 - quant_config = vllm_config.quant_config
930 multimodal_config = vllm_config.model_config.multimodal_config
931 self.config = config
932 self.multimodal_config = multimodal_config
939 - self.model = ChameleonModel(config, cache_config, quant_config)
933 + self.model = ChameleonModel(vllm_config=vllm_config,
934 + prefix=maybe_prefix(prefix, "model"))
935 self.unpadded_vocab_size = config.vocab_size
936 self.lm_head = ParallelLMHead(
937 self.unpadded_vocab_size,
vllm/model_executor/models/chatglm.py
+10 -9
@@ -39,7 +39,8 @@ from vllm.transformers_utils.configs import ChatGLMConfig
39
40 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
41 from .utils import (is_pp_missing_parameter,
42 - make_empty_intermediate_tensors_factory, make_layers)
42 + make_empty_intermediate_tensors_factory, make_layers,
43 + maybe_prefix)
44
45 logger = init_logger(__name__)
46
@@ -481,14 +482,13 @@ class GLMTransformer(nn.Module):
482
483 class ChatGLMModel(nn.Module):
484
484 - def __init__(
485 - self,
486 - config: ChatGLMConfig,
487 - cache_config: Optional[CacheConfig] = None,
488 - quant_config: Optional[QuantizationConfig] = None,
489 - ):
485 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
486 super().__init__()
487
488 + config = vllm_config.model_config.hf_config
489 + cache_config = vllm_config.cache_config
490 + quant_config = vllm_config.quant_config
491 +
492 self.config = config
493
494 self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
@@ -600,7 +600,6 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
600 ):
601 super().__init__()
602 config = vllm_config.model_config.hf_config
603 - cache_config = vllm_config.cache_config
603 quant_config = vllm_config.quant_config
604 lora_config = vllm_config.lora_config
605 multimodal_config = vllm_config.model_config.multimodal_config
@@ -611,7 +610,9 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
610 self.quant_config = quant_config
611 self.max_position_embeddings = getattr(config, "max_sequence_length",
612 8192)
614 - self.transformer = ChatGLMModel(config, cache_config, quant_config)
613 + self.transformer = ChatGLMModel(vllm_config=vllm_config,
614 + prefix=maybe_prefix(
615 + prefix, "transformer"))
616 if self.config.tie_word_embeddings:
617 self.transformer.output_layer.weight = (
618 self.transformer.embedding.weight)
vllm/model_executor/models/commandr.py
+13 -20
@@ -28,7 +28,7 @@ from transformers import CohereConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -49,7 +49,8 @@ from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA, SupportsPP
51 from .utils import (is_pp_missing_parameter,
52 - make_empty_intermediate_tensors_factory, make_layers)
52 + make_empty_intermediate_tensors_factory, make_layers,
53 + maybe_prefix)
54
55
56 @torch.compile
@@ -253,15 +254,14 @@ class CohereDecoderLayer(nn.Module):
254 @support_torch_compile
255 class CohereModel(nn.Module):
256
256 - def __init__(
257 - self,
258 - config: CohereConfig,
259 - cache_config: Optional[CacheConfig] = None,
260 - quant_config: Optional[QuantizationConfig] = None,
261 - lora_config: Optional[LoRAConfig] = None,
262 - prefix: str = "",
263 - ):
257 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
258 super().__init__()
259 +
260 + config = vllm_config.model_config.hf_config
261 + cache_config = vllm_config.cache_config
262 + quant_config = vllm_config.quant_config
263 + lora_config = vllm_config.lora_config
264 +
265 self.config = config
266 lora_vocab = (lora_config.lora_extra_vocab_size *
267 (lora_config.max_loras or 1)) if lora_config else 0
@@ -332,14 +332,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332 embedding_modules = {"embed_tokens": "input_embeddings"}
333 embedding_padding_modules = []
334
335 - def __init__(
336 - self,
337 - vllm_config: VllmConfig,
338 - prefix: str = "",
339 - ) -> None:
335 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
336 super().__init__()
337 config = vllm_config.model_config.hf_config
342 - cache_config = vllm_config.cache_config
338 quant_config = vllm_config.quant_config
339 lora_config = vllm_config.lora_config
340 self.config = config
@@ -353,10 +348,8 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
348 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
349 config.vocab_size,
350 scale=config.logit_scale)
356 - self.model = CohereModel(config,
357 - cache_config,
358 - quant_config,
359 - lora_config=lora_config)
351 + self.model = CohereModel(vllm_config=vllm_config,
352 + prefix=maybe_prefix(prefix, "model"))
353 self.sampler = get_sampler()
354 self.make_empty_intermediate_tensors = (
355 self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/dbrx.py
+11 -10
@@ -25,7 +25,8 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
25
26 from .interfaces import SupportsPP
27 from .utils import (is_pp_missing_parameter,
28 - make_empty_intermediate_tensors_factory, make_layers)
28 + make_empty_intermediate_tensors_factory, make_layers,
29 + maybe_prefix)
30
31
32 class DbrxRouter(nn.Module):
@@ -294,14 +295,13 @@ class DbrxBlock(nn.Module):
295
296 class DbrxModel(nn.Module):
297
297 - def __init__(
298 - self,
299 - config: DbrxConfig,
300 - cache_config: Optional[CacheConfig] = None,
301 - quant_config: Optional[QuantizationConfig] = None,
302 - prefix: str = "",
303 - ):
298 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
299 super().__init__()
300 +
301 + config = vllm_config.model_config.hf_config
302 + cache_config = vllm_config.cache_config
303 + quant_config = vllm_config.quant_config
304 +
305 self.wte = VocabParallelEmbedding(
306 config.vocab_size,
307 config.d_model,
@@ -357,7 +357,6 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
357 ):
358 super().__init__()
359 config = vllm_config.model_config.hf_config
360 - cache_config = vllm_config.cache_config
360 quant_config = vllm_config.quant_config
361 self.config = config
362 if config.tie_word_embeddings:
@@ -365,7 +364,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
364 "tie_word_embeddings is not supported for Dbrx models.")
365 self.quant_config = quant_config
366 self.unpadded_vocab_size = config.vocab_size
368 - self.transformer = DbrxModel(config, cache_config, quant_config)
367 + self.transformer = DbrxModel(vllm_config=vllm_config,
368 + prefix=maybe_prefix(
369 + prefix, "transformer"))
370 self.lm_head = ParallelLMHead(
371 config.vocab_size,
372 config.d_model,
vllm/model_executor/models/decilm.py
+1 -5
@@ -51,11 +51,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
51 instead.
52 """
53
54 - def __init__(
55 - self,
56 - vllm_config: VllmConfig,
57 - prefix: str = "",
58 - ) -> None:
54 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
55 config = vllm_config.model_config.hf_config
56 config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
57 delattr(config, "num_key_value_heads_per_layer")
vllm/model_executor/models/deepseek.py
+11 -15
@@ -50,7 +50,8 @@ from vllm.sequence import IntermediateTensors
50
51 from .interfaces import SupportsPP
52 from .utils import (is_pp_missing_parameter,
53 - make_empty_intermediate_tensors_factory, make_layers)
53 + make_empty_intermediate_tensors_factory, make_layers,
54 + maybe_prefix)
55
56
57 class DeepseekMLP(nn.Module):
@@ -326,14 +327,13 @@ class DeepseekModel(nn.Module):
327
328 fall_back_to_pt_during_load = False
329
329 - def __init__(
330 - self,
331 - config: PretrainedConfig,
332 - cache_config: Optional[CacheConfig] = None,
333 - quant_config: Optional[QuantizationConfig] = None,
334 - prefix: str = "",
335 - ) -> None:
330 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
331 super().__init__()
332 +
333 + config = vllm_config.model_config.hf_config
334 + cache_config = vllm_config.cache_config
335 + quant_config = vllm_config.quant_config
336 +
337 self.padding_idx = config.pad_token_id
338 self.vocab_size = config.vocab_size
339
@@ -383,18 +383,14 @@ class DeepseekModel(nn.Module):
383
384 class DeepseekForCausalLM(nn.Module, SupportsPP):
385
386 - def __init__(
387 - self,
388 - vllm_config: VllmConfig,
389 - prefix: str = "",
390 - ) -> None:
386 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
387 super().__init__()
388 config = vllm_config.model_config.hf_config
393 - cache_config = vllm_config.cache_config
389 quant_config = vllm_config.quant_config
390 self.config = config
391 self.quant_config = quant_config
397 - self.model = DeepseekModel(config, cache_config, quant_config)
392 + self.model = DeepseekModel(vllm_config=vllm_config,
393 + prefix=maybe_prefix(prefix, "model"))
394 self.lm_head = ParallelLMHead(config.vocab_size,
395 config.hidden_size,
396 quant_config=quant_config)
vllm/model_executor/models/deepseek_v2.py
+11 -18
@@ -51,7 +51,8 @@ from vllm.sequence import IntermediateTensors
51
52 from .interfaces import SupportsPP
53 from .utils import (PPMissingLayer, is_pp_missing_parameter,
54 - make_empty_intermediate_tensors_factory, make_layers)
54 + make_empty_intermediate_tensors_factory, make_layers,
55 + maybe_prefix)
56
57
58 class DeepseekV2MLP(nn.Module):
@@ -408,14 +409,13 @@ class DeepseekV2Model(nn.Module):
409
410 fall_back_to_pt_during_load = False
411
411 - def __init__(
412 - self,
413 - config: PretrainedConfig,
414 - cache_config: Optional[CacheConfig] = None,
415 - quant_config: Optional[QuantizationConfig] = None,
416 - prefix: str = "",
417 - ) -> None:
412 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
413 super().__init__()
414 +
415 + config = vllm_config.model_config.hf_config
416 + cache_config = vllm_config.cache_config
417 + quant_config = vllm_config.quant_config
418 +
419 self.padding_idx = config.pad_token_id
420 self.vocab_size = config.vocab_size
421
@@ -479,21 +479,14 @@ class DeepseekV2Model(nn.Module):
479
480 class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
481
482 - def __init__(
483 - self,
484 - vllm_config: VllmConfig,
485 - prefix: str = "",
486 - ) -> None:
482 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
483 super().__init__()
484 config = vllm_config.model_config.hf_config
489 - cache_config = vllm_config.cache_config
485 quant_config = vllm_config.quant_config
486 self.config = config
487 self.quant_config = quant_config
493 - self.model = DeepseekV2Model(config,
494 - cache_config,
495 - quant_config,
496 - prefix="model")
488 + self.model = DeepseekV2Model(vllm_config=vllm_config,
489 + prefix=maybe_prefix(prefix, "model"))
490 self.lm_head = ParallelLMHead(config.vocab_size,
491 config.hidden_size,
492 quant_config=quant_config)
vllm/model_executor/models/eagle.py
+4 -1
@@ -14,6 +14,8 @@ from vllm.model_executor.models import ModelRegistry
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 from vllm.sequence import IntermediateTensors
16
17 +from .utils import maybe_prefix
18 +
19
20 class EAGLE(nn.Module):
21 """This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077
@@ -42,7 +44,8 @@ class EAGLE(nn.Module):
44 architectures = getattr(self.config.model, "architectures", [])
45 model_cls, _ = ModelRegistry.resolve_model_cls(architectures)
46
45 - self.model = model_cls(vllm_config, prefix)
47 + self.model = model_cls(vllm_config=vllm_config,
48 + prefix=maybe_prefix(prefix, "model"))
49 self.fc = nn.Linear(config.model.hidden_size * 2,
50 config.model.hidden_size,
51 bias=getattr(self.config, "eagle_fc_bias", False))
vllm/model_executor/models/exaone.py
+13 -21
@@ -29,7 +29,7 @@ from torch import nn
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 +from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -54,7 +54,8 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
54
55 from .interfaces import SupportsLoRA, SupportsPP
56 from .utils import (PPMissingLayer, is_pp_missing_parameter,
57 - make_empty_intermediate_tensors_factory, make_layers)
57 + make_empty_intermediate_tensors_factory, make_layers,
58 + maybe_prefix)
59
60
61 class ExaoneGatedMLP(nn.Module):
@@ -314,15 +315,14 @@ class ExaoneDecoderLayer(nn.Module):
315 @support_torch_compile
316 class ExaoneModel(nn.Module):
317
317 - def __init__(
318 - self,
319 - config: ExaoneConfig,
320 - cache_config: Optional[CacheConfig] = None,
321 - quant_config: Optional[QuantizationConfig] = None,
322 - lora_config: Optional[LoRAConfig] = None,
323 - prefix: str = "",
324 - ) -> None:
318 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
319 super().__init__()
320 +
321 + config = vllm_config.model_config.hf_config
322 + cache_config = vllm_config.cache_config
323 + quant_config = vllm_config.quant_config
324 + lora_config = vllm_config.lora_config
325 +
326 self.config = config
327 self.padding_idx = config.pad_token_id
328 lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -438,14 +438,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
438 "c_fc_1": ("gate_up_proj", 1),
439 }
440
441 - def __init__(
442 - self,
443 - vllm_config: VllmConfig,
444 - prefix: str = "",
445 - ) -> None:
441 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
442 super().__init__()
443 config = vllm_config.model_config.hf_config
448 - cache_config = vllm_config.cache_config
444 quant_config = vllm_config.quant_config
445 lora_config = vllm_config.lora_config
446
@@ -453,11 +448,8 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
448 self.lora_config = lora_config
449
450 self.transformer = ExaoneModel(
456 - config,
457 - cache_config,
458 - quant_config,
459 - lora_config=lora_config,
460 - prefix="model",
451 + vllm_config=vllm_config,
452 + prefix=maybe_prefix(prefix, "model"),
453 )
454 if get_pp_group().is_last_rank:
455 self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/falcon.py
+11 -10
@@ -48,7 +48,8 @@ from vllm.transformers_utils.configs import RWConfig
48
49 from .interfaces import SupportsPP
50 from .utils import (is_pp_missing_parameter,
51 - make_empty_intermediate_tensors_factory, make_layers)
51 + make_empty_intermediate_tensors_factory, make_layers,
52 + maybe_prefix)
53
54 FalconConfig = Union[HF_FalconConfig, RWConfig]
55
@@ -332,14 +333,13 @@ class FalconDecoderLayer(nn.Module):
333 @support_torch_compile
334 class FalconModel(nn.Module):
335
335 - def __init__(
336 - self,
337 - config: FalconConfig,
338 - cache_config: Optional[CacheConfig] = None,
339 - quant_config: Optional[QuantizationConfig] = None,
340 - prefix: str = "",
341 - ):
336 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
337 super().__init__()
338 +
339 + config = vllm_config.model_config.hf_config
340 + cache_config = vllm_config.cache_config
341 + quant_config = vllm_config.quant_config
342 +
343 self.config = config
344 self.embed_dim = config.hidden_size
345 self.num_heads = config.num_attention_heads
@@ -408,11 +408,12 @@ class FalconForCausalLM(nn.Module, SupportsPP):
408 ):
409 super().__init__()
410 config = vllm_config.model_config.hf_config
411 - cache_config = vllm_config.cache_config
411 quant_config = vllm_config.quant_config
412 self.config = config
413 self.quant_config = quant_config
415 - self.transformer = FalconModel(config, cache_config, quant_config)
414 + self.transformer = FalconModel(vllm_config=vllm_config,
415 + prefix=maybe_prefix(
416 + prefix, "transformer"))
417 # only Falcon-11B doesn't share lm_head weight with word embeddings
418 # and previous Falcon model doesn't have tie_word_embeddings config
419 # so we set tie_word_embeddings to True by default
vllm/model_executor/models/florence2.py
+17 -21
@@ -3,13 +3,10 @@ from typing import Iterable, List, Optional, Tuple
3
4 import torch
5 import torch.nn as nn
6 -from transformers import PretrainedConfig
6
7 from vllm.attention import AttentionMetadata
9 -from vllm.config import CacheConfig, VllmConfig
8 +from vllm.config import VllmConfig
9 from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 -from vllm.model_executor.layers.quantization.base_config import (
12 - QuantizationConfig)
10 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
11 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
12 from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
@@ -23,11 +20,13 @@ from .utils import AutoWeightsLoader
20
21 class Florence2LanguageModel(nn.Module):
22
26 - def __init__(self,
27 - config: PretrainedConfig,
28 - cache_config: Optional[CacheConfig] = None,
29 - quant_config: Optional[QuantizationConfig] = None):
23 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
24 super().__init__()
25 +
26 + config = vllm_config.model_config.hf_config
27 + cache_config = vllm_config.cache_config
28 + quant_config = vllm_config.quant_config
29 +
30 self.config = config
31
32 self.padding_idx = config.pad_token_id
@@ -93,15 +92,14 @@ class Florence2LanguageModel(nn.Module):
92
93 class Florence2LanguageForConditionalGeneration(nn.Module):
94
96 - def __init__(self,
97 - config: PretrainedConfig,
98 - cache_config: Optional[CacheConfig] = None,
99 - quant_config: Optional[QuantizationConfig] = None):
95 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
96 super().__init__()
97 +
98 + config = vllm_config.model_config.hf_config
99 +
100 self.config = config
102 - self.model = Florence2LanguageModel(config,
103 - cache_config=cache_config,
104 - quant_config=quant_config)
101 + self.model = Florence2LanguageModel(vllm_config=vllm_config,
102 + prefix=prefix)
103 embed_scale = math.sqrt(
104 config.d_model) if config.scale_embedding else 1.0
105
@@ -189,17 +187,15 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
187
188 class Florence2ForConditionalGeneration(nn.Module):
189
192 - def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
190 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
191 super().__init__()
192 config = vllm_config.model_config.hf_config
195 - cache_config = vllm_config.cache_config
196 - quant_config = vllm_config.quant_config
193
194 # TODO(Isotr0py): Add vision backbone
195 self.language_model = Florence2LanguageForConditionalGeneration(
200 - config=config.text_config,
201 - cache_config=cache_config,
202 - quant_config=quant_config)
196 + vllm_config=vllm_config.with_hf_config(config.text_config),
197 + prefix=prefix,
198 + )
199
200 @property
201 def sampler(self):
vllm/model_executor/models/gemma.py
+8 -16
@@ -258,14 +258,13 @@ class GemmaDecoderLayer(nn.Module):
258 @support_torch_compile
259 class GemmaModel(nn.Module):
260
261 - def __init__(
262 - self,
263 - config: GemmaConfig,
264 - cache_config: Optional[CacheConfig] = None,
265 - quant_config: Optional[QuantizationConfig] = None,
266 - prefix: str = "",
267 - ) -> None:
261 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
262 super().__init__()
263 +
264 + config = vllm_config.model_config.hf_config
265 + cache_config = vllm_config.cache_config
266 + quant_config = vllm_config.quant_config
267 +
268 self.config = config
269
270 self.embed_tokens = VocabParallelEmbedding(
@@ -372,14 +371,9 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
371 embedding_modules = {}
372 embedding_padding_modules = []
373
375 - def __init__(
376 - self,
377 - vllm_config: VllmConfig,
378 - prefix: str = "",
379 - ) -> None:
374 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
375 super().__init__()
376 config = vllm_config.model_config.hf_config
382 - cache_config = vllm_config.cache_config
377 quant_config = vllm_config.quant_config
378 lora_config = vllm_config.lora_config
379
@@ -389,9 +383,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
383 self.lora_config = lora_config
384
385 self.quant_config = quant_config
392 - self.model = GemmaModel(config,
393 - cache_config,
394 - quant_config,
386 + self.model = GemmaModel(vllm_config=vllm_config,
387 prefix=maybe_prefix(prefix, "model"))
388 self.logits_processor = LogitsProcessor(config.vocab_size)
389 self.sampler = get_sampler()
vllm/model_executor/models/gemma2.py
+9 -19
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
43
44 from .interfaces import SupportsLoRA, SupportsPP
45 from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
46 - make_empty_intermediate_tensors_factory, make_layers)
46 + make_empty_intermediate_tensors_factory, make_layers,
47 + maybe_prefix)
48
49 logger = init_logger(__name__)
50
@@ -243,11 +244,7 @@ class Gemma2DecoderLayer(nn.Module):
244 @support_torch_compile
245 class Gemma2Model(nn.Module):
246
246 - def __init__(
247 - self,
248 - vllm_config: VllmConfig,
249 - prefix: str = "",
250 - ) -> None:
247 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
248 super().__init__()
249 config = vllm_config.model_config.hf_config
250 cache_config = vllm_config.cache_config
@@ -399,13 +396,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
396 "up_proj": ("gate_up_proj", 1),
397 }
398
402 - def __init__(
403 - self,
404 - vllm_config: VllmConfig,
405 - prefix: str = "",
406 - ) -> None:
399 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
400 config = vllm_config.model_config.hf_config
408 - cache_config = vllm_config.cache_config
401 quant_config = vllm_config.quant_config
402 lora_config = vllm_config.lora_config
403 del lora_config # Unused.
@@ -414,7 +406,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
406 # currently all existing Gemma models have `tie_word_embeddings` enabled
407 assert config.tie_word_embeddings
408 self.quant_config = quant_config
417 - self.model = Gemma2Model(config, cache_config, quant_config)
409 + self.model = Gemma2Model(vllm_config=vllm_config,
410 + prefix=maybe_prefix(prefix, "model"))
411 self.logits_processor = LogitsProcessor(
412 config.vocab_size, soft_cap=config.final_logit_softcapping)
413 self.sampler = get_sampler()
@@ -471,14 +464,11 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
464 _pooler: An instance of Pooler used for pooling operations.
465 """
466
474 - def __init__(
475 - self,
476 - vllm_config: VllmConfig,
477 - prefix: str = "",
478 - ) -> None:
467 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
468 super().__init__()
469
481 - self.model = Gemma2Model(vllm_config, prefix)
470 + self.model = Gemma2Model(vllm_config=vllm_config,
471 + prefix=maybe_prefix(prefix, "model"))
472 self._pooler = Pooler.from_config_with_defaults(
473 vllm_config.model_config.pooler_config,
474 pooling_type=PoolingType.LAST,
vllm/model_executor/models/gpt2.py
+11 -13
@@ -42,7 +42,8 @@ from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsPP
44 from .utils import (is_pp_missing_parameter,
45 - make_empty_intermediate_tensors_factory, make_layers)
45 + make_empty_intermediate_tensors_factory, make_layers,
46 + maybe_prefix)
47
48
49 class GPT2Attention(nn.Module):
@@ -184,14 +185,13 @@ class GPT2Block(nn.Module):
185 @support_torch_compile
186 class GPT2Model(nn.Module):
187
187 - def __init__(
188 - self,
189 - config: GPT2Config,
190 - cache_config: Optional[CacheConfig] = None,
191 - quant_config: Optional[QuantizationConfig] = None,
192 - prefix: str = "",
193 - ):
188 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
189 super().__init__()
190 +
191 + config = vllm_config.model_config.hf_config
192 + cache_config = vllm_config.cache_config
193 + quant_config = vllm_config.quant_config
194 +
195 self.config = config
196 assert not config.add_cross_attention
197 assert not config.scale_attn_by_inverse_layer_idx
@@ -247,14 +247,12 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
247 ):
248 super().__init__()
249 config = vllm_config.model_config.hf_config
250 - cache_config = vllm_config.cache_config
250 quant_config = vllm_config.quant_config
251 self.config = config
252 self.quant_config = quant_config
254 - self.transformer = GPT2Model(config,
255 - cache_config,
256 - quant_config,
257 - prefix="transformer")
253 + self.transformer = GPT2Model(vllm_config=vllm_config,
254 + prefix=maybe_prefix(
255 + prefix, "transformer"))
256 if self.config.tie_word_embeddings:
257 self.lm_head = self.transformer.wte
258 else:
vllm/model_executor/models/gpt_bigcode.py
+10 -12
@@ -25,7 +25,7 @@ from transformers import GPTBigCodeConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 from vllm.compilation.decorators import support_torch_compile
28 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
28 +from vllm.config import CacheConfig, VllmConfig
29 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -189,15 +189,14 @@ class GPTBigCodeBlock(nn.Module):
189 @support_torch_compile
190 class GPTBigCodeModel(nn.Module):
191
192 - def __init__(
193 - self,
194 - config: GPTBigCodeConfig,
195 - cache_config: Optional[CacheConfig] = None,
196 - quant_config: Optional[QuantizationConfig] = None,
197 - lora_config: Optional[LoRAConfig] = None,
198 - prefix: str = "",
199 - ):
192 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
193 super().__init__()
194 +
195 + config = vllm_config.model_config.hf_config
196 + cache_config = vllm_config.cache_config
197 + quant_config = vllm_config.quant_config
198 + lora_config = vllm_config.lora_config
199 +
200 self.config = config
201 assert not config.add_cross_attention
202
@@ -265,7 +264,6 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
264 ):
265 super().__init__()
266 config = vllm_config.model_config.hf_config
268 - cache_config = vllm_config.cache_config
267 quant_config = vllm_config.quant_config
268 lora_config = vllm_config.lora_config
269
@@ -273,8 +271,8 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
271 self.lora_config = lora_config
272
273 self.quant_config = quant_config
276 - self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
277 - lora_config)
274 + self.transformer = GPTBigCodeModel(vllm_config=vllm_config,
275 + prefix=prefix)
276 if self.config.tie_word_embeddings:
277 self.lm_head = self.transformer.wte
278 else:
vllm/model_executor/models/gpt_j.py
+11 -10
@@ -42,7 +42,8 @@ from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsPP
44 from .utils import (is_pp_missing_parameter,
45 - make_empty_intermediate_tensors_factory, make_layers)
45 + make_empty_intermediate_tensors_factory, make_layers,
46 + maybe_prefix)
47
48
49 class GPTJAttention(nn.Module):
@@ -177,14 +178,13 @@ class GPTJBlock(nn.Module):
178 @support_torch_compile
179 class GPTJModel(nn.Module):
180
180 - def __init__(
181 - self,
182 - config: GPTJConfig,
183 - cache_config: Optional[CacheConfig] = None,
184 - quant_config: Optional[QuantizationConfig] = None,
185 - prefix: str = "",
186 - ):
181 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
182 super().__init__()
183 +
184 + config = vllm_config.model_config.hf_config
185 + cache_config = vllm_config.cache_config
186 + quant_config = vllm_config.quant_config
187 +
188 self.config = config
189 self.embed_dim = config.n_embd
190 self.wte = VocabParallelEmbedding(
@@ -236,12 +236,13 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
236 ):
237 super().__init__()
238 config = vllm_config.model_config.hf_config
239 - cache_config = vllm_config.cache_config
239 quant_config = vllm_config.quant_config
240 self.config = config
241 self.quant_config = quant_config
242 assert not config.tie_word_embeddings
244 - self.transformer = GPTJModel(config, cache_config, quant_config)
243 + self.transformer = GPTJModel(vllm_config=vllm_config,
244 + prefix=maybe_prefix(
245 + prefix, "transformer"))
246 self.lm_head = ParallelLMHead(
247 config.vocab_size,
248 config.n_embd,
vllm/model_executor/models/gpt_neox.py
+10 -10
@@ -41,7 +41,8 @@ from vllm.sequence import IntermediateTensors
41
42 from .interfaces import SupportsPP
43 from .utils import (is_pp_missing_parameter,
44 - make_empty_intermediate_tensors_factory, make_layers)
44 + make_empty_intermediate_tensors_factory, make_layers,
45 + maybe_prefix)
46
47
48 class GPTNeoXAttention(nn.Module):
@@ -189,14 +190,13 @@ class GPTNeoXLayer(nn.Module):
190 @support_torch_compile
191 class GPTNeoXModel(nn.Module):
192
192 - def __init__(
193 - self,
194 - config: GPTNeoXConfig,
195 - cache_config: Optional[CacheConfig] = None,
196 - quant_config: Optional[QuantizationConfig] = None,
197 - prefix: str = "",
198 - ):
193 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
194 super().__init__()
195 +
196 + config = vllm_config.model_config.hf_config
197 + cache_config = vllm_config.cache_config
198 + quant_config = vllm_config.quant_config
199 +
200 self.config = config
201
202 self.embed_in = VocabParallelEmbedding(
@@ -249,11 +249,11 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
249 ):
250 super().__init__()
251 config = vllm_config.model_config.hf_config
252 - cache_config = vllm_config.cache_config
252 quant_config = vllm_config.quant_config
253 self.config = config
254 self.quant_config = quant_config
256 - self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config)
255 + self.gpt_neox = GPTNeoXModel(vllm_config=vllm_config,
256 + prefix=maybe_prefix(prefix, "gpt_neox"))
257 self.embed_out = ParallelLMHead(
258 config.vocab_size,
259 config.hidden_size,
vllm/model_executor/models/granite.py
+13 -21
@@ -28,7 +28,7 @@ from transformers import GraniteConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -52,7 +52,8 @@ from vllm.platforms import current_platform
52 from vllm.sequence import IntermediateTensors
53
54 from .interfaces import SupportsLoRA, SupportsPP
55 -from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
55 +from .utils import (PPMissingLayer, is_pp_missing_parameter, make_layers,
56 + maybe_prefix)
57
58
59 class GraniteMLP(nn.Module):
@@ -257,15 +258,14 @@ class GraniteDecoderLayer(nn.Module):
258 @support_torch_compile
259 class GraniteModel(nn.Module):
260
260 - def __init__(
261 - self,
262 - config: GraniteConfig,
263 - cache_config: Optional[CacheConfig] = None,
264 - quant_config: Optional[QuantizationConfig] = None,
265 - lora_config: Optional[LoRAConfig] = None,
266 - prefix: str = "",
267 - ) -> None:
261 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
262 super().__init__()
263 +
264 + config = vllm_config.model_config.hf_config
265 + cache_config = vllm_config.cache_config
266 + quant_config = vllm_config.quant_config
267 + lora_config = vllm_config.lora_config
268 +
269 self.config = config
270 self.padding_idx = config.pad_token_id
271 lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -370,25 +370,17 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
370 "up_proj": ("gate_up_proj", 1),
371 }
372
373 - def __init__(
374 - self,
375 - vllm_config: VllmConfig,
376 - prefix: str = "",
377 - ) -> None:
373 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
374 super().__init__()
375 config = vllm_config.model_config.hf_config
380 - cache_config = vllm_config.cache_config
376 quant_config = vllm_config.quant_config
377 lora_config = vllm_config.lora_config
378
379 self.config = config
380 self.lora_config = lora_config
381
387 - self.model = GraniteModel(config,
388 - cache_config,
389 - quant_config,
390 - lora_config=lora_config,
391 - prefix="model")
382 + self.model = GraniteModel(vllm_config=vllm_config,
383 + prefix=maybe_prefix(prefix, "model"))
384 if get_pp_group().is_last_rank:
385 self.unpadded_vocab_size = config.vocab_size
386 if lora_config:
vllm/model_executor/models/granitemoe.py
+12 -21
@@ -28,7 +28,7 @@ from transformers.models.granitemoe import GraniteMoeConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -47,7 +47,7 @@ from vllm.sequence import IntermediateTensors
47
48 from . import mixtral
49 from .interfaces import SupportsLoRA, SupportsPP
50 -from .utils import make_layers
50 +from .utils import make_layers, maybe_prefix
51
52
53 class GraniteMoeMoE(nn.Module):
@@ -247,15 +247,14 @@ class GraniteMoeDecoderLayer(nn.Module):
247 @support_torch_compile
248 class GraniteMoeModel(nn.Module):
249
250 - def __init__(
251 - self,
252 - config: GraniteMoeConfig,
253 - cache_config: Optional[CacheConfig] = None,
254 - quant_config: Optional[QuantizationConfig] = None,
255 - lora_config: Optional[LoRAConfig] = None,
256 - prefix: str = "",
257 - ) -> None:
250 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
251 super().__init__()
252 +
253 + config = vllm_config.model_config.hf_config
254 + cache_config = vllm_config.cache_config
255 + quant_config = vllm_config.quant_config
256 + lora_config = vllm_config.lora_config
257 +
258 self.padding_idx = config.pad_token_id
259 lora_vocab = (lora_config.lora_extra_vocab_size *
260 (lora_config.max_loras or 1)) if lora_config else 0
@@ -333,25 +332,17 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332 }
333 embedding_padding_modules = ["lm_head"]
334
336 - def __init__(
337 - self,
338 - vllm_config: VllmConfig,
339 - prefix: str = "",
340 - ) -> None:
335 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
336 super().__init__()
337 config = vllm_config.model_config.hf_config
343 - cache_config = vllm_config.cache_config
338 quant_config = vllm_config.quant_config
339 lora_config = vllm_config.lora_config
340
341 self.config = config
342 self.lora_config = lora_config
343
350 - self.model = GraniteMoeModel(config,
351 - cache_config,
352 - quant_config,
353 - lora_config=lora_config,
354 - prefix="model")
344 + self.model = GraniteMoeModel(vllm_config=vllm_config,
345 + prefix=maybe_prefix(prefix, "model"))
346 self.unpadded_vocab_size = config.vocab_size
347 if lora_config:
348 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/idefics3.py
+12 -17
@@ -22,17 +22,15 @@ import torch.utils.checkpoint
22 from PIL import Image
23 from torch import nn
24 # Temporary solution for transformers below 4.46.0.
25 -from transformers import PretrainedConfig as Idefics3Config
25 from transformers import ProcessorMixin as Idefics3ImageProcessor
26
27 from vllm.attention import AttentionMetadata
29 -from vllm.config import CacheConfig, VllmConfig
28 +from vllm.config import VllmConfig
29 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30 InputContext, token_inputs)
31 from vllm.logger import init_logger
32 from vllm.model_executor.layers.linear import ReplicatedLinear
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 -from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -48,7 +46,8 @@ from .idefics2_vision_model import (
46 # yapf: enable
47 from .interfaces import SupportsMultiModal
48 from .llama import LlamaModel
51 -from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
49 +from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
50 + merge_multimodal_embeddings)
51
52 logger = init_logger(__name__)
53
@@ -417,13 +416,13 @@ class Idefics3Connector(nn.Module):
416
417 class Idefics3Model(nn.Module):
418
420 - def __init__(
421 - self,
422 - config: Idefics3Config,
423 - cache_config: Optional[CacheConfig] = None,
424 - quant_config: Optional[QuantizationConfig] = None,
425 - ):
419 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
420 super().__init__()
421 +
422 + config = vllm_config.model_config.hf_config
423 + cache_config = vllm_config.cache_config
424 + quant_config = vllm_config.quant_config
425 +
426 self.config = config
427 self.padding_idx = self.config.text_config.pad_token_id
428 self.vocab_size = self.config.text_config.vocab_size
@@ -613,22 +612,18 @@ class Idefics3Model(nn.Module):
612 @INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
613 class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
614
616 - def __init__(
617 - self,
618 - vllm_config: VllmConfig,
619 - prefix: str = "",
620 - ) -> None:
615 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
616 super().__init__()
617
618 config = vllm_config.model_config.hf_config
624 - cache_config = vllm_config.cache_config
619 quant_config = vllm_config.quant_config
620 multimodal_config = vllm_config.model_config.multimodal_config
621
622 self.config = config
623 self.multimodal_config = multimodal_config
624
631 - self.model = Idefics3Model(config, cache_config, quant_config)
625 + self.model = Idefics3Model(vllm_config=vllm_config,
626 + prefix=maybe_prefix(prefix, "model"))
627 self.image_token_id = self.config.image_token_id
628
629 self.lm_head = ParallelLMHead(
vllm/model_executor/models/internlm2.py
+8 -16
@@ -250,14 +250,13 @@ class InternLMDecoderLayer(nn.Module):
250 @support_torch_compile
251 class InternLM2Model(nn.Module):
252
253 - def __init__(
254 - self,
255 - config: PretrainedConfig,
256 - cache_config: Optional[CacheConfig] = None,
257 - quant_config: Optional[QuantizationConfig] = None,
258 - prefix: str = "",
259 - ) -> None:
253 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
254 super().__init__()
255 +
256 + config = vllm_config.model_config.hf_config
257 + cache_config = vllm_config.cache_config
258 + quant_config = vllm_config.quant_config
259 +
260 self.config = config
261 self.padding_idx = config.pad_token_id
262 self.vocab_size = config.vocab_size
@@ -317,20 +316,13 @@ class InternLM2Model(nn.Module):
316
317 class InternLM2ForCausalLM(nn.Module, SupportsPP):
318
320 - def __init__(
321 - self,
322 - vllm_config: VllmConfig,
323 - prefix: str = "",
324 - ) -> None:
319 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
320 super().__init__()
321 config = vllm_config.model_config.hf_config
327 - cache_config = vllm_config.cache_config
322 quant_config = vllm_config.quant_config
323 self.config = config
324 self.quant_config = quant_config
331 - self.model = InternLM2Model(config,
332 - cache_config,
333 - quant_config,
325 + self.model = InternLM2Model(vllm_config=vllm_config,
326 prefix=maybe_prefix(prefix, "model"))
327 self.output = ParallelLMHead(config.vocab_size,
328 config.hidden_size,
vllm/model_executor/models/internlm2_ve.py
+9 -14
@@ -104,14 +104,13 @@ class InternLM2VEDecoderLayer(nn.Module):
104
105 class InternLM2VEModel(InternLM2Model):
106
107 - def __init__(
108 - self,
109 - config: PretrainedConfig,
110 - cache_config: Optional[CacheConfig] = None,
111 - quant_config: Optional[QuantizationConfig] = None,
112 - prefix: str = "",
113 - ) -> None:
114 - super().__init__(config, cache_config, quant_config)
107 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
108 + super().__init__(vllm_config=vllm_config, prefix=prefix)
109 +
110 + config = vllm_config.model_config.hf_config
111 + cache_config = vllm_config.cache_config
112 + quant_config = vllm_config.quant_config
113 +
114 self.start_layer, self.end_layer, self.layers = make_layers(
115 config.num_hidden_layers,
116 lambda prefix: InternLM2VEDecoderLayer(
@@ -159,12 +158,8 @@ class InternLM2VEModel(InternLM2Model):
158
159 class InternLM2VEForCausalLM(InternLM2ForCausalLM):
160
162 - def __init__(
163 - self,
164 - vllm_config: VllmConfig,
165 - prefix: str = "",
166 - ) -> None:
167 - super().__init__(vllm_config, prefix=prefix)
161 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
162 + super().__init__(vllm_config=vllm_config, prefix=prefix)
163
164 config = vllm_config.model_config.hf_config
165 cache_config = vllm_config.cache_config
vllm/model_executor/models/internvl.py
+3 -3
@@ -35,7 +35,7 @@ from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
35 get_clip_num_patches)
36 from .interfaces import SupportsMultiModal, SupportsPP
37 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38 - merge_multimodal_embeddings)
38 + maybe_prefix, merge_multimodal_embeddings)
39
40 IMG_START = '<img>'
41 IMG_END = '</img>'
@@ -435,13 +435,13 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
435 config,
436 quant_config=quant_config,
437 is_mono=self.is_mono,
438 - prefix="vision_model",
438 + prefix=maybe_prefix(prefix, "vision_model"),
439 )
440
441 self.language_model = init_vllm_registered_model(
442 config.text_config,
443 vllm_config=vllm_config,
444 - prefix="language_model")
444 + prefix=maybe_prefix(prefix, "language_model"))
445
446 self.mlp1 = self._init_mlp1(config)
447
vllm/model_executor/models/jais.py
+11 -10
@@ -44,7 +44,8 @@ from vllm.transformers_utils.configs import JAISConfig
44
45 from .interfaces import SupportsPP
46 from .utils import (is_pp_missing_parameter,
47 - make_empty_intermediate_tensors_factory, make_layers)
47 + make_empty_intermediate_tensors_factory, make_layers,
48 + maybe_prefix)
49
50
51 class SwiGLUActivation(nn.Module):
@@ -215,14 +216,13 @@ class JAISBlock(nn.Module):
216 @support_torch_compile
217 class JAISModel(nn.Module):
218
218 - def __init__(
219 - self,
220 - config: JAISConfig,
221 - cache_config: Optional[CacheConfig] = None,
222 - quant_config: Optional[QuantizationConfig] = None,
223 - prefix: str = "",
224 - ):
219 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
220 super().__init__()
221 +
222 + config = vllm_config.model_config.hf_config
223 + cache_config = vllm_config.cache_config
224 + quant_config = vllm_config.quant_config
225 +
226 self.config = config
227 assert not config.add_cross_attention
228 assert not config.scale_attn_by_inverse_layer_idx
@@ -293,11 +293,12 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
293 ):
294 super().__init__()
295 config = vllm_config.model_config.hf_config
296 - cache_config = vllm_config.cache_config
296 quant_config = vllm_config.quant_config
297 self.config = config
298 self.quant_config = quant_config
300 - self.transformer = JAISModel(config, cache_config, quant_config)
299 + self.transformer = JAISModel(vllm_config=vllm_config,
300 + prefix=maybe_prefix(
301 + prefix, "transformer"))
302 if self.config.tie_word_embeddings:
303 self.lm_head = self.transformer.wte
304 else:
vllm/model_executor/models/jamba.py
+12 -18
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 from vllm.attention.layer import Attention
10 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
10 +from vllm.config import CacheConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_world_size
12 from vllm.model_executor.layers.fused_moe import FusedMoE
13 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -29,6 +29,7 @@ from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
29 _get_graph_batch_size)
30
31 from .interfaces import HasInnerState, SupportsLoRA
32 +from .utils import maybe_prefix
33
34 KVCache = Tuple[torch.Tensor, torch.Tensor]
35
@@ -258,14 +259,14 @@ ALL_DECODER_LAYER_TYPES = {
259
260 class JambaModel(nn.Module):
261
261 - def __init__(
262 - self,
263 - config: JambaConfig,
264 - quant_config: Optional[QuantizationConfig] = None,
265 - cache_config: Optional[CacheConfig] = None,
266 - lora_config: Optional[LoRAConfig] = None,
267 - ) -> None:
262 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
263 super().__init__()
264 +
265 + config = vllm_config.model_config.hf_config
266 + cache_config = vllm_config.cache_config
267 + quant_config = vllm_config.quant_config
268 + lora_config = vllm_config.lora_config
269 +
270 self.config = config
271 self.padding_idx = config.pad_token_id
272 lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -348,14 +349,9 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
349 }
350 embedding_padding_modules = ["lm_head"]
351
351 - def __init__(
352 - self,
353 - vllm_config: VllmConfig,
354 - prefix: str = "",
355 - ) -> None:
352 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
353 config = vllm_config.model_config.hf_config
354 cache_config = vllm_config.cache_config
358 - quant_config = vllm_config.quant_config
355 lora_config = vllm_config.lora_config
356 scheduler_config = vllm_config.scheduler_config
357 assert not cache_config.enable_prefix_caching, \
@@ -364,10 +360,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
360 super().__init__()
361 self.config = config
362 self.scheduler_config = scheduler_config
367 - self.model = JambaModel(config,
368 - cache_config=cache_config,
369 - quant_config=quant_config,
370 - lora_config=lora_config)
363 + self.model = JambaModel(vllm_config=vllm_config,
364 + prefix=maybe_prefix(prefix, "model"))
365 self.unpadded_vocab_size = config.vocab_size
366 if lora_config:
367 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/llama.py
+12 -32
@@ -28,7 +28,7 @@ from transformers import LlamaConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -271,15 +271,14 @@ class LlamaDecoderLayer(nn.Module):
271 @support_torch_compile
272 class LlamaModel(nn.Module):
273
274 - def __init__(
275 - self,
276 - config: LlamaConfig,
277 - cache_config: Optional[CacheConfig] = None,
278 - quant_config: Optional[QuantizationConfig] = None,
279 - lora_config: Optional[LoRAConfig] = None,
280 - prefix: str = "",
281 - ) -> None:
274 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
275 super().__init__()
276 +
277 + config = vllm_config.model_config.hf_config
278 + cache_config = vllm_config.cache_config
279 + quant_config = vllm_config.quant_config
280 + lora_config = vllm_config.lora_config
281 +
282 self.config = config
283 self.padding_idx = config.pad_token_id
284 lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -492,24 +491,16 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
491 "norm": "model.norm"
492 }
493
495 - def __init__(
496 - self,
497 - vllm_config: VllmConfig,
498 - prefix: str = "",
499 - ) -> None:
494 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
495 super().__init__()
496 config = vllm_config.model_config.hf_config
502 - cache_config = vllm_config.cache_config
497 quant_config = vllm_config.quant_config
498 lora_config = vllm_config.lora_config
499 pooler_config = vllm_config.model_config.pooler_config
500 self.config = config
501 self.lora_config = lora_config
502
509 - self.model = LlamaModel(config,
510 - cache_config,
511 - quant_config,
512 - lora_config=lora_config,
503 + self.model = LlamaModel(vllm_config=vllm_config,
504 prefix=maybe_prefix(prefix, "model"))
505 if get_pp_group().is_last_rank:
506 self.unpadded_vocab_size = config.vocab_size
@@ -652,23 +643,12 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
643 }
644 embedding_padding_modules = []
645
655 - def __init__(
656 - self,
657 - vllm_config: VllmConfig,
658 - prefix: str = "",
659 - ) -> None:
646 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
647 super().__init__()
648
662 - config = vllm_config.model_config.hf_config
663 - cache_config = vllm_config.cache_config
664 - quant_config = vllm_config.quant_config
665 - lora_config = vllm_config.lora_config
649 pooler_config = vllm_config.model_config.pooler_config
650
668 - self.model = LlamaModel(config,
669 - cache_config,
670 - quant_config,
671 - lora_config,
651 + self.model = LlamaModel(vllm_config=vllm_config,
652 prefix=maybe_prefix(prefix, "model"))
653 self._pooler = Pooler.from_config_with_defaults(
654 pooler_config,
vllm/model_executor/models/llava.py
+3 -3
@@ -32,7 +32,7 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
32 dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
33 input_processor_for_siglip)
34 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
35 - merge_multimodal_embeddings)
35 + maybe_prefix, merge_multimodal_embeddings)
36
37
38 class LlavaImagePixelInputs(TypedDict):
@@ -282,7 +282,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
282 config,
283 quant_config,
284 require_post_norm=False,
285 - prefix="vision_tower")
285 + prefix=maybe_prefix(prefix, "vision_tower"))
286 self.multi_modal_projector = LlavaMultiModalProjector(
287 vision_hidden_size=config.vision_config.hidden_size,
288 text_hidden_size=config.text_config.hidden_size,
@@ -291,7 +291,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
291 self.language_model = init_vllm_registered_model(
292 config.text_config,
293 vllm_config=vllm_config,
294 - prefix="language_model")
294 + prefix=maybe_prefix(prefix, "language_model"))
295
296 self.make_empty_intermediate_tensors = (
297 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_next.py
+3 -3
@@ -31,7 +31,7 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
32 get_siglip_patch_grid_length, input_processor_for_siglip)
33 from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
34 - init_vllm_registered_model)
34 + init_vllm_registered_model, maybe_prefix)
35
36
37 class LlavaNextImagePixelInputs(TypedDict):
@@ -296,7 +296,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
296 config,
297 quant_config,
298 require_post_norm=False,
299 - prefix="vision_tower")
299 + prefix=maybe_prefix(prefix, "vision_tower"))
300 self.image_newline = nn.Parameter(
301 torch.empty(config.text_config.hidden_size))
302 self.multi_modal_projector = LlavaMultiModalProjector(
@@ -307,7 +307,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
307 self.language_model = init_vllm_registered_model(
308 config.text_config,
309 vllm_config=vllm_config,
310 - prefix="language_model")
310 + prefix=maybe_prefix(prefix, "language_model"))
311
312 # The same model class supports both language generation and embedding
313 # because the architecture name is the same
vllm/model_executor/models/llava_next_video.py
+3 -3
@@ -29,7 +29,7 @@ from .llava import init_vision_tower_for_llava
29 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30 dummy_seq_data_for_siglip)
31 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
32 - merge_multimodal_embeddings)
32 + maybe_prefix, merge_multimodal_embeddings)
33
34 # For profile run
35 _MAX_FRAMES_PER_VIDEO = 32
@@ -267,7 +267,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
267 config,
268 quant_config,
269 require_post_norm=False,
270 - prefix="vision_tower")
270 + prefix=maybe_prefix(prefix, "vision_tower"))
271 self.vision_resampler = LlavaNextVideoPooler(config)
272 self.multi_modal_projector = LlavaNextMultiModalProjector(
273 vision_hidden_size=config.vision_config.hidden_size,
@@ -276,7 +276,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
276 self.language_model = init_vllm_registered_model(
277 config.text_config,
278 vllm_config=vllm_config,
279 - prefix="language_model")
279 + prefix=maybe_prefix(prefix, "language_model"))
280
281 self.make_empty_intermediate_tensors = (
282 self.language_model.model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_onevision.py
+3 -3
@@ -35,7 +35,7 @@ from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
35 dummy_video_for_siglip, get_siglip_image_feature_size,
36 get_siglip_patch_grid_length, input_processor_for_siglip)
37 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38 - merge_multimodal_embeddings)
38 + maybe_prefix, merge_multimodal_embeddings)
39
40 # Result in the max possible feature size (2x2 grid of 336x336px tiles)
41 MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
@@ -418,12 +418,12 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
418 config,
419 quant_config,
420 require_post_norm=False,
421 - prefix="vision_tower")
421 + prefix=maybe_prefix(prefix, "vision_tower"))
422 self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
423 self.language_model = init_vllm_registered_model(
424 config.text_config,
425 vllm_config=vllm_config,
426 - prefix="language_model")
426 + prefix=maybe_prefix(prefix, "language_model"))
427 self.image_newline = nn.Parameter(
428 torch.empty(config.text_config.hidden_size))
429
vllm/model_executor/models/mamba.py
+13 -18
@@ -6,7 +6,7 @@ from torch import nn
6 from transformers import MambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
9 +from vllm.config import CacheConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -26,6 +26,8 @@ from vllm.sequence import IntermediateTensors
26 from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
27 _get_graph_batch_size)
28
29 +from .utils import maybe_prefix
30 +
31 KVCache = Tuple[torch.Tensor, torch.Tensor]
32
33
@@ -73,14 +75,14 @@ class MambaDecoderLayer(nn.Module):
75
76 class MambaModel(nn.Module):
77
76 - def __init__(
77 - self,
78 - config: MambaConfig,
79 - quant_config: Optional[QuantizationConfig] = None,
80 - cache_config: Optional[CacheConfig] = None,
81 - lora_config: Optional[LoRAConfig] = None,
82 - ) -> None:
78 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
79 super().__init__()
80 +
81 + config = vllm_config.model_config.hf_config
82 + cache_config = vllm_config.cache_config
83 + quant_config = vllm_config.quant_config
84 + lora_config = vllm_config.lora_config
85 +
86 self.config = config
87 self.padding_idx = config.pad_token_id
88 lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -130,14 +132,9 @@ class MambaModel(nn.Module):
132
133 class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
134
133 - def __init__(
134 - self,
135 - vllm_config: VllmConfig,
136 - prefix: str = "",
137 - ) -> None:
135 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
136 config = vllm_config.model_config.hf_config
137 cache_config = vllm_config.cache_config
140 - quant_config = vllm_config.quant_config
138 lora_config = vllm_config.lora_config
139 scheduler_config = vllm_config.scheduler_config
140 assert not cache_config.enable_prefix_caching, \
@@ -146,10 +143,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
143 super().__init__()
144 self.config = config
145 self.scheduler_config = scheduler_config
149 - self.backbone = MambaModel(config,
150 - cache_config=cache_config,
151 - quant_config=quant_config,
152 - lora_config=lora_config)
146 + self.backbone = MambaModel(vllm_config=vllm_config,
147 + prefix=maybe_prefix(prefix, "backbone"))
148 self.unpadded_vocab_size = config.vocab_size
149 if lora_config:
150 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/minicpm.py
+17 -21
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 +from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
@@ -53,7 +53,8 @@ from vllm.sequence import IntermediateTensors
53
54 from .interfaces import SupportsLoRA, SupportsPP
55 from .utils import (is_pp_missing_parameter,
56 - make_empty_intermediate_tensors_factory, make_layers)
56 + make_empty_intermediate_tensors_factory, make_layers,
57 + maybe_prefix)
58
59
60 class MiniCPMMoE(nn.Module):
@@ -351,15 +352,14 @@ class MiniCPMDecoderLayer(nn.Module):
352 @support_torch_compile
353 class MiniCPMModel(nn.Module):
354
354 - def __init__(
355 - self,
356 - config: PretrainedConfig,
357 - cache_config: Optional[CacheConfig] = None,
358 - quant_config: Optional[QuantizationConfig] = None,
359 - lora_config: Optional[LoRAConfig] = None,
360 - prefix: str = "",
361 - ) -> None:
355 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
356 super().__init__()
357 +
358 + config = vllm_config.model_config.hf_config
359 + cache_config = vllm_config.cache_config
360 + quant_config = vllm_config.quant_config
361 + lora_config = vllm_config.lora_config
362 +
363 self.config = config
364 self.cache_config = cache_config
365 self.quant_config = quant_config
@@ -461,24 +461,22 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
461 }
462 embedding_padding_modules = ["lm_head"]
463
464 - def __init__(
465 - self,
466 - vllm_config: VllmConfig,
467 - prefix: str = "",
468 - ) -> None:
464 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
465 super().__init__()
466 config = vllm_config.model_config.hf_config
467 cache_config = vllm_config.cache_config
468 quant_config = vllm_config.quant_config
469 lora_config = vllm_config.lora_config
470
471 + self.prefix = prefix
472 + self.vllm_config = vllm_config
473 self.config = config
474 self.lora_config = lora_config
475 self.cache_config = cache_config
476 self.quant_config = quant_config
477
478 self.num_experts = getattr(self.config, "num_experts", 0)
481 - self._init_model()
479 + self._init_model(vllm_config=vllm_config, prefix=prefix)
480 unpadded_vocab_size = config.vocab_size
481 if lora_config:
482 unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -502,11 +500,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
500 self.make_empty_intermediate_tensors = (
501 self.model.make_empty_intermediate_tensors)
502
505 - def _init_model(self):
506 - self.model = MiniCPMModel(config=self.config,
507 - cache_config=self.cache_config,
508 - quant_config=self.quant_config,
509 - lora_config=self.lora_config)
503 + def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
504 + self.model = MiniCPMModel(vllm_config=vllm_config,
505 + prefix=maybe_prefix(prefix, "model"))
506
507 def forward(
508 self,
vllm/model_executor/models/minicpm3.py
+5 -7
@@ -28,7 +28,7 @@ from torch import nn
28 from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 -from vllm.config import CacheConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -40,7 +40,7 @@ from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer,
40 MiniCPMForCausalLM,
41 MiniCPMModel)
42
43 -from .utils import make_layers
43 +from .utils import make_layers, maybe_prefix
44
45
46 class MiniCPM3Attention(nn.Module):
@@ -238,8 +238,6 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
238 # `embedding_modules` and `embedding_padding_modules`
239 # are inherited from MiniCPMForCausalLM
240
241 - def _init_model(self):
242 - self.model = MiniCPM3Model(config=self.config,
243 - cache_config=self.cache_config,
244 - quant_config=self.quant_config,
245 - lora_config=self.lora_config)
241 + def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
242 + self.model = MiniCPM3Model(vllm_config=vllm_config,
243 + prefix=maybe_prefix(prefix, "model"))
vllm/model_executor/models/minicpmv.py
+19 -37
@@ -34,7 +34,7 @@ from transformers import PretrainedConfig
34 from typing_extensions import NotRequired
35
36 from vllm.attention import AttentionMetadata
37 -from vllm.config import CacheConfig, VllmConfig
37 +from vllm.config import VllmConfig
38 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39 InputContext, token_inputs)
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -59,7 +59,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
59
60 from .idefics2_vision_model import Idefics2VisionTransformer
61 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
62 -from .utils import is_pp_missing_parameter
62 +from .utils import is_pp_missing_parameter, maybe_prefix
63
64 _KEYS_TO_MODIFY_MAPPING = {
65 "llm.lm_head": "lm_head",
@@ -390,7 +390,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
390 ):
391 config = vllm_config.model_config.hf_config
392 multimodal_config = vllm_config.model_config.multimodal_config
393 - cache_config = vllm_config.cache_config
393 quant_config = vllm_config.quant_config
394 super().__init__()
395 # All MiniCPM-V models disable `tie_word_embeddings` but
@@ -401,11 +400,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
400 self.multimodal_config = multimodal_config
401
402 self.version = get_version_by_config(self.config)
404 - self.llm = self.init_llm(config,
405 - cache_config,
406 - quant_config,
407 - prefix="llm")
408 - self.vpm = self.init_vision_module(config, quant_config, prefix="vpm")
403 + self.llm = self.init_llm(vllm_config=vllm_config,
404 + prefix=maybe_prefix(prefix, "llm"))
405 + self.vpm = self.init_vision_module(config,
406 + quant_config,
407 + prefix=maybe_prefix(prefix, "vpm"))
408 param_dtype = torch.get_default_dtype()
409 self.vpm.to(dtype=param_dtype)
410 self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
@@ -414,13 +413,15 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
413 self.resampler = self.init_resampler(self.embed_dim,
414 self.vision_dim,
415 quant_config=quant_config,
417 - prefix="resampler")
416 + prefix=maybe_prefix(
417 + prefix, "resampler"))
418 self.resampler.to(device="cuda", dtype=param_dtype)
419 # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
420 self.lm_head = ParallelLMHead(config.vocab_size,
421 config.hidden_size,
422 quant_config=quant_config,
423 - prefix="llm.lm_head")
423 + prefix=maybe_prefix(
424 + prefix, "llm.lm_head"))
425 self.logits_processor = LogitsProcessor(config.vocab_size)
426 self.sampler = get_sampler()
427
@@ -661,9 +662,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
662
663 def init_llm(
664 self,
664 - config: PretrainedConfig,
665 - cache_config: Optional[CacheConfig] = None,
666 - quant_config: Optional[QuantizationConfig] = None,
665 + vllm_config: VllmConfig,
666 prefix: str = "",
667 ) -> nn.Module:
668 raise NotImplementedError
@@ -711,16 +710,10 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
710
711 def init_llm(
712 self,
714 - config: PretrainedConfig,
715 - cache_config: Optional[CacheConfig] = None,
716 - quant_config: Optional[QuantizationConfig] = None,
713 + vllm_config: VllmConfig,
714 prefix: str = "",
715 ) -> nn.Module:
719 -
720 - return LLMWrapper(MiniCPMModel(config,
721 - cache_config=cache_config,
722 - quant_config=quant_config,
723 - prefix=prefix),
716 + return LLMWrapper(MiniCPMModel(vllm_config=vllm_config, prefix=prefix),
717 name="model")
718
719 def init_vision_module(
@@ -875,15 +868,10 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
868
869 def init_llm(
870 self,
878 - config: PretrainedConfig,
879 - cache_config: Optional[CacheConfig] = None,
880 - quant_config: Optional[QuantizationConfig] = None,
871 + vllm_config: VllmConfig,
872 prefix: str = "",
873 ) -> nn.Module:
883 - return LLMWrapper(LlamaModel(config,
884 - cache_config=cache_config,
885 - quant_config=quant_config,
886 - prefix=prefix),
874 + return LLMWrapper(LlamaModel(vllm_config=vllm_config, prefix=prefix),
875 name="model")
876
877 def init_vision_module(
@@ -1022,16 +1010,10 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1010
1011 def init_llm(
1012 self,
1025 - config: PretrainedConfig,
1026 - cache_config: Optional[CacheConfig] = None,
1027 - quant_config: Optional[QuantizationConfig] = None,
1013 + vllm_config: VllmConfig,
1014 prefix: str = "",
1015 ) -> nn.Module:
1030 -
1031 - return LLMWrapper(Qwen2Model(config,
1032 - cache_config=cache_config,
1033 - quant_config=quant_config,
1034 - prefix=prefix),
1016 + return LLMWrapper(Qwen2Model(vllm_config=vllm_config, prefix=prefix),
1017 name="model")
1018
1019 def init_vision_module(
@@ -1151,4 +1133,4 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1133 if instance_class is None:
1134 raise ValueError(
1135 "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1154 - return instance_class(vllm_config, prefix=prefix)
1136 + return instance_class(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/mixtral.py
+13 -21
@@ -28,7 +28,7 @@ from transformers import MixtralConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -48,7 +48,8 @@ from vllm.sequence import IntermediateTensors
48
49 from .interfaces import SupportsLoRA, SupportsPP
50 from .utils import (is_pp_missing_parameter,
51 - make_empty_intermediate_tensors_factory, make_layers)
51 + make_empty_intermediate_tensors_factory, make_layers,
52 + maybe_prefix)
53
54
55 class MixtralMoE(nn.Module):
@@ -248,15 +249,14 @@ class MixtralDecoderLayer(nn.Module):
249 @support_torch_compile
250 class MixtralModel(nn.Module):
251
251 - def __init__(
252 - self,
253 - config: MixtralConfig,
254 - cache_config: Optional[CacheConfig] = None,
255 - quant_config: Optional[QuantizationConfig] = None,
256 - lora_config: Optional[LoRAConfig] = None,
257 - prefix: str = "",
258 - ) -> None:
252 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
253 super().__init__()
254 +
255 + config = vllm_config.model_config.hf_config
256 + cache_config = vllm_config.cache_config
257 + quant_config = vllm_config.quant_config
258 + lora_config = vllm_config.lora_config
259 +
260 self.padding_idx = config.pad_token_id
261 lora_vocab = (lora_config.lora_extra_vocab_size *
262 (lora_config.max_loras or 1)) if lora_config else 0
@@ -332,24 +332,16 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332 }
333 embedding_padding_modules = ["lm_head"]
334
335 - def __init__(
336 - self,
337 - vllm_config: VllmConfig,
338 - prefix: str = "",
339 - ) -> None:
335 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
336 super().__init__()
337 config = vllm_config.model_config.hf_config
342 - cache_config = vllm_config.cache_config
338 quant_config = vllm_config.quant_config
339 lora_config = vllm_config.lora_config
340 self.config = config
341 self.lora_config = lora_config
342
348 - self.model = MixtralModel(config,
349 - cache_config,
350 - quant_config,
351 - lora_config=lora_config,
352 - prefix="model")
343 + self.model = MixtralModel(vllm_config=vllm_config,
344 + prefix=maybe_prefix(prefix, "model"))
345 self.unpadded_vocab_size = config.vocab_size
346 if lora_config:
347 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/mixtral_quant.py
+11 -15
@@ -49,7 +49,8 @@ from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsPP
51 from .utils import (is_pp_missing_parameter,
52 - make_empty_intermediate_tensors_factory, make_layers)
52 + make_empty_intermediate_tensors_factory, make_layers,
53 + maybe_prefix)
54
55
56 class MixtralMLP(nn.Module):
@@ -293,14 +294,13 @@ class MixtralDecoderLayer(nn.Module):
294
295 class MixtralModel(nn.Module):
296
296 - def __init__(
297 - self,
298 - config: MixtralConfig,
299 - cache_config: Optional[CacheConfig] = None,
300 - quant_config: Optional[QuantizationConfig] = None,
301 - prefix: str = "",
302 - ) -> None:
297 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
298 super().__init__()
299 +
300 + config = vllm_config.model_config.hf_config
301 + cache_config = vllm_config.cache_config
302 + quant_config = vllm_config.quant_config
303 +
304 self.padding_idx = config.pad_token_id
305 self.vocab_size = config.vocab_size
306
@@ -350,18 +350,14 @@ class MixtralModel(nn.Module):
350 class MixtralForCausalLM(nn.Module, SupportsPP):
351 fall_back_to_pt_during_load = False
352
353 - def __init__(
354 - self,
355 - vllm_config: VllmConfig,
356 - prefix: str = "",
357 - ) -> None:
353 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
354 super().__init__()
355 config = vllm_config.model_config.hf_config
360 - cache_config = vllm_config.cache_config
356 quant_config = vllm_config.quant_config
357 self.config = config
358 self.quant_config = quant_config
364 - self.model = MixtralModel(config, cache_config, quant_config)
359 + self.model = MixtralModel(vllm_config=vllm_config,
360 + prefix=maybe_prefix(prefix, "model"))
361 self.lm_head = ParallelLMHead(config.vocab_size,
362 config.hidden_size,
363 quant_config=quant_config)
vllm/model_executor/models/mllama.py
+19 -30
@@ -33,7 +33,7 @@ from transformers.models.mllama.processing_mllama import (
33 import vllm.distributed.parallel_state as ps
34 from vllm.attention import Attention, AttentionMetadata, AttentionType
35 from vllm.attention.ops.paged_attn import PagedAttention
36 -from vllm.config import CacheConfig, VllmConfig
36 +from vllm.config import VllmConfig
37 from vllm.distributed import get_tensor_model_parallel_world_size
38 from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
39 InputContext, TokenInputs, token_inputs)
@@ -56,6 +56,7 @@ from vllm.utils import is_list_of
56 from .clip import CLIPMLP
57 from .interfaces import SupportsMultiModal
58 from .llama import LlamaDecoderLayer, LlamaMLP
59 +from .utils import maybe_prefix
60
61 logger = init_logger(__name__)
62 MLLAMA_IMAGE_TOKEN_ID = 128256
@@ -939,15 +940,13 @@ class MllamaTextModel(nn.Module):
940 config_class = config_mllama.MllamaTextConfig
941 base_model_prefix = "model"
942
942 - def __init__(
943 - self,
944 - config: config_mllama.MllamaTextConfig,
945 - cache_config: Optional[CacheConfig],
946 - quant_config: Optional[QuantizationConfig],
947 - prefix: str = "",
948 - ) -> None:
943 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
944 super().__init__()
945
946 + config = vllm_config.model_config.hf_config.text_config
947 + cache_config = vllm_config.cache_config
948 + quant_config = vllm_config.quant_config
949 +
950 self.padding_idx = config.pad_token_id
951 self.vocab_size = config.vocab_size
952 self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
@@ -1029,18 +1028,14 @@ class MllamaForCausalLM(nn.Module):
1028 "MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
1029 ]
1030
1032 - def __init__(
1033 - self,
1034 - config: config_mllama.MllamaTextConfig,
1035 - cache_config: Optional[CacheConfig],
1036 - quant_config: Optional[QuantizationConfig],
1037 - prefix: str = "",
1038 - ) -> None:
1031 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1032 super().__init__()
1033 +
1034 + config = vllm_config.model_config.hf_config.text_config
1035 + quant_config = vllm_config.quant_config
1036 +
1037 self.vocab_size = config.vocab_size
1041 - self.model = MllamaTextModel(config,
1042 - cache_config,
1043 - quant_config,
1038 + self.model = MllamaTextModel(vllm_config=vllm_config,
1039 prefix=f"{prefix}.model")
1040 self.lm_head = ParallelLMHead(
1041 config.vocab_size,
@@ -1108,14 +1103,9 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1103 "up_proj": ("gate_up_proj", 1),
1104 }
1105
1111 - def __init__(
1112 - self,
1113 - vllm_config: VllmConfig,
1114 - prefix: str = "",
1115 - ) -> None:
1106 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1107 super().__init__()
1108 config = vllm_config.model_config.hf_config
1118 - cache_config = vllm_config.cache_config
1109 quant_config = vllm_config.quant_config
1110 self.vocab_size = config.text_config.vocab_size
1111 self.hidden_size = config.text_config.hidden_size
@@ -1127,12 +1117,11 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1117
1118 self.vision_model = MllamaVisionModel(config.vision_config,
1119 quant_config,
1130 - prefix="vision_model")
1120 + prefix=maybe_prefix(
1121 + prefix, "vision_model"))
1122 self.language_model = MllamaForCausalLM(
1132 - config.text_config,
1133 - cache_config=cache_config,
1134 - quant_config=quant_config,
1135 - prefix="language_model",
1123 + vllm_config=vllm_config,
1124 + prefix=maybe_prefix(prefix, "language_model"),
1125 )
1126 self.multi_modal_projector = ColumnParallelLinear(
1127 config.vision_config.vision_output_dim,
@@ -1140,7 +1129,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1129 bias=True,
1130 quant_config=quant_config,
1131 gather_output=True,
1143 - prefix="multi_modal_projector",
1132 + prefix=maybe_prefix(prefix, "multi_modal_projector"),
1133 )
1134 self.logits_processor = LogitsProcessor(config.output_hidden_states,
1135 config.text_config.vocab_size)
vllm/model_executor/models/molmo.py
+11 -15
@@ -44,7 +44,8 @@ from vllm.transformers_utils.processor import get_processor
44
45 from .interfaces import SupportsMultiModal, SupportsPP
46 from .utils import (get_vit_attn_backend,
47 - make_empty_intermediate_tensors_factory, make_layers)
47 + make_empty_intermediate_tensors_factory, make_layers,
48 + maybe_prefix)
49
50 # TODO: hard-coded for now. Consider making it configurable.
51 VIT_LAYERS = [-2, -9]
@@ -716,14 +717,13 @@ class MolmoVisionBackbone(nn.Module):
717 @support_torch_compile
718 class MolmoModel(nn.Module):
719
719 - def __init__(
720 - self,
721 - config: PretrainedConfig,
722 - cache_config: Optional[CacheConfig] = None,
723 - quant_config: Optional[QuantizationConfig] = None,
724 - prefix: str = "",
725 - ) -> None:
720 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
721 super().__init__()
722 +
723 + config = vllm_config.model_config.hf_config
724 + cache_config = vllm_config.cache_config
725 + quant_config = vllm_config.quant_config
726 +
727 self.config = config
728
729 self.embedding_size = config.embedding_size or config.vocab_size
@@ -1024,14 +1024,9 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1024 @INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1025 class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1026
1027 - def __init__(
1028 - self,
1029 - vllm_config: VllmConfig,
1030 - prefix: str = "",
1031 - ) -> None:
1027 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1028 super().__init__()
1029 config = vllm_config.model_config.hf_config
1034 - cache_config = vllm_config.cache_config
1030 quant_config = vllm_config.quant_config
1031 multimodal_config = vllm_config.model_config.multimodal_config
1032 self.config = config
@@ -1040,7 +1035,8 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1035 vision_config = VisionBackboneConfig()
1036 self.vision_backbone = MolmoVisionBackbone(config, vision_config,
1037 quant_config)
1043 - self.model = MolmoModel(config, cache_config, quant_config)
1038 + self.model = MolmoModel(vllm_config=vllm_config,
1039 + prefix=maybe_prefix(prefix, "model"))
1040
1041 if self.config.weight_tying:
1042 self.lm_head = self.model.transformer.wte
vllm/model_executor/models/mpt.py
+11 -15
@@ -26,7 +26,8 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
26
27 from .interfaces import SupportsPP
28 from .utils import (is_pp_missing_parameter,
29 - make_empty_intermediate_tensors_factory, make_layers)
29 + make_empty_intermediate_tensors_factory, make_layers,
30 + maybe_prefix)
31
32
33 def _get_alibi_slopes(
@@ -207,14 +208,13 @@ class MPTBlock(nn.Module):
208 @support_torch_compile
209 class MPTModel(nn.Module):
210
210 - def __init__(
211 - self,
212 - config: MPTConfig,
213 - cache_config: Optional[CacheConfig] = None,
214 - quant_config: Optional[QuantizationConfig] = None,
215 - prefix: str = "",
216 - ):
211 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
212 super().__init__()
213 +
214 + config = vllm_config.model_config.hf_config
215 + cache_config = vllm_config.cache_config
216 + quant_config = vllm_config.quant_config
217 +
218 assert config.embedding_fraction == 1.0
219 assert config.norm_type == "low_precision_layernorm"
220
@@ -267,20 +267,16 @@ class MPTModel(nn.Module):
267
268 class MPTForCausalLM(nn.Module, SupportsPP):
269
270 - def __init__(
271 - self,
272 - vllm_config: VllmConfig,
273 - prefix: str = "",
274 - ) -> None:
270 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271 super().__init__()
272 config = vllm_config.model_config.hf_config
277 - cache_config = vllm_config.cache_config
273 quant_config = vllm_config.quant_config
274 self.config = config
275 assert config.tie_word_embeddings
276 self.quant_config = quant_config
277
283 - self.transformer = MPTModel(config, cache_config, quant_config)
278 + self.transformer = MPTModel(vllm_config=vllm_config,
279 + prefix=maybe_prefix(prefix, "transformer"))
280 self.lm_head = self.transformer.wte
281 self.logits_processor = LogitsProcessor(config.vocab_size)
282 self.sampler = get_sampler()
vllm/model_executor/models/nemotron.py
+13 -21
@@ -27,7 +27,7 @@ from torch import nn
27
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.compilation.decorators import support_torch_compile
30 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
30 +from vllm.config import CacheConfig, VllmConfig
31 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -47,7 +47,8 @@ from vllm.transformers_utils.configs import NemotronConfig
47
48 from .interfaces import SupportsLoRA, SupportsPP
49 from .utils import (PPMissingLayer, is_pp_missing_parameter,
50 - make_empty_intermediate_tensors_factory, make_layers)
50 + make_empty_intermediate_tensors_factory, make_layers,
51 + maybe_prefix)
52
53 # The architecture is pretty similar to Llama, with these changes:
54 # - There is no gate_proj, just up_proj
@@ -293,15 +294,14 @@ class NemotronDecoderLayer(nn.Module):
294 @support_torch_compile
295 class NemotronModel(nn.Module):
296
296 - def __init__(
297 - self,
298 - config: NemotronConfig,
299 - cache_config: Optional[CacheConfig] = None,
300 - quant_config: Optional[QuantizationConfig] = None,
301 - lora_config: Optional[LoRAConfig] = None,
302 - prefix: str = "",
303 - ) -> None:
297 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
298 super().__init__()
299 +
300 + config = vllm_config.model_config.hf_config
301 + cache_config = vllm_config.cache_config
302 + quant_config = vllm_config.quant_config
303 + lora_config = vllm_config.lora_config
304 +
305 self.config = config
306 self.padding_idx = config.pad_token_id
307 lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -401,14 +401,9 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
401 "v_proj": ("qkv_proj", 2),
402 }
403
404 - def __init__(
405 - self,
406 - vllm_config: VllmConfig,
407 - prefix: str = "",
408 - ) -> None:
404 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
405 super().__init__()
406 config = vllm_config.model_config.hf_config
411 - cache_config = vllm_config.cache_config
407 quant_config = vllm_config.quant_config
408 lora_config = vllm_config.lora_config
409 assert isinstance(config, NemotronConfig)
@@ -416,11 +411,8 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411 self.config = config
412 self.lora_config = lora_config
413
419 - self.model = NemotronModel(config,
420 - cache_config,
421 - quant_config,
422 - lora_config=lora_config,
423 - prefix="model")
414 + self.model = NemotronModel(vllm_config=vllm_config,
415 + prefix=maybe_prefix(prefix, "model"))
416 if get_pp_group().is_last_rank:
417 self.unpadded_vocab_size = config.vocab_size
418 if lora_config:
vllm/model_executor/models/olmo.py
+11 -13
@@ -46,7 +46,8 @@ from vllm.sequence import IntermediateTensors
46
47 from .interfaces import SupportsPP
48 from .utils import (is_pp_missing_parameter,
49 - make_empty_intermediate_tensors_factory, make_layers)
49 + make_empty_intermediate_tensors_factory, make_layers,
50 + maybe_prefix)
51
52
53 class OlmoAttention(nn.Module):
@@ -224,12 +225,13 @@ class OlmoDecoderLayer(nn.Module):
225 @support_torch_compile
226 class OlmoModel(nn.Module):
227
227 - def __init__(self,
228 - config: OlmoConfig,
229 - cache_config: Optional[CacheConfig] = None,
230 - quant_config: Optional[QuantizationConfig] = None,
231 - prefix: str = ""):
228 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
229 super().__init__()
230 +
231 + config = vllm_config.model_config.hf_config
232 + cache_config = vllm_config.cache_config
233 + quant_config = vllm_config.quant_config
234 +
235 self.config = config
236
237 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
@@ -291,17 +293,13 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
293 Extremely barebones HF model wrapper.
294 """
295
294 - def __init__(
295 - self,
296 - vllm_config: VllmConfig,
297 - prefix: str = "",
298 - ) -> None:
296 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
297 super().__init__()
298 config = vllm_config.model_config.hf_config
301 - cache_config = vllm_config.cache_config
299 quant_config = vllm_config.quant_config
300 self.config = config
304 - self.model = OlmoModel(config, cache_config, quant_config)
301 + self.model = OlmoModel(vllm_config=vllm_config,
302 + prefix=maybe_prefix(prefix, "model"))
303 if config.tie_word_embeddings:
304 self.lm_head = self.model.embed_tokens
305 else:
vllm/model_executor/models/olmoe.py
+11 -15
@@ -38,7 +38,8 @@ from vllm.utils import print_warning_once
38
39 from .interfaces import SupportsPP
40 from .utils import (is_pp_missing_parameter,
41 - make_empty_intermediate_tensors_factory, make_layers)
41 + make_empty_intermediate_tensors_factory, make_layers,
42 + maybe_prefix)
43
44
45 class OlmoeMoE(nn.Module):
@@ -243,14 +244,13 @@ class OlmoeDecoderLayer(nn.Module):
244 @support_torch_compile
245 class OlmoeModel(nn.Module):
246
246 - def __init__(
247 - self,
248 - config: PretrainedConfig,
249 - cache_config: Optional[CacheConfig] = None,
250 - quant_config: Optional[QuantizationConfig] = None,
251 - prefix: str = "",
252 - ) -> None:
247 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
248 super().__init__()
249 +
250 + config = vllm_config.model_config.hf_config
251 + cache_config = vllm_config.cache_config
252 + quant_config = vllm_config.quant_config
253 +
254 self.padding_idx = config.pad_token_id
255 self.vocab_size = config.vocab_size
256
@@ -309,18 +309,14 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
309
310 fall_back_to_pt_during_load = False
311
312 - def __init__(
313 - self,
314 - vllm_config: VllmConfig,
315 - prefix: str = "",
316 - ) -> None:
312 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
313 super().__init__()
314 config = vllm_config.model_config.hf_config
319 - cache_config = vllm_config.cache_config
315 quant_config = vllm_config.quant_config
316 self.config = config
317 self.quant_config = quant_config
323 - self.model = OlmoeModel(config, cache_config, quant_config)
318 + self.model = OlmoeModel(vllm_config=vllm_config,
319 + prefix=maybe_prefix(prefix, "model"))
320 self.lm_head = ParallelLMHead(config.vocab_size,
321 config.hidden_size,
322 quant_config=quant_config)
vllm/model_executor/models/opt.py
+8 -16
@@ -293,14 +293,13 @@ class OPTDecoder(nn.Module):
293 @support_torch_compile
294 class OPTModel(nn.Module):
295
296 - def __init__(
297 - self,
298 - config: OPTConfig,
299 - cache_config: Optional[CacheConfig] = None,
300 - quant_config: Optional[QuantizationConfig] = None,
301 - prefix: str = "",
302 - ):
296 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
297 super().__init__()
298 +
299 + config = vllm_config.model_config.hf_config
300 + cache_config = vllm_config.cache_config
301 + quant_config = vllm_config.quant_config
302 +
303 self.decoder = OPTDecoder(config,
304 cache_config,
305 quant_config,
@@ -342,21 +341,14 @@ class OPTForCausalLM(nn.Module, SupportsPP):
341 ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
342 ]
343
345 - def __init__(
346 - self,
347 - vllm_config: VllmConfig,
348 - prefix: str = "",
349 - ) -> None:
344 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
345 super().__init__()
346 config = vllm_config.model_config.hf_config
352 - cache_config = vllm_config.cache_config
347 quant_config = vllm_config.quant_config
348 super().__init__()
349 self.config = config
350 self.quant_config = quant_config
357 - self.model = OPTModel(config,
358 - cache_config,
359 - quant_config,
351 + self.model = OPTModel(vllm_config=vllm_config,
352 prefix=maybe_prefix(prefix, "model"))
353 if self.config.tie_word_embeddings:
354 self.lm_head = self.model.decoder.embed_tokens
vllm/model_executor/models/orion.py
+11 -15
@@ -29,7 +29,8 @@ from vllm.sequence import IntermediateTensors
29
30 from .interfaces import SupportsPP
31 from .utils import (is_pp_missing_parameter,
32 - make_empty_intermediate_tensors_factory, make_layers)
32 + make_empty_intermediate_tensors_factory, make_layers,
33 + maybe_prefix)
34
35
36 class OrionMLP(nn.Module):
@@ -208,14 +209,13 @@ class OrionDecoderLayer(nn.Module):
209 @support_torch_compile
210 class OrionModel(nn.Module):
211
211 - def __init__(
212 - self,
213 - config: PretrainedConfig,
214 - cache_config: Optional[CacheConfig] = None,
215 - quant_config: Optional[QuantizationConfig] = None,
216 - prefix: str = "",
217 - ) -> None:
212 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
213 super().__init__()
214 +
215 + config = vllm_config.model_config.hf_config
216 + cache_config = vllm_config.cache_config
217 + quant_config = vllm_config.quant_config
218 +
219 self.config = config
220 self.padding_idx = config.pad_token_id
221 self.vocab_size = config.vocab_size
@@ -268,18 +268,14 @@ class OrionModel(nn.Module):
268
269 class OrionForCausalLM(nn.Module, SupportsPP):
270
271 - def __init__(
272 - self,
273 - vllm_config: VllmConfig,
274 - prefix: str = "",
275 - ) -> None:
271 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
272 super().__init__()
273 config = vllm_config.model_config.hf_config
278 - cache_config = vllm_config.cache_config
274 quant_config = vllm_config.quant_config
275 self.config = config
276 self.quant_config = quant_config
282 - self.model = OrionModel(config, cache_config, quant_config)
277 + self.model = OrionModel(vllm_config=vllm_config,
278 + prefix=maybe_prefix(prefix, "model"))
279 self.lm_head = ParallelLMHead(config.vocab_size,
280 config.hidden_size,
281 quant_config=quant_config)
vllm/model_executor/models/paligemma.py
+5 -8
@@ -20,7 +20,7 @@ from .interfaces import SupportsMultiModal, SupportsPP
20 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
21 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
22 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
23 - merge_multimodal_embeddings)
23 + maybe_prefix, merge_multimodal_embeddings)
24
25 logger = init_logger(__name__)
26
@@ -131,11 +131,7 @@ class PaliGemmaMultiModalProjector(nn.Module):
131 class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
132 SupportsPP):
133
134 - def __init__(
135 - self,
136 - vllm_config: VllmConfig,
137 - prefix: str = "",
138 - ) -> None:
134 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
135 super().__init__()
136 config = vllm_config.model_config.hf_config
137 quant_config = vllm_config.quant_config
@@ -145,7 +141,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
141
142 self.vision_tower = SiglipVisionModel(config.vision_config,
143 quant_config,
148 - prefix="vision_tower")
144 + prefix=maybe_prefix(
145 + prefix, "vision_tower"))
146 self.multi_modal_projector = PaliGemmaMultiModalProjector(
147 vision_hidden_size=config.vision_config.hidden_size,
148 projection_dim=config.vision_config.projection_dim)
@@ -155,7 +152,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
152 self.language_model = init_vllm_registered_model(
153 config.text_config,
154 vllm_config=vllm_config,
158 - prefix="language_model")
155 + prefix=maybe_prefix(prefix, "language_model"))
156 logit_scale = getattr(config, "logit_scale", 1.0)
157 self.language_model.logits_processor.scale *= logit_scale
158
vllm/model_executor/models/persimmon.py
+11 -16
@@ -45,7 +45,8 @@ from vllm.sequence import IntermediateTensors
45
46 from .interfaces import SupportsPP
47 from .utils import (is_pp_missing_parameter,
48 - make_empty_intermediate_tensors_factory, make_layers)
48 + make_empty_intermediate_tensors_factory, make_layers,
49 + maybe_prefix)
50
51
52 class PersimmonMLP(nn.Module):
@@ -212,12 +213,13 @@ class PersimmonDecoderLayer(nn.Module):
213 @support_torch_compile
214 class PersimmonModel(nn.Module):
215
215 - def __init__(self,
216 - config: PersimmonConfig,
217 - cache_config: Optional[CacheConfig] = None,
218 - quant_config: Optional[QuantizationConfig] = None,
219 - prefix: str = ""):
216 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
217 super().__init__()
218 +
219 + config = vllm_config.model_config.hf_config
220 + cache_config = vllm_config.cache_config
221 + quant_config = vllm_config.quant_config
222 +
223 self.vocab_size = config.vocab_size
224
225 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
@@ -265,20 +267,13 @@ class PersimmonModel(nn.Module):
267
268 class PersimmonForCausalLM(nn.Module, SupportsPP):
269
268 - def __init__(
269 - self,
270 - vllm_config: VllmConfig,
271 - prefix: str = "",
272 - ) -> None:
270 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271 super().__init__()
272 config = vllm_config.model_config.hf_config
275 - cache_config = vllm_config.cache_config
276 - quant_config = vllm_config.quant_config
273 self.config = config
274 self.vocab_size = config.vocab_size
279 - self.model = PersimmonModel(config,
280 - cache_config=cache_config,
281 - quant_config=quant_config)
275 + self.model = PersimmonModel(vllm_config=vllm_config,
276 + prefix=maybe_prefix(prefix, "model"))
277 self.lm_head = ParallelLMHead(config.vocab_size,
278 config.hidden_size,
279 bias=False)
vllm/model_executor/models/phi.py
+11 -13
@@ -60,7 +60,8 @@ from vllm.sequence import IntermediateTensors
60
61 from .interfaces import SupportsLoRA, SupportsPP
62 from .utils import (is_pp_missing_parameter,
63 - make_empty_intermediate_tensors_factory, make_layers)
63 + make_empty_intermediate_tensors_factory, make_layers,
64 + maybe_prefix)
65
66
67 class PhiAttention(nn.Module):
@@ -196,12 +197,13 @@ class PhiLayer(nn.Module):
197 @support_torch_compile
198 class PhiModel(nn.Module):
199
199 - def __init__(self,
200 - config: PhiConfig,
201 - cache_config: Optional[CacheConfig] = None,
202 - quant_config: Optional[QuantizationConfig] = None,
203 - prefix: str = ""):
200 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
201 super().__init__()
202 +
203 + config = vllm_config.model_config.hf_config
204 + cache_config = vllm_config.cache_config
205 + quant_config = vllm_config.quant_config
206 +
207 self.config = config
208 self.quant_config = quant_config
209 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
@@ -277,14 +279,9 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
279 embedding_modules = {}
280 embedding_padding_modules = []
281
280 - def __init__(
281 - self,
282 - vllm_config: VllmConfig,
283 - prefix: str = "",
284 - ) -> None:
282 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
283 super().__init__()
284 config = vllm_config.model_config.hf_config
287 - cache_config = vllm_config.cache_config
285 quant_config = vllm_config.quant_config
286 lora_config = vllm_config.lora_config
287 self.config = config
@@ -294,7 +291,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
291
292 self.quant_config = quant_config
293
297 - self.model = PhiModel(config, cache_config, quant_config)
294 + self.model = PhiModel(vllm_config=vllm_config,
295 + prefix=maybe_prefix(prefix, "model"))
296
297 self.lm_head = ParallelLMHead(config.vocab_size,
298 config.hidden_size,
vllm/model_executor/models/phi3_small.py
+11 -15
@@ -24,7 +24,8 @@ from vllm.sequence import IntermediateTensors
24
25 from .interfaces import SupportsPP
26 from .utils import (is_pp_missing_parameter,
27 - make_empty_intermediate_tensors_factory, make_layers)
27 + make_empty_intermediate_tensors_factory, make_layers,
28 + maybe_prefix)
29
30
31 def load_column_parallel_weight(param: torch.nn.Parameter,
@@ -299,14 +300,13 @@ class Phi3SmallDecoderLayer(nn.Module):
300
301 class Phi3SmallModel(nn.Module):
302
302 - def __init__(
303 - self,
304 - config: PretrainedConfig,
305 - cache_config: Optional[CacheConfig] = None,
306 - quant_config: Optional[QuantizationConfig] = None,
307 - prefix: str = "",
308 - ):
303 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
304 super().__init__()
305 +
306 + config = vllm_config.model_config.hf_config
307 + cache_config = vllm_config.cache_config
308 + quant_config = vllm_config.quant_config
309 +
310 self.config = config
311 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
312 config.hidden_size)
@@ -363,18 +363,14 @@ class Phi3SmallModel(nn.Module):
363 class Phi3SmallForCausalLM(nn.Module, SupportsPP):
364 _tied_weights_keys = ["lm_head.weight"]
365
366 - def __init__(
367 - self,
368 - vllm_config: VllmConfig,
369 - prefix: str = "",
370 - ) -> None:
366 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
367 super().__init__()
368 config = vllm_config.model_config.hf_config
373 - cache_config = vllm_config.cache_config
369 quant_config = vllm_config.quant_config
370 self.config = config
371 self.quant_config = quant_config
377 - self.model = Phi3SmallModel(config, cache_config, quant_config)
372 + self.model = Phi3SmallModel(vllm_config=vllm_config,
373 + prefix=maybe_prefix(prefix, "model"))
374 self.vocab_size = config.vocab_size
375 self.mup_width_multiplier = config.mup_width_multiplier
376 self.lm_head = ParallelLMHead(
vllm/model_executor/models/phi3v.py
+6 -8
@@ -45,7 +45,7 @@ from vllm.utils import is_list_of
45
46 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
47 from .interfaces import SupportsMultiModal, SupportsPP
48 -from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
48 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,
49 merge_multimodal_embeddings)
50
51 logger = init_logger(__name__)
@@ -525,11 +525,7 @@ def input_processor_for_phi3v(ctx: InputContext,
525 @INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
526 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
527
528 - def __init__(
529 - self,
530 - vllm_config: VllmConfig,
531 - prefix: str = "",
532 - ) -> None:
528 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
529 super().__init__()
530 config = vllm_config.model_config.hf_config
531 quant_config = vllm_config.quant_config
@@ -544,12 +540,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
540 config.hidden_size,
541 org_num_embeddings=config.vocab_size,
542 quant_config=quant_config,
547 - prefix="model.embed_tokens",
543 + prefix=maybe_prefix(prefix, "model.embed_tokens"),
544 )
545
546 # TODO: Optionally initializes this for supporting input embeddings.
547 self.vision_embed_tokens = Phi3HDImageEmbedding(
552 - config, quant_config, prefix="model.vision_embed_tokens")
548 + config,
549 + quant_config,
550 + prefix=maybe_prefix(prefix, "model.vision_embed_tokens"))
551
552 # The prefix is empty intentionally because default prefix of
553 # LlamaForCausalLM is "model"
vllm/model_executor/models/phimoe.py
+13 -21
@@ -28,7 +28,7 @@ from transformers.configuration_utils import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -48,7 +48,8 @@ from vllm.sequence import IntermediateTensors
48
49 from .interfaces import SupportsLoRA, SupportsPP
50 from .utils import (is_pp_missing_parameter,
51 - make_empty_intermediate_tensors_factory, make_layers)
51 + make_empty_intermediate_tensors_factory, make_layers,
52 + maybe_prefix)
53
54
55 class PhiMoEConfig(PretrainedConfig):
@@ -432,15 +433,14 @@ class PhiMoEDecoderLayer(nn.Module):
433 @support_torch_compile
434 class PhiMoEModel(nn.Module):
435
435 - def __init__(
436 - self,
437 - config: PhiMoEConfig,
438 - cache_config: Optional[CacheConfig] = None,
439 - quant_config: Optional[QuantizationConfig] = None,
440 - lora_config: Optional[LoRAConfig] = None,
441 - prefix: str = "",
442 - ) -> None:
436 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
437 super().__init__()
438 +
439 + config = vllm_config.model_config.hf_config
440 + cache_config = vllm_config.cache_config
441 + quant_config = vllm_config.quant_config
442 + lora_config = vllm_config.lora_config
443 +
444 self.padding_idx = config.pad_token_id
445 lora_vocab = ((lora_config.lora_extra_vocab_size *
446 (lora_config.max_loras or 1)) if lora_config else 0)
@@ -529,23 +529,15 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
529 }
530 embedding_padding_modules = ["lm_head"]
531
532 - def __init__(
533 - self,
534 - vllm_config: VllmConfig,
535 - prefix: str = "",
536 - ) -> None:
532 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
533 super().__init__()
534 config = vllm_config.model_config.hf_config
539 - cache_config = vllm_config.cache_config
540 - quant_config = vllm_config.quant_config
535 lora_config = vllm_config.lora_config
536 self.config = config
537 self.lora_config = lora_config
538
545 - self.model = PhiMoEModel(config,
546 - cache_config,
547 - quant_config,
548 - lora_config=lora_config)
539 + self.model = PhiMoEModel(vllm_config=vllm_config,
540 + prefix=maybe_prefix(prefix, "model"))
541 self.unpadded_vocab_size = config.vocab_size
542 if lora_config:
543 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/pixtral.py
+3 -7
@@ -38,7 +38,7 @@ from vllm.transformers_utils.processor import cached_get_processor
38 from vllm.utils import is_list_of
39
40 from .interfaces import SupportsMultiModal, SupportsPP
41 -from .utils import init_vllm_registered_model
41 +from .utils import init_vllm_registered_model, maybe_prefix
42
43 try:
44 from xformers import ops as xops
@@ -152,11 +152,7 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
152 class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
153 SupportsPP):
154
155 - def __init__(
156 - self,
157 - vllm_config: VllmConfig,
158 - prefix: str = "",
159 - ) -> None:
155 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
156 super().__init__()
157 config = vllm_config.model_config.hf_config
158 multimodal_config = vllm_config.model_config.multimodal_config
@@ -176,7 +172,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
172 self.language_model = init_vllm_registered_model(
173 config.text_config,
174 vllm_config=vllm_config,
179 - prefix="language_model")
175 + prefix=maybe_prefix(prefix, "language_model"))
176
177 self.vision_encoder = VisionTransformer(self.vision_args)
178 self.vision_language_adapter = VisionLanguageAdapter(
vllm/model_executor/models/qwen.py
+12 -15
@@ -50,7 +50,8 @@ from vllm.utils import is_list_of
50
51 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
52 from .utils import (flatten_bn, is_pp_missing_parameter,
53 - make_empty_intermediate_tensors_factory, make_layers)
53 + make_empty_intermediate_tensors_factory, make_layers,
54 + maybe_prefix)
55
56 logger = init_logger(__name__)
57
@@ -552,14 +553,13 @@ class QWenBlock(nn.Module):
553 @support_torch_compile
554 class QWenModel(nn.Module):
555
555 - def __init__(
556 - self,
557 - config: PretrainedConfig,
558 - cache_config: Optional[CacheConfig] = None,
559 - quant_config: Optional[QuantizationConfig] = None,
560 - prefix: str = "",
561 - ):
556 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
557 super().__init__()
558 +
559 + config = vllm_config.model_config.hf_config
560 + cache_config = vllm_config.cache_config
561 + quant_config = vllm_config.quant_config
562 +
563 self.config = config
564 self.vocab_size = config.vocab_size
565
@@ -865,20 +865,17 @@ def dummy_data_for_qwen(
865
866 class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
867
868 - def __init__(
869 - self,
870 - vllm_config: VllmConfig,
871 - prefix: str = "",
872 - ) -> None:
868 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
869 super().__init__()
870 config = vllm_config.model_config.hf_config
875 - cache_config = vllm_config.cache_config
871 quant_config = vllm_config.quant_config
872 multimodal_config = vllm_config.model_config.multimodal_config
873 self.config = config
874 self.multimodal_config = multimodal_config
875 self.quant_config = quant_config
881 - self.transformer = QWenModel(config, cache_config, quant_config)
876 + self.transformer = QWenModel(vllm_config=vllm_config,
877 + prefix=maybe_prefix(
878 + prefix, "transformer"))
879 self.lm_head = ParallelLMHead(config.vocab_size,
880 config.hidden_size,
881 quant_config=quant_config)
vllm/model_executor/models/qwen2.py
+8 -15
@@ -240,14 +240,13 @@ class Qwen2DecoderLayer(nn.Module):
240 @support_torch_compile
241 class Qwen2Model(nn.Module):
242
243 - def __init__(
244 - self,
245 - config: Qwen2Config,
246 - cache_config: Optional[CacheConfig] = None,
247 - quant_config: Optional[QuantizationConfig] = None,
248 - prefix: str = "",
249 - ) -> None:
243 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
244 super().__init__()
245 +
246 + config = vllm_config.model_config.hf_config
247 + cache_config = vllm_config.cache_config
248 + quant_config = vllm_config.quant_config
249 +
250 self.config = config
251 self.padding_idx = config.pad_token_id
252 self.vocab_size = config.vocab_size
@@ -403,11 +402,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
402 "up_proj": ("gate_up_proj", 1),
403 }
404
406 - def __init__(
407 - self,
408 - vllm_config: VllmConfig,
409 - prefix: str = "",
410 - ) -> None:
405 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
406 super().__init__()
407 config = vllm_config.model_config.hf_config
408 cache_config = vllm_config.cache_config
@@ -429,9 +424,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
424 self.lora_config = lora_config
425
426 self.quant_config = quant_config
432 - self.model = Qwen2Model(config,
433 - cache_config,
434 - quant_config,
427 + self.model = Qwen2Model(vllm_config=vllm_config,
428 prefix=maybe_prefix(prefix, "model"))
429
430 if config.tie_word_embeddings:
vllm/model_executor/models/qwen2_audio.py
+4 -8
@@ -264,14 +264,9 @@ def input_mapper_for_qwen2_audio(
264 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
265 SupportsPP):
266
267 - def __init__(
268 - self,
269 - vllm_config: VllmConfig,
270 - prefix: str = "",
271 - ) -> None:
267 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
268 super().__init__()
269 config = vllm_config.model_config.hf_config
274 - cache_config = vllm_config.cache_config
270 quant_config = vllm_config.quant_config
271 multimodal_config = vllm_config.model_config.multimodal_config
272 self.config = config
@@ -283,8 +278,9 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
278
279 self.quant_config = quant_config
280
286 - self.language_model = Qwen2Model(config.text_config, cache_config,
287 - quant_config)
281 + self.language_model = Qwen2Model(
282 + vllm_config=vllm_config.with_hf_config(config.text_config),
283 + prefix=prefix)
284 self.unpadded_vocab_size = config.text_config.vocab_size
285 if config.text_config.tie_word_embeddings:
286 self.lm_head = self.language_model.embed_tokens
vllm/model_executor/models/qwen2_cls.py
+4 -7
@@ -17,7 +17,7 @@ from vllm.model_executor.models.qwen2 import Qwen2Model
17 from vllm.model_executor.pooling_metadata import PoolingMetadata
18 from vllm.sequence import IntermediateTensors, PoolerOutput
19
20 -from .utils import AutoWeightsLoader
20 +from .utils import AutoWeightsLoader, maybe_prefix
21
22
23 class Qwen2ForSequenceClassification(nn.Module):
@@ -43,11 +43,7 @@ class Qwen2ForSequenceClassification(nn.Module):
43 embedding_modules = {}
44 embedding_padding_modules = []
45
46 - def __init__(
47 - self,
48 - vllm_config: VllmConfig,
49 - prefix: str = "",
50 - ) -> None:
46 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
47 super().__init__()
48 config = vllm_config.model_config.hf_config
49 cache_config = vllm_config.cache_config
@@ -70,7 +66,8 @@ class Qwen2ForSequenceClassification(nn.Module):
66 self.lora_config = lora_config
67
68 self.quant_config = quant_config
73 - self.model = Qwen2Model(config, cache_config, quant_config)
69 + self.model = Qwen2Model(vllm_config=vllm_config,
70 + prefix=maybe_prefix(prefix, "model"))
71
72 self.score = RowParallelLinear(config.hidden_size,
73 config.num_labels,
vllm/model_executor/models/qwen2_moe.py
+11 -15
@@ -54,7 +54,8 @@ from vllm.utils import print_warning_once
54
55 from .interfaces import SupportsPP
56 from .utils import (is_pp_missing_parameter,
57 - make_empty_intermediate_tensors_factory, make_layers)
57 + make_empty_intermediate_tensors_factory, make_layers,
58 + maybe_prefix)
59
60
61 class Qwen2MoeMLP(nn.Module):
@@ -315,14 +316,13 @@ class Qwen2MoeDecoderLayer(nn.Module):
316 @support_torch_compile
317 class Qwen2MoeModel(nn.Module):
318
318 - def __init__(
319 - self,
320 - config: PretrainedConfig,
321 - cache_config: Optional[CacheConfig] = None,
322 - quant_config: Optional[QuantizationConfig] = None,
323 - prefix: str = "",
324 - ) -> None:
319 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
320 super().__init__()
321 +
322 + config = vllm_config.model_config.hf_config
323 + cache_config = vllm_config.cache_config
324 + quant_config = vllm_config.quant_config
325 +
326 self.padding_idx = config.pad_token_id
327 self.vocab_size = config.vocab_size
328
@@ -377,18 +377,14 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
377
378 fall_back_to_pt_during_load = False
379
380 - def __init__(
381 - self,
382 - vllm_config: VllmConfig,
383 - prefix: str = "",
384 - ) -> None:
380 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
381 super().__init__()
382 config = vllm_config.model_config.hf_config
387 - cache_config = vllm_config.cache_config
383 quant_config = vllm_config.quant_config
384 self.config = config
385 self.quant_config = quant_config
391 - self.model = Qwen2MoeModel(config, cache_config, quant_config)
386 + self.model = Qwen2MoeModel(vllm_config=vllm_config,
387 + prefix=maybe_prefix(prefix, "model"))
388 self.lm_head = ParallelLMHead(config.vocab_size,
389 config.hidden_size,
390 quant_config=quant_config)
vllm/model_executor/models/qwen2_rm.py
+4 -7
@@ -18,7 +18,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
18
19 from .interfaces import SupportsPP
20 from .qwen2 import Qwen2Model
21 -from .utils import AutoWeightsLoader
21 +from .utils import AutoWeightsLoader, maybe_prefix
22
23
24 class ReLU(nn.Module):
@@ -55,11 +55,7 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
55 embedding_modules = {}
56 embedding_padding_modules = []
57
58 - def __init__(
59 - self,
60 - vllm_config: VllmConfig,
61 - prefix: str = "",
62 - ) -> None:
58 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
59 super().__init__()
60 config = vllm_config.model_config.hf_config
61 cache_config = vllm_config.cache_config
@@ -82,7 +78,8 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
78 self.lora_config = lora_config
79
80 self.quant_config = quant_config
85 - self.model = Qwen2Model(config, cache_config, quant_config)
81 + self.model = Qwen2Model(vllm_config=vllm_config,
82 + prefix=maybe_prefix(prefix, "model"))
83
84 self.score = nn.Sequential(
85 ColumnParallelLinear(config.hidden_size,
vllm/model_executor/models/qwen2_vl.py
+7 -12
@@ -70,7 +70,7 @@ from vllm.transformers_utils.processor import cached_get_processor
70 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
71 from .utils import (PPMissingLayer, get_vit_attn_backend,
72 is_pp_missing_parameter,
73 - make_empty_intermediate_tensors_factory)
73 + make_empty_intermediate_tensors_factory, maybe_prefix)
74
75 logger = init_logger(__name__)
76
@@ -966,11 +966,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
966 embedding_modules = {}
967 embedding_padding_modules = []
968
969 - def __init__(
970 - self,
971 - vllm_config: VllmConfig,
972 - prefix: str = "",
973 - ) -> None:
969 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
970 super().__init__()
971 config = vllm_config.model_config.hf_config
972 cache_config = vllm_config.cache_config
@@ -986,13 +982,11 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
982 config.vision_config,
983 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
984 quant_config=self._maybe_ignore_quant_config(quant_config),
989 - prefix="visual",
985 + prefix=maybe_prefix(prefix, "visual"),
986 )
987
992 - self.model = Qwen2Model(config,
993 - cache_config,
994 - quant_config,
995 - prefix="model")
988 + self.model = Qwen2Model(vllm_config=vllm_config,
989 + prefix=maybe_prefix(prefix, "model"))
990
991 if get_pp_group().is_last_rank:
992 if config.tie_word_embeddings:
@@ -1001,7 +995,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
995 self.lm_head = ParallelLMHead(config.vocab_size,
996 config.hidden_size,
997 quant_config=quant_config,
1004 - prefix="lm_head")
998 + prefix=maybe_prefix(
999 + prefix, "lm_head"))
1000 else:
1001 self.lm_head = PPMissingLayer()
1002
vllm/model_executor/models/solar.py
+13 -21
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 +from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -53,7 +53,8 @@ from vllm.sequence import IntermediateTensors
53
54 from .interfaces import SupportsLoRA, SupportsPP
55 from .utils import (PPMissingLayer, is_pp_missing_parameter,
56 - make_empty_intermediate_tensors_factory, make_layers)
56 + make_empty_intermediate_tensors_factory, make_layers,
57 + maybe_prefix)
58
59
60 class SolarMLP(nn.Module):
@@ -266,15 +267,14 @@ class SolarDecoderLayer(nn.Module):
267 @support_torch_compile
268 class SolarModel(nn.Module):
269
269 - def __init__(
270 - self,
271 - config: PretrainedConfig,
272 - cache_config: Optional[CacheConfig] = None,
273 - quant_config: Optional[QuantizationConfig] = None,
274 - lora_config: Optional[LoRAConfig] = None,
275 - prefix: str = "",
276 - ) -> None:
270 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271 super().__init__()
272 +
273 + config = vllm_config.model_config.hf_config
274 + cache_config = vllm_config.cache_config
275 + quant_config = vllm_config.quant_config
276 + lora_config = vllm_config.lora_config
277 +
278 self.config = config
279 self.padding_idx = config.pad_token_id
280 lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -409,25 +409,17 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
409 "up_proj": ("gate_up_proj", 1),
410 }
411
412 - def __init__(
413 - self,
414 - vllm_config: VllmConfig,
415 - prefix: str = "",
416 - ) -> None:
412 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
413 super().__init__()
414 config = vllm_config.model_config.hf_config
419 - cache_config = vllm_config.cache_config
415 quant_config = vllm_config.quant_config
416 lora_config = vllm_config.lora_config
417 self.config = config
418 self.lora_config = lora_config
419
420 self.model = SolarModel(
426 - config,
427 - cache_config,
428 - quant_config,
429 - lora_config=lora_config,
430 - prefix="model",
421 + vllm_config=vllm_config,
422 + prefix=maybe_prefix(prefix, "model"),
423 )
424 if get_pp_group().is_last_rank:
425 self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/stablelm.py
+11 -13
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44 from .interfaces import SupportsPP
45 from .utils import (is_pp_missing_parameter,
46 - make_empty_intermediate_tensors_factory, make_layers)
46 + make_empty_intermediate_tensors_factory, make_layers,
47 + maybe_prefix)
48
49
50 class StablelmMLP(nn.Module):
@@ -193,12 +194,13 @@ class StablelmDecoderLayer(nn.Module):
194
195 class StableLMEpochModel(nn.Module):
196
196 - def __init__(self,
197 - config: PretrainedConfig,
198 - cache_config: Optional[CacheConfig] = None,
199 - quant_config: Optional[QuantizationConfig] = None,
200 - prefix: str = '') -> None:
197 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
198 super().__init__()
199 +
200 + config = vllm_config.model_config.hf_config
201 + cache_config = vllm_config.cache_config
202 + quant_config = vllm_config.quant_config
203 +
204 self.embed_tokens = VocabParallelEmbedding(
205 config.vocab_size,
206 config.hidden_size,
@@ -245,18 +247,14 @@ class StableLMEpochModel(nn.Module):
247
248 class StablelmForCausalLM(nn.Module, SupportsPP):
249
248 - def __init__(
249 - self,
250 - vllm_config: VllmConfig,
251 - prefix: str = "",
252 - ) -> None:
250 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
251 super().__init__()
252 config = vllm_config.model_config.hf_config
255 - cache_config = vllm_config.cache_config
253 quant_config = vllm_config.quant_config
254 self.config = config
255 self.quant_config = quant_config
259 - self.model = StableLMEpochModel(config, cache_config, quant_config)
256 + self.model = StableLMEpochModel(vllm_config=vllm_config,
257 + prefix=maybe_prefix(prefix, "model"))
258 self.lm_head = ParallelLMHead(config.vocab_size,
259 config.hidden_size,
260 quant_config=quant_config)
vllm/model_executor/models/starcoder2.py
+11 -15
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44 from .interfaces import SupportsPP
45 from .utils import (is_pp_missing_parameter,
46 - make_empty_intermediate_tensors_factory, make_layers)
46 + make_empty_intermediate_tensors_factory, make_layers,
47 + maybe_prefix)
48
49
50 class Starcoder2Attention(nn.Module):
@@ -195,12 +196,13 @@ class Starcoder2DecoderLayer(nn.Module):
196 @support_torch_compile
197 class Starcoder2Model(nn.Module):
198
198 - def __init__(self,
199 - config: Starcoder2Config,
200 - cache_config: Optional[CacheConfig] = None,
201 - quant_config: Optional[QuantizationConfig] = None,
202 - prefix: str = ""):
199 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
200 super().__init__()
201 +
202 + config = vllm_config.model_config.hf_config
203 + cache_config = vllm_config.cache_config
204 + quant_config = vllm_config.quant_config
205 +
206 self.config = config
207 self.padding_idx = config.pad_token_id
208 self.vocab_size = config.vocab_size
@@ -245,19 +247,13 @@ class Starcoder2Model(nn.Module):
247
248 class Starcoder2ForCausalLM(nn.Module, SupportsPP):
249
248 - def __init__(
249 - self,
250 - vllm_config: VllmConfig,
251 - prefix: str = "",
252 - ) -> None:
250 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
251 super().__init__()
252 config = vllm_config.model_config.hf_config
255 - cache_config = vllm_config.cache_config
253 quant_config = vllm_config.quant_config
254 self.config = config
258 - self.model = Starcoder2Model(config,
259 - cache_config,
260 - quant_config=quant_config)
255 + self.model = Starcoder2Model(vllm_config=vllm_config,
256 + prefix=maybe_prefix(prefix, "model"))
257 self.vocab_size = config.vocab_size
258 self.unpadded_vocab_size = config.vocab_size
259 if config.tie_word_embeddings:
vllm/model_executor/models/ultravox.py
+9 -7
@@ -34,7 +34,7 @@ from vllm.utils import is_list_of
34
35 from .interfaces import SupportsMultiModal, SupportsPP
36 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
37 - init_vllm_registered_model,
37 + init_vllm_registered_model, maybe_prefix,
38 merge_multimodal_embeddings_from_map)
39
40 _AUDIO_PLACEHOLDER_TOKEN = 128002
@@ -339,11 +339,7 @@ class ModifiedWhisperEncoder(WhisperEncoder):
339 @INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox)
340 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
341
342 - def __init__(
343 - self,
344 - vllm_config: VllmConfig,
345 - prefix: str = "",
346 - ) -> None:
342 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
343 super().__init__()
344 config = vllm_config.model_config.hf_config
345 multimodal_config = vllm_config.model_config.multimodal_config
@@ -354,6 +350,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
350 self.secondary_weights = []
351 self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
352 if config.audio_model_id is not None:
353 + # this prefix is not for initialization, but for loading weights
354 + # note the trailing dot
355 self.secondary_weights.append(
356 DefaultModelLoader.Source(
357 model_or_path=config.audio_model_id,
@@ -362,8 +360,12 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
360 ))
361 self.multi_modal_projector = UltravoxProjector(config)
362 self.language_model = init_vllm_registered_model(
365 - config.text_config, vllm_config, prefix="language_model")
363 + config.text_config,
364 + vllm_config=vllm_config,
365 + prefix=maybe_prefix(prefix, "language_model"))
366 if config.text_model_id is not None:
367 + # this prefix is not for initialization, but for loading weights
368 + # note the trailing dot
369 self.secondary_weights.append(
370 DefaultModelLoader.Source(model_or_path=config.text_model_id,
371 revision=None,
vllm/model_executor/models/xverse.py
+6 -13
@@ -46,7 +46,8 @@ from vllm.sequence import IntermediateTensors
46
47 from .interfaces import SupportsLoRA, SupportsPP
48 from .utils import (is_pp_missing_parameter,
49 - make_empty_intermediate_tensors_factory, make_layers)
49 + make_empty_intermediate_tensors_factory, make_layers,
50 + maybe_prefix)
51
52
53 class XverseMLP(nn.Module):
@@ -223,11 +224,7 @@ class XverseDecoderLayer(nn.Module):
224 @support_torch_compile
225 class XverseModel(nn.Module):
226
226 - def __init__(
227 - self,
228 - vllm_config: VllmConfig,
229 - prefix: str = "",
230 - ) -> None:
227 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
228 super().__init__()
229 config = vllm_config.model_config.hf_config
230 cache_config = vllm_config.cache_config
@@ -315,15 +312,10 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
312 }
313 embedding_padding_modules = ["lm_head"]
314
318 - def __init__(
319 - self,
320 - vllm_config: VllmConfig,
321 - prefix: str = "",
322 - ) -> None:
315 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
316 super().__init__()
317
318 config = vllm_config.model_config.hf_config
326 - cache_config = vllm_config.cache_config
319 quant_config = vllm_config.quant_config
320 lora_config = vllm_config.lora_config
321
@@ -331,7 +323,8 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
323 self.lora_config = lora_config
324
325 self.quant_config = quant_config
334 - self.model = XverseModel(config, cache_config, quant_config)
326 + self.model = XverseModel(vllm_config=vllm_config,
327 + prefix=maybe_prefix(prefix, "model"))
328 self.lm_head = ParallelLMHead(config.vocab_size,
329 config.hidden_size,
330 quant_config=quant_config)