A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[6/N] pass whole config to inner model (#10205)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 10, 2024 at 22:41 UTC
f89d18ff74e48f97c76afbab31956218d2486e36
69 files changed
+681
-963
vllm/model_executor/models/arctic.py
+10
-13
@@ -34,7 +34,8 @@ from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35
from .interfaces import SupportsPP
36
from .utils import (is_pp_missing_parameter,
37
- make_empty_intermediate_tensors_factory, make_layers)
37
+ make_empty_intermediate_tensors_factory, make_layers,
38
+ maybe_prefix)
39
40
logger = init_logger(__name__)
41
@@ -364,14 +365,13 @@ class ArcticDecoderLayer(nn.Module):
365
@support_torch_compile
366
class ArcticModel(nn.Module):
367
367
- def __init__(
368
- self,
369
- config: ArcticConfig,
370
- cache_config: Optional[CacheConfig] = None,
371
- quant_config: Optional[QuantizationConfig] = None,
372
- prefix: str = "",
373
- ) -> None:
368
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
369
super().__init__()
370
+
371
+ config = vllm_config.model_config.hf_config
372
+ cache_config = vllm_config.cache_config
373
+ quant_config = vllm_config.quant_config
374
+
375
self.padding_idx = config.pad_token_id
376
self.vocab_size = config.vocab_size
377
self.embed_tokens = VocabParallelEmbedding(
@@ -418,13 +418,10 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
418
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
419
super().__init__()
420
config = vllm_config.model_config.hf_config
421
- cache_config = vllm_config.cache_config
421
quant_config = vllm_config.quant_config
422
self.config = config
424
- self.model = ArcticModel(config,
425
- cache_config,
426
- quant_config,
427
- prefix=prefix)
423
+ self.model = ArcticModel(vllm_config=vllm_config,
424
+ prefix=maybe_prefix(prefix, "model"))
425
self.vocab_size = config.vocab_size
426
self.lm_head = ParallelLMHead(
427
self.vocab_size,
vllm/model_executor/models/baichuan.py
+26
-22
@@ -253,13 +253,18 @@ class BaiChuanDecoderLayer(nn.Module):
253
@support_torch_compile
254
class BaiChuanModel(nn.Module):
255
256
- def __init__(self,
257
- config: PretrainedConfig,
258
- position_embedding: str,
259
- cache_config: Optional[CacheConfig] = None,
260
- quant_config: Optional[QuantizationConfig] = None,
261
- prefix: str = ""):
256
+ def __init__(
257
+ self,
258
+ vllm_config: VllmConfig,
259
+ prefix: str = "",
260
+ position_embedding: str = "ROPE",
261
+ ) -> None:
262
super().__init__()
263
+
264
+ config = vllm_config.model_config.hf_config
265
+ cache_config = vllm_config.cache_config
266
+ quant_config = vllm_config.quant_config
267
+
268
self.config = config
269
self.padding_idx = config.pad_token_id
270
self.vocab_size = config.vocab_size
@@ -332,21 +337,22 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
337
338
def __init__(
339
self,
340
+ *,
341
vllm_config: VllmConfig,
342
prefix: str = "",
343
position_embedding: str = "ROPE",
344
):
345
super().__init__()
346
config = vllm_config.model_config.hf_config
341
- cache_config = vllm_config.cache_config
347
quant_config = vllm_config.quant_config
348
lora_config = vllm_config.lora_config
349
self.config = config
350
self.lora_config = lora_config
351
352
self.quant_config = quant_config
348
- self.model = BaiChuanModel(config, position_embedding, cache_config,
349
- quant_config)
353
+ self.model = BaiChuanModel(vllm_config=vllm_config,
354
+ prefix=prefix,
355
+ position_embedding=position_embedding)
356
self.lm_head = ParallelLMHead(config.vocab_size,
357
config.hidden_size,
358
quant_config=quant_config)
@@ -438,16 +444,16 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
444
NOTE: the class name has a lower case 'c'.
445
"""
446
441
- def __init__(
442
- self,
443
- vllm_config: VllmConfig,
444
- prefix: str = "",
445
- ):
447
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
448
config = vllm_config.model_config.hf_config
449
if config.hidden_size == 4096: # baichuan2 7b
448
- super().__init__(vllm_config, prefix, "ROPE")
450
+ super().__init__(vllm_config=vllm_config,
451
+ prefix=prefix,
452
+ position_embedding="ROPE")
453
else: # baichuan 13b, baichuan2 13b
450
- super().__init__(vllm_config, prefix, "ALIBI")
454
+ super().__init__(vllm_config=vllm_config,
455
+ prefix=prefix,
456
+ position_embedding="ALIBI")
457
458
459
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -455,9 +461,7 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
461
NOTE: the class name has an upper case 'C'.
462
"""
463
458
- def __init__(
459
- self,
460
- vllm_config: VllmConfig,
461
- prefix: str = "",
462
- ):
463
- super().__init__(vllm_config, prefix, "ROPE")
464
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
465
+ super().__init__(vllm_config=vllm_config,
466
+ prefix=prefix,
467
+ position_embedding="ROPE")
vllm/model_executor/models/bart.py
+11
-12
@@ -41,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import IntermediateTensors
43
44
+from .utils import maybe_prefix
45
+
46
logger = logging.get_logger(__name__)
47
48
@@ -739,13 +741,14 @@ class BartModel(nn.Module):
741
"encoder.embed_tokens.weight", "decoder.embed_tokens.weight"
742
]
743
742
- def __init__(self,
743
- config: BartConfig,
744
- cache_config: Optional[CacheConfig] = None,
745
- quant_config: Optional[QuantizationConfig] = None,
746
- lora_config: Optional[LoRAConfig] = None):
744
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
745
super().__init__()
746
747
+ config = vllm_config.model_config.hf_config
748
+ cache_config = vllm_config.cache_config
749
+ quant_config = vllm_config.quant_config
750
+ lora_config = vllm_config.lora_config
751
+
752
self.config = config
753
754
self.padding_idx = config.pad_token_id
@@ -810,20 +813,16 @@ class BartModel(nn.Module):
813
class BartForConditionalGeneration(nn.Module):
814
base_model_prefix = "model"
815
813
- def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
816
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
817
818
super().__init__()
819
config = vllm_config.model_config.hf_config
817
- cache_config = vllm_config.cache_config
818
- quant_config = vllm_config.quant_config
820
lora_config = vllm_config.lora_config
821
# currently all existing BART models have `tie_word_embeddings` enabled
822
assert config.tie_word_embeddings
823
self.config = config
823
- self.model = BartModel(config,
824
- cache_config,
825
- quant_config,
826
- lora_config=lora_config)
824
+ self.model = BartModel(vllm_config=vllm_config,
825
+ prefix=maybe_prefix(prefix, "model"))
826
827
self.unpadded_vocab_size = config.vocab_size
828
if lora_config:
vllm/model_executor/models/bert.py
+11
-14
@@ -21,6 +21,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
from vllm.model_executor.pooling_metadata import PoolingMetadata
22
from vllm.sequence import IntermediateTensors, PoolerOutput
23
24
+from .utils import maybe_prefix
25
+
26
27
class BertEmbedding(nn.Module):
28
@@ -309,12 +311,13 @@ class BertOutput(nn.Module):
311
312
class BertModel(nn.Module):
313
312
- def __init__(self,
313
- config: BertConfig,
314
- cache_config: Optional[CacheConfig] = None,
315
- quant_config: Optional[QuantizationConfig] = None,
316
- prefix: str = ""):
314
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
315
super().__init__()
316
+
317
+ config = vllm_config.model_config.hf_config
318
+ cache_config = vllm_config.cache_config
319
+ quant_config = vllm_config.quant_config
320
+
321
self.embeddings = BertEmbedding(config)
322
self.encoder = BertEncoder(config,
323
cache_config,
@@ -382,17 +385,11 @@ class BertEmbeddingModel(nn.Module):
385
_pooler: An instance of Pooler used for pooling operations.
386
"""
387
385
- def __init__(
386
- self,
387
- vllm_config: VllmConfig,
388
- prefix: str = "",
389
- ) -> None:
388
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
389
super().__init__()
391
- config = vllm_config.model_config.hf_config
392
- cache_config = vllm_config.cache_config
393
- quant_config = vllm_config.quant_config
390
pooler_config = vllm_config.model_config.pooler_config
395
- self.model = BertModel(config, cache_config, quant_config)
391
+ self.model = BertModel(vllm_config=vllm_config,
392
+ prefix=maybe_prefix(prefix, "model"))
393
self._pooler = Pooler.from_config_with_defaults(
394
pooler_config,
395
pooling_type=PoolingType.CLS,
vllm/model_executor/models/blip2.py
+3
-7
@@ -23,7 +23,7 @@ from .blip import (BlipVisionModel, dummy_image_for_blip,
23
get_max_blip_image_tokens)
24
from .interfaces import SupportsMultiModal, SupportsPP
25
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
26
- merge_multimodal_embeddings)
26
+ maybe_prefix, merge_multimodal_embeddings)
27
28
# We use this internally as placeholders since there is no image token
29
# defined on the HuggingFace repo
@@ -483,11 +483,7 @@ def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
483
@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
484
class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
485
486
- def __init__(
487
- self,
488
- vllm_config: VllmConfig,
489
- prefix: str = "",
490
- ) -> None:
486
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
487
488
super().__init__()
489
config = vllm_config.model_config.hf_config
@@ -517,7 +513,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
513
self.language_model = init_vllm_registered_model(
514
config.text_config,
515
vllm_config=vllm_config,
520
- prefix="language_model")
516
+ prefix=maybe_prefix(prefix, "language_model"))
517
518
self.make_empty_intermediate_tensors = (
519
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/bloom.py
+11
-10
@@ -42,7 +42,8 @@ from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsPP
44
from .utils import (is_pp_missing_parameter,
45
- make_empty_intermediate_tensors_factory, make_layers)
45
+ make_empty_intermediate_tensors_factory, make_layers,
46
+ maybe_prefix)
47
48
49
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
@@ -221,14 +222,13 @@ class BloomBlock(nn.Module):
222
@support_torch_compile
223
class BloomModel(nn.Module):
224
224
- def __init__(
225
- self,
226
- config: BloomConfig,
227
- cache_config: Optional[CacheConfig] = None,
228
- quant_config: Optional[QuantizationConfig] = None,
229
- prefix: str = "",
230
- ):
225
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
226
super().__init__()
227
+
228
+ config = vllm_config.model_config.hf_config
229
+ cache_config = vllm_config.cache_config
230
+ quant_config = vllm_config.quant_config
231
+
232
self.embed_dim = config.hidden_size
233
234
# Embedding + LN Embedding
@@ -288,11 +288,12 @@ class BloomForCausalLM(nn.Module, SupportsPP):
288
):
289
super().__init__()
290
config = vllm_config.model_config.hf_config
291
- cache_config = vllm_config.cache_config
291
quant_config = vllm_config.quant_config
292
self.config = config
293
self.quant_config = quant_config
295
- self.transformer = BloomModel(config, cache_config, quant_config)
294
+ self.transformer = BloomModel(vllm_config=vllm_config,
295
+ prefix=maybe_prefix(
296
+ prefix, "transformer"))
297
if self.config.tie_word_embeddings:
298
self.lm_head = self.transformer.word_embeddings
299
else:
vllm/model_executor/models/chameleon.py
+11
-16
@@ -37,7 +37,8 @@ from vllm.utils import print_warning_once
37
38
from .interfaces import SupportsMultiModal, SupportsPP
39
from .utils import (is_pp_missing_parameter,
40
- make_empty_intermediate_tensors_factory, make_layers)
40
+ make_empty_intermediate_tensors_factory, make_layers,
41
+ maybe_prefix)
42
43
# These configs are not part of the model config but the preprocessor
44
# and processor files, so we hardcode them in the model file for now.
@@ -831,14 +832,13 @@ class ChameleonImageVocabularyMapping:
832
833
class ChameleonModel(nn.Module):
834
834
- def __init__(
835
- self,
836
- config: ChameleonConfig,
837
- cache_config: Optional[CacheConfig] = None,
838
- quant_config: Optional[QuantizationConfig] = None,
839
- prefix: str = "",
840
- ) -> None:
835
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
836
super().__init__()
837
+
838
+ config = vllm_config.model_config.hf_config
839
+ cache_config = vllm_config.cache_config
840
+ quant_config = vllm_config.quant_config
841
+
842
self.config = config
843
self.padding_idx = config.pad_token_id
844
self.vocab_size = config.vocab_size
@@ -924,19 +924,14 @@ class ChameleonModel(nn.Module):
924
class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
925
SupportsPP):
926
927
- def __init__(
928
- self,
929
- vllm_config: VllmConfig,
930
- prefix: str = "",
931
- ) -> None:
927
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
928
super().__init__()
929
config = vllm_config.model_config.hf_config
934
- cache_config = vllm_config.cache_config
935
- quant_config = vllm_config.quant_config
930
multimodal_config = vllm_config.model_config.multimodal_config
931
self.config = config
932
self.multimodal_config = multimodal_config
939
- self.model = ChameleonModel(config, cache_config, quant_config)
933
+ self.model = ChameleonModel(vllm_config=vllm_config,
934
+ prefix=maybe_prefix(prefix, "model"))
935
self.unpadded_vocab_size = config.vocab_size
936
self.lm_head = ParallelLMHead(
937
self.unpadded_vocab_size,
vllm/model_executor/models/chatglm.py
+10
-9
@@ -39,7 +39,8 @@ from vllm.transformers_utils.configs import ChatGLMConfig
39
40
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
41
from .utils import (is_pp_missing_parameter,
42
- make_empty_intermediate_tensors_factory, make_layers)
42
+ make_empty_intermediate_tensors_factory, make_layers,
43
+ maybe_prefix)
44
45
logger = init_logger(__name__)
46
@@ -481,14 +482,13 @@ class GLMTransformer(nn.Module):
482
483
class ChatGLMModel(nn.Module):
484
484
- def __init__(
485
- self,
486
- config: ChatGLMConfig,
487
- cache_config: Optional[CacheConfig] = None,
488
- quant_config: Optional[QuantizationConfig] = None,
489
- ):
485
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
486
super().__init__()
487
488
+ config = vllm_config.model_config.hf_config
489
+ cache_config = vllm_config.cache_config
490
+ quant_config = vllm_config.quant_config
491
+
492
self.config = config
493
494
self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
@@ -600,7 +600,6 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
600
):
601
super().__init__()
602
config = vllm_config.model_config.hf_config
603
- cache_config = vllm_config.cache_config
603
quant_config = vllm_config.quant_config
604
lora_config = vllm_config.lora_config
605
multimodal_config = vllm_config.model_config.multimodal_config
@@ -611,7 +610,9 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
610
self.quant_config = quant_config
611
self.max_position_embeddings = getattr(config, "max_sequence_length",
612
8192)
614
- self.transformer = ChatGLMModel(config, cache_config, quant_config)
613
+ self.transformer = ChatGLMModel(vllm_config=vllm_config,
614
+ prefix=maybe_prefix(
615
+ prefix, "transformer"))
616
if self.config.tie_word_embeddings:
617
self.transformer.output_layer.weight = (
618
self.transformer.embedding.weight)
vllm/model_executor/models/commandr.py
+13
-20
@@ -28,7 +28,7 @@ from transformers import CohereConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -49,7 +49,8 @@ from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA, SupportsPP
51
from .utils import (is_pp_missing_parameter,
52
- make_empty_intermediate_tensors_factory, make_layers)
52
+ make_empty_intermediate_tensors_factory, make_layers,
53
+ maybe_prefix)
54
55
56
@torch.compile
@@ -253,15 +254,14 @@ class CohereDecoderLayer(nn.Module):
254
@support_torch_compile
255
class CohereModel(nn.Module):
256
256
- def __init__(
257
- self,
258
- config: CohereConfig,
259
- cache_config: Optional[CacheConfig] = None,
260
- quant_config: Optional[QuantizationConfig] = None,
261
- lora_config: Optional[LoRAConfig] = None,
262
- prefix: str = "",
263
- ):
257
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
258
super().__init__()
259
+
260
+ config = vllm_config.model_config.hf_config
261
+ cache_config = vllm_config.cache_config
262
+ quant_config = vllm_config.quant_config
263
+ lora_config = vllm_config.lora_config
264
+
265
self.config = config
266
lora_vocab = (lora_config.lora_extra_vocab_size *
267
(lora_config.max_loras or 1)) if lora_config else 0
@@ -332,14 +332,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
embedding_modules = {"embed_tokens": "input_embeddings"}
333
embedding_padding_modules = []
334
335
- def __init__(
336
- self,
337
- vllm_config: VllmConfig,
338
- prefix: str = "",
339
- ) -> None:
335
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
336
super().__init__()
337
config = vllm_config.model_config.hf_config
342
- cache_config = vllm_config.cache_config
338
quant_config = vllm_config.quant_config
339
lora_config = vllm_config.lora_config
340
self.config = config
@@ -353,10 +348,8 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
348
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
349
config.vocab_size,
350
scale=config.logit_scale)
356
- self.model = CohereModel(config,
357
- cache_config,
358
- quant_config,
359
- lora_config=lora_config)
351
+ self.model = CohereModel(vllm_config=vllm_config,
352
+ prefix=maybe_prefix(prefix, "model"))
353
self.sampler = get_sampler()
354
self.make_empty_intermediate_tensors = (
355
self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/dbrx.py
+11
-10
@@ -25,7 +25,8 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
25
26
from .interfaces import SupportsPP
27
from .utils import (is_pp_missing_parameter,
28
- make_empty_intermediate_tensors_factory, make_layers)
28
+ make_empty_intermediate_tensors_factory, make_layers,
29
+ maybe_prefix)
30
31
32
class DbrxRouter(nn.Module):
@@ -294,14 +295,13 @@ class DbrxBlock(nn.Module):
295
296
class DbrxModel(nn.Module):
297
297
- def __init__(
298
- self,
299
- config: DbrxConfig,
300
- cache_config: Optional[CacheConfig] = None,
301
- quant_config: Optional[QuantizationConfig] = None,
302
- prefix: str = "",
303
- ):
298
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
299
super().__init__()
300
+
301
+ config = vllm_config.model_config.hf_config
302
+ cache_config = vllm_config.cache_config
303
+ quant_config = vllm_config.quant_config
304
+
305
self.wte = VocabParallelEmbedding(
306
config.vocab_size,
307
config.d_model,
@@ -357,7 +357,6 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
357
):
358
super().__init__()
359
config = vllm_config.model_config.hf_config
360
- cache_config = vllm_config.cache_config
360
quant_config = vllm_config.quant_config
361
self.config = config
362
if config.tie_word_embeddings:
@@ -365,7 +364,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
364
"tie_word_embeddings is not supported for Dbrx models.")
365
self.quant_config = quant_config
366
self.unpadded_vocab_size = config.vocab_size
368
- self.transformer = DbrxModel(config, cache_config, quant_config)
367
+ self.transformer = DbrxModel(vllm_config=vllm_config,
368
+ prefix=maybe_prefix(
369
+ prefix, "transformer"))
370
self.lm_head = ParallelLMHead(
371
config.vocab_size,
372
config.d_model,
vllm/model_executor/models/decilm.py
+1
-5
@@ -51,11 +51,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
51
instead.
52
"""
53
54
- def __init__(
55
- self,
56
- vllm_config: VllmConfig,
57
- prefix: str = "",
58
- ) -> None:
54
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
55
config = vllm_config.model_config.hf_config
56
config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
57
delattr(config, "num_key_value_heads_per_layer")
vllm/model_executor/models/deepseek.py
+11
-15
@@ -50,7 +50,8 @@ from vllm.sequence import IntermediateTensors
50
51
from .interfaces import SupportsPP
52
from .utils import (is_pp_missing_parameter,
53
- make_empty_intermediate_tensors_factory, make_layers)
53
+ make_empty_intermediate_tensors_factory, make_layers,
54
+ maybe_prefix)
55
56
57
class DeepseekMLP(nn.Module):
@@ -326,14 +327,13 @@ class DeepseekModel(nn.Module):
327
328
fall_back_to_pt_during_load = False
329
329
- def __init__(
330
- self,
331
- config: PretrainedConfig,
332
- cache_config: Optional[CacheConfig] = None,
333
- quant_config: Optional[QuantizationConfig] = None,
334
- prefix: str = "",
335
- ) -> None:
330
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
331
super().__init__()
332
+
333
+ config = vllm_config.model_config.hf_config
334
+ cache_config = vllm_config.cache_config
335
+ quant_config = vllm_config.quant_config
336
+
337
self.padding_idx = config.pad_token_id
338
self.vocab_size = config.vocab_size
339
@@ -383,18 +383,14 @@ class DeepseekModel(nn.Module):
383
384
class DeepseekForCausalLM(nn.Module, SupportsPP):
385
386
- def __init__(
387
- self,
388
- vllm_config: VllmConfig,
389
- prefix: str = "",
390
- ) -> None:
386
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
387
super().__init__()
388
config = vllm_config.model_config.hf_config
393
- cache_config = vllm_config.cache_config
389
quant_config = vllm_config.quant_config
390
self.config = config
391
self.quant_config = quant_config
397
- self.model = DeepseekModel(config, cache_config, quant_config)
392
+ self.model = DeepseekModel(vllm_config=vllm_config,
393
+ prefix=maybe_prefix(prefix, "model"))
394
self.lm_head = ParallelLMHead(config.vocab_size,
395
config.hidden_size,
396
quant_config=quant_config)
vllm/model_executor/models/deepseek_v2.py
+11
-18
@@ -51,7 +51,8 @@ from vllm.sequence import IntermediateTensors
51
52
from .interfaces import SupportsPP
53
from .utils import (PPMissingLayer, is_pp_missing_parameter,
54
- make_empty_intermediate_tensors_factory, make_layers)
54
+ make_empty_intermediate_tensors_factory, make_layers,
55
+ maybe_prefix)
56
57
58
class DeepseekV2MLP(nn.Module):
@@ -408,14 +409,13 @@ class DeepseekV2Model(nn.Module):
409
410
fall_back_to_pt_during_load = False
411
411
- def __init__(
412
- self,
413
- config: PretrainedConfig,
414
- cache_config: Optional[CacheConfig] = None,
415
- quant_config: Optional[QuantizationConfig] = None,
416
- prefix: str = "",
417
- ) -> None:
412
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
413
super().__init__()
414
+
415
+ config = vllm_config.model_config.hf_config
416
+ cache_config = vllm_config.cache_config
417
+ quant_config = vllm_config.quant_config
418
+
419
self.padding_idx = config.pad_token_id
420
self.vocab_size = config.vocab_size
421
@@ -479,21 +479,14 @@ class DeepseekV2Model(nn.Module):
479
480
class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
481
482
- def __init__(
483
- self,
484
- vllm_config: VllmConfig,
485
- prefix: str = "",
486
- ) -> None:
482
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
483
super().__init__()
484
config = vllm_config.model_config.hf_config
489
- cache_config = vllm_config.cache_config
485
quant_config = vllm_config.quant_config
486
self.config = config
487
self.quant_config = quant_config
493
- self.model = DeepseekV2Model(config,
494
- cache_config,
495
- quant_config,
496
- prefix="model")
488
+ self.model = DeepseekV2Model(vllm_config=vllm_config,
489
+ prefix=maybe_prefix(prefix, "model"))
490
self.lm_head = ParallelLMHead(config.vocab_size,
491
config.hidden_size,
492
quant_config=quant_config)
vllm/model_executor/models/eagle.py
+4
-1
@@ -14,6 +14,8 @@ from vllm.model_executor.models import ModelRegistry
14
from vllm.model_executor.sampling_metadata import SamplingMetadata
15
from vllm.sequence import IntermediateTensors
16
17
+from .utils import maybe_prefix
18
+
19
20
class EAGLE(nn.Module):
21
"""This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077
@@ -42,7 +44,8 @@ class EAGLE(nn.Module):
44
architectures = getattr(self.config.model, "architectures", [])
45
model_cls, _ = ModelRegistry.resolve_model_cls(architectures)
46
45
- self.model = model_cls(vllm_config, prefix)
47
+ self.model = model_cls(vllm_config=vllm_config,
48
+ prefix=maybe_prefix(prefix, "model"))
49
self.fc = nn.Linear(config.model.hidden_size * 2,
50
config.model.hidden_size,
51
bias=getattr(self.config, "eagle_fc_bias", False))
vllm/model_executor/models/exaone.py
+13
-21
@@ -29,7 +29,7 @@ from torch import nn
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
+from vllm.config import CacheConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -54,7 +54,8 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
54
55
from .interfaces import SupportsLoRA, SupportsPP
56
from .utils import (PPMissingLayer, is_pp_missing_parameter,
57
- make_empty_intermediate_tensors_factory, make_layers)
57
+ make_empty_intermediate_tensors_factory, make_layers,
58
+ maybe_prefix)
59
60
61
class ExaoneGatedMLP(nn.Module):
@@ -314,15 +315,14 @@ class ExaoneDecoderLayer(nn.Module):
315
@support_torch_compile
316
class ExaoneModel(nn.Module):
317
317
- def __init__(
318
- self,
319
- config: ExaoneConfig,
320
- cache_config: Optional[CacheConfig] = None,
321
- quant_config: Optional[QuantizationConfig] = None,
322
- lora_config: Optional[LoRAConfig] = None,
323
- prefix: str = "",
324
- ) -> None:
318
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
319
super().__init__()
320
+
321
+ config = vllm_config.model_config.hf_config
322
+ cache_config = vllm_config.cache_config
323
+ quant_config = vllm_config.quant_config
324
+ lora_config = vllm_config.lora_config
325
+
326
self.config = config
327
self.padding_idx = config.pad_token_id
328
lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -438,14 +438,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
438
"c_fc_1": ("gate_up_proj", 1),
439
}
440
441
- def __init__(
442
- self,
443
- vllm_config: VllmConfig,
444
- prefix: str = "",
445
- ) -> None:
441
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
442
super().__init__()
443
config = vllm_config.model_config.hf_config
448
- cache_config = vllm_config.cache_config
444
quant_config = vllm_config.quant_config
445
lora_config = vllm_config.lora_config
446
@@ -453,11 +448,8 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
448
self.lora_config = lora_config
449
450
self.transformer = ExaoneModel(
456
- config,
457
- cache_config,
458
- quant_config,
459
- lora_config=lora_config,
460
- prefix="model",
451
+ vllm_config=vllm_config,
452
+ prefix=maybe_prefix(prefix, "model"),
453
)
454
if get_pp_group().is_last_rank:
455
self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/falcon.py
+11
-10
@@ -48,7 +48,8 @@ from vllm.transformers_utils.configs import RWConfig
48
49
from .interfaces import SupportsPP
50
from .utils import (is_pp_missing_parameter,
51
- make_empty_intermediate_tensors_factory, make_layers)
51
+ make_empty_intermediate_tensors_factory, make_layers,
52
+ maybe_prefix)
53
54
FalconConfig = Union[HF_FalconConfig, RWConfig]
55
@@ -332,14 +333,13 @@ class FalconDecoderLayer(nn.Module):
333
@support_torch_compile
334
class FalconModel(nn.Module):
335
335
- def __init__(
336
- self,
337
- config: FalconConfig,
338
- cache_config: Optional[CacheConfig] = None,
339
- quant_config: Optional[QuantizationConfig] = None,
340
- prefix: str = "",
341
- ):
336
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
337
super().__init__()
338
+
339
+ config = vllm_config.model_config.hf_config
340
+ cache_config = vllm_config.cache_config
341
+ quant_config = vllm_config.quant_config
342
+
343
self.config = config
344
self.embed_dim = config.hidden_size
345
self.num_heads = config.num_attention_heads
@@ -408,11 +408,12 @@ class FalconForCausalLM(nn.Module, SupportsPP):
408
):
409
super().__init__()
410
config = vllm_config.model_config.hf_config
411
- cache_config = vllm_config.cache_config
411
quant_config = vllm_config.quant_config
412
self.config = config
413
self.quant_config = quant_config
415
- self.transformer = FalconModel(config, cache_config, quant_config)
414
+ self.transformer = FalconModel(vllm_config=vllm_config,
415
+ prefix=maybe_prefix(
416
+ prefix, "transformer"))
417
# only Falcon-11B doesn't share lm_head weight with word embeddings
418
# and previous Falcon model doesn't have tie_word_embeddings config
419
# so we set tie_word_embeddings to True by default
vllm/model_executor/models/florence2.py
+17
-21
@@ -3,13 +3,10 @@ from typing import Iterable, List, Optional, Tuple
3
4
import torch
5
import torch.nn as nn
6
-from transformers import PretrainedConfig
6
7
from vllm.attention import AttentionMetadata
9
-from vllm.config import CacheConfig, VllmConfig
8
+from vllm.config import VllmConfig
9
from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
-from vllm.model_executor.layers.quantization.base_config import (
12
- QuantizationConfig)
10
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
11
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
12
from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
@@ -23,11 +20,13 @@ from .utils import AutoWeightsLoader
20
21
class Florence2LanguageModel(nn.Module):
22
26
- def __init__(self,
27
- config: PretrainedConfig,
28
- cache_config: Optional[CacheConfig] = None,
29
- quant_config: Optional[QuantizationConfig] = None):
23
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
24
super().__init__()
25
+
26
+ config = vllm_config.model_config.hf_config
27
+ cache_config = vllm_config.cache_config
28
+ quant_config = vllm_config.quant_config
29
+
30
self.config = config
31
32
self.padding_idx = config.pad_token_id
@@ -93,15 +92,14 @@ class Florence2LanguageModel(nn.Module):
92
93
class Florence2LanguageForConditionalGeneration(nn.Module):
94
96
- def __init__(self,
97
- config: PretrainedConfig,
98
- cache_config: Optional[CacheConfig] = None,
99
- quant_config: Optional[QuantizationConfig] = None):
95
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
96
super().__init__()
97
+
98
+ config = vllm_config.model_config.hf_config
99
+
100
self.config = config
102
- self.model = Florence2LanguageModel(config,
103
- cache_config=cache_config,
104
- quant_config=quant_config)
101
+ self.model = Florence2LanguageModel(vllm_config=vllm_config,
102
+ prefix=prefix)
103
embed_scale = math.sqrt(
104
config.d_model) if config.scale_embedding else 1.0
105
@@ -189,17 +187,15 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
187
188
class Florence2ForConditionalGeneration(nn.Module):
189
192
- def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
190
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
191
super().__init__()
192
config = vllm_config.model_config.hf_config
195
- cache_config = vllm_config.cache_config
196
- quant_config = vllm_config.quant_config
193
194
# TODO(Isotr0py): Add vision backbone
195
self.language_model = Florence2LanguageForConditionalGeneration(
200
- config=config.text_config,
201
- cache_config=cache_config,
202
- quant_config=quant_config)
196
+ vllm_config=vllm_config.with_hf_config(config.text_config),
197
+ prefix=prefix,
198
+ )
199
200
@property
201
def sampler(self):
vllm/model_executor/models/gemma.py
+8
-16
@@ -258,14 +258,13 @@ class GemmaDecoderLayer(nn.Module):
258
@support_torch_compile
259
class GemmaModel(nn.Module):
260
261
- def __init__(
262
- self,
263
- config: GemmaConfig,
264
- cache_config: Optional[CacheConfig] = None,
265
- quant_config: Optional[QuantizationConfig] = None,
266
- prefix: str = "",
267
- ) -> None:
261
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
262
super().__init__()
263
+
264
+ config = vllm_config.model_config.hf_config
265
+ cache_config = vllm_config.cache_config
266
+ quant_config = vllm_config.quant_config
267
+
268
self.config = config
269
270
self.embed_tokens = VocabParallelEmbedding(
@@ -372,14 +371,9 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
371
embedding_modules = {}
372
embedding_padding_modules = []
373
375
- def __init__(
376
- self,
377
- vllm_config: VllmConfig,
378
- prefix: str = "",
379
- ) -> None:
374
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
375
super().__init__()
376
config = vllm_config.model_config.hf_config
382
- cache_config = vllm_config.cache_config
377
quant_config = vllm_config.quant_config
378
lora_config = vllm_config.lora_config
379
@@ -389,9 +383,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
383
self.lora_config = lora_config
384
385
self.quant_config = quant_config
392
- self.model = GemmaModel(config,
393
- cache_config,
394
- quant_config,
386
+ self.model = GemmaModel(vllm_config=vllm_config,
387
prefix=maybe_prefix(prefix, "model"))
388
self.logits_processor = LogitsProcessor(config.vocab_size)
389
self.sampler = get_sampler()
vllm/model_executor/models/gemma2.py
+9
-19
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
43
44
from .interfaces import SupportsLoRA, SupportsPP
45
from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
46
- make_empty_intermediate_tensors_factory, make_layers)
46
+ make_empty_intermediate_tensors_factory, make_layers,
47
+ maybe_prefix)
48
49
logger = init_logger(__name__)
50
@@ -243,11 +244,7 @@ class Gemma2DecoderLayer(nn.Module):
244
@support_torch_compile
245
class Gemma2Model(nn.Module):
246
246
- def __init__(
247
- self,
248
- vllm_config: VllmConfig,
249
- prefix: str = "",
250
- ) -> None:
247
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
248
super().__init__()
249
config = vllm_config.model_config.hf_config
250
cache_config = vllm_config.cache_config
@@ -399,13 +396,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
396
"up_proj": ("gate_up_proj", 1),
397
}
398
402
- def __init__(
403
- self,
404
- vllm_config: VllmConfig,
405
- prefix: str = "",
406
- ) -> None:
399
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
400
config = vllm_config.model_config.hf_config
408
- cache_config = vllm_config.cache_config
401
quant_config = vllm_config.quant_config
402
lora_config = vllm_config.lora_config
403
del lora_config # Unused.
@@ -414,7 +406,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
406
# currently all existing Gemma models have `tie_word_embeddings` enabled
407
assert config.tie_word_embeddings
408
self.quant_config = quant_config
417
- self.model = Gemma2Model(config, cache_config, quant_config)
409
+ self.model = Gemma2Model(vllm_config=vllm_config,
410
+ prefix=maybe_prefix(prefix, "model"))
411
self.logits_processor = LogitsProcessor(
412
config.vocab_size, soft_cap=config.final_logit_softcapping)
413
self.sampler = get_sampler()
@@ -471,14 +464,11 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
464
_pooler: An instance of Pooler used for pooling operations.
465
"""
466
474
- def __init__(
475
- self,
476
- vllm_config: VllmConfig,
477
- prefix: str = "",
478
- ) -> None:
467
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
468
super().__init__()
469
481
- self.model = Gemma2Model(vllm_config, prefix)
470
+ self.model = Gemma2Model(vllm_config=vllm_config,
471
+ prefix=maybe_prefix(prefix, "model"))
472
self._pooler = Pooler.from_config_with_defaults(
473
vllm_config.model_config.pooler_config,
474
pooling_type=PoolingType.LAST,
vllm/model_executor/models/gpt2.py
+11
-13
@@ -42,7 +42,8 @@ from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsPP
44
from .utils import (is_pp_missing_parameter,
45
- make_empty_intermediate_tensors_factory, make_layers)
45
+ make_empty_intermediate_tensors_factory, make_layers,
46
+ maybe_prefix)
47
48
49
class GPT2Attention(nn.Module):
@@ -184,14 +185,13 @@ class GPT2Block(nn.Module):
185
@support_torch_compile
186
class GPT2Model(nn.Module):
187
187
- def __init__(
188
- self,
189
- config: GPT2Config,
190
- cache_config: Optional[CacheConfig] = None,
191
- quant_config: Optional[QuantizationConfig] = None,
192
- prefix: str = "",
193
- ):
188
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
189
super().__init__()
190
+
191
+ config = vllm_config.model_config.hf_config
192
+ cache_config = vllm_config.cache_config
193
+ quant_config = vllm_config.quant_config
194
+
195
self.config = config
196
assert not config.add_cross_attention
197
assert not config.scale_attn_by_inverse_layer_idx
@@ -247,14 +247,12 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
247
):
248
super().__init__()
249
config = vllm_config.model_config.hf_config
250
- cache_config = vllm_config.cache_config
250
quant_config = vllm_config.quant_config
251
self.config = config
252
self.quant_config = quant_config
254
- self.transformer = GPT2Model(config,
255
- cache_config,
256
- quant_config,
257
- prefix="transformer")
253
+ self.transformer = GPT2Model(vllm_config=vllm_config,
254
+ prefix=maybe_prefix(
255
+ prefix, "transformer"))
256
if self.config.tie_word_embeddings:
257
self.lm_head = self.transformer.wte
258
else:
vllm/model_executor/models/gpt_bigcode.py
+10
-12
@@ -25,7 +25,7 @@ from transformers import GPTBigCodeConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
from vllm.compilation.decorators import support_torch_compile
28
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
28
+from vllm.config import CacheConfig, VllmConfig
29
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -189,15 +189,14 @@ class GPTBigCodeBlock(nn.Module):
189
@support_torch_compile
190
class GPTBigCodeModel(nn.Module):
191
192
- def __init__(
193
- self,
194
- config: GPTBigCodeConfig,
195
- cache_config: Optional[CacheConfig] = None,
196
- quant_config: Optional[QuantizationConfig] = None,
197
- lora_config: Optional[LoRAConfig] = None,
198
- prefix: str = "",
199
- ):
192
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
193
super().__init__()
194
+
195
+ config = vllm_config.model_config.hf_config
196
+ cache_config = vllm_config.cache_config
197
+ quant_config = vllm_config.quant_config
198
+ lora_config = vllm_config.lora_config
199
+
200
self.config = config
201
assert not config.add_cross_attention
202
@@ -265,7 +264,6 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
264
):
265
super().__init__()
266
config = vllm_config.model_config.hf_config
268
- cache_config = vllm_config.cache_config
267
quant_config = vllm_config.quant_config
268
lora_config = vllm_config.lora_config
269
@@ -273,8 +271,8 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
271
self.lora_config = lora_config
272
273
self.quant_config = quant_config
276
- self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
277
- lora_config)
274
+ self.transformer = GPTBigCodeModel(vllm_config=vllm_config,
275
+ prefix=prefix)
276
if self.config.tie_word_embeddings:
277
self.lm_head = self.transformer.wte
278
else:
vllm/model_executor/models/gpt_j.py
+11
-10
@@ -42,7 +42,8 @@ from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsPP
44
from .utils import (is_pp_missing_parameter,
45
- make_empty_intermediate_tensors_factory, make_layers)
45
+ make_empty_intermediate_tensors_factory, make_layers,
46
+ maybe_prefix)
47
48
49
class GPTJAttention(nn.Module):
@@ -177,14 +178,13 @@ class GPTJBlock(nn.Module):
178
@support_torch_compile
179
class GPTJModel(nn.Module):
180
180
- def __init__(
181
- self,
182
- config: GPTJConfig,
183
- cache_config: Optional[CacheConfig] = None,
184
- quant_config: Optional[QuantizationConfig] = None,
185
- prefix: str = "",
186
- ):
181
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
182
super().__init__()
183
+
184
+ config = vllm_config.model_config.hf_config
185
+ cache_config = vllm_config.cache_config
186
+ quant_config = vllm_config.quant_config
187
+
188
self.config = config
189
self.embed_dim = config.n_embd
190
self.wte = VocabParallelEmbedding(
@@ -236,12 +236,13 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
236
):
237
super().__init__()
238
config = vllm_config.model_config.hf_config
239
- cache_config = vllm_config.cache_config
239
quant_config = vllm_config.quant_config
240
self.config = config
241
self.quant_config = quant_config
242
assert not config.tie_word_embeddings
244
- self.transformer = GPTJModel(config, cache_config, quant_config)
243
+ self.transformer = GPTJModel(vllm_config=vllm_config,
244
+ prefix=maybe_prefix(
245
+ prefix, "transformer"))
246
self.lm_head = ParallelLMHead(
247
config.vocab_size,
248
config.n_embd,
vllm/model_executor/models/gpt_neox.py
+10
-10
@@ -41,7 +41,8 @@ from vllm.sequence import IntermediateTensors
41
42
from .interfaces import SupportsPP
43
from .utils import (is_pp_missing_parameter,
44
- make_empty_intermediate_tensors_factory, make_layers)
44
+ make_empty_intermediate_tensors_factory, make_layers,
45
+ maybe_prefix)
46
47
48
class GPTNeoXAttention(nn.Module):
@@ -189,14 +190,13 @@ class GPTNeoXLayer(nn.Module):
190
@support_torch_compile
191
class GPTNeoXModel(nn.Module):
192
192
- def __init__(
193
- self,
194
- config: GPTNeoXConfig,
195
- cache_config: Optional[CacheConfig] = None,
196
- quant_config: Optional[QuantizationConfig] = None,
197
- prefix: str = "",
198
- ):
193
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
194
super().__init__()
195
+
196
+ config = vllm_config.model_config.hf_config
197
+ cache_config = vllm_config.cache_config
198
+ quant_config = vllm_config.quant_config
199
+
200
self.config = config
201
202
self.embed_in = VocabParallelEmbedding(
@@ -249,11 +249,11 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
249
):
250
super().__init__()
251
config = vllm_config.model_config.hf_config
252
- cache_config = vllm_config.cache_config
252
quant_config = vllm_config.quant_config
253
self.config = config
254
self.quant_config = quant_config
256
- self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config)
255
+ self.gpt_neox = GPTNeoXModel(vllm_config=vllm_config,
256
+ prefix=maybe_prefix(prefix, "gpt_neox"))
257
self.embed_out = ParallelLMHead(
258
config.vocab_size,
259
config.hidden_size,
vllm/model_executor/models/granite.py
+13
-21
@@ -28,7 +28,7 @@ from transformers import GraniteConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -52,7 +52,8 @@ from vllm.platforms import current_platform
52
from vllm.sequence import IntermediateTensors
53
54
from .interfaces import SupportsLoRA, SupportsPP
55
-from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
55
+from .utils import (PPMissingLayer, is_pp_missing_parameter, make_layers,
56
+ maybe_prefix)
57
58
59
class GraniteMLP(nn.Module):
@@ -257,15 +258,14 @@ class GraniteDecoderLayer(nn.Module):
258
@support_torch_compile
259
class GraniteModel(nn.Module):
260
260
- def __init__(
261
- self,
262
- config: GraniteConfig,
263
- cache_config: Optional[CacheConfig] = None,
264
- quant_config: Optional[QuantizationConfig] = None,
265
- lora_config: Optional[LoRAConfig] = None,
266
- prefix: str = "",
267
- ) -> None:
261
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
262
super().__init__()
263
+
264
+ config = vllm_config.model_config.hf_config
265
+ cache_config = vllm_config.cache_config
266
+ quant_config = vllm_config.quant_config
267
+ lora_config = vllm_config.lora_config
268
+
269
self.config = config
270
self.padding_idx = config.pad_token_id
271
lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -370,25 +370,17 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
370
"up_proj": ("gate_up_proj", 1),
371
}
372
373
- def __init__(
374
- self,
375
- vllm_config: VllmConfig,
376
- prefix: str = "",
377
- ) -> None:
373
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
374
super().__init__()
375
config = vllm_config.model_config.hf_config
380
- cache_config = vllm_config.cache_config
376
quant_config = vllm_config.quant_config
377
lora_config = vllm_config.lora_config
378
379
self.config = config
380
self.lora_config = lora_config
381
387
- self.model = GraniteModel(config,
388
- cache_config,
389
- quant_config,
390
- lora_config=lora_config,
391
- prefix="model")
382
+ self.model = GraniteModel(vllm_config=vllm_config,
383
+ prefix=maybe_prefix(prefix, "model"))
384
if get_pp_group().is_last_rank:
385
self.unpadded_vocab_size = config.vocab_size
386
if lora_config:
vllm/model_executor/models/granitemoe.py
+12
-21
@@ -28,7 +28,7 @@ from transformers.models.granitemoe import GraniteMoeConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -47,7 +47,7 @@ from vllm.sequence import IntermediateTensors
47
48
from . import mixtral
49
from .interfaces import SupportsLoRA, SupportsPP
50
-from .utils import make_layers
50
+from .utils import make_layers, maybe_prefix
51
52
53
class GraniteMoeMoE(nn.Module):
@@ -247,15 +247,14 @@ class GraniteMoeDecoderLayer(nn.Module):
247
@support_torch_compile
248
class GraniteMoeModel(nn.Module):
249
250
- def __init__(
251
- self,
252
- config: GraniteMoeConfig,
253
- cache_config: Optional[CacheConfig] = None,
254
- quant_config: Optional[QuantizationConfig] = None,
255
- lora_config: Optional[LoRAConfig] = None,
256
- prefix: str = "",
257
- ) -> None:
250
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
251
super().__init__()
252
+
253
+ config = vllm_config.model_config.hf_config
254
+ cache_config = vllm_config.cache_config
255
+ quant_config = vllm_config.quant_config
256
+ lora_config = vllm_config.lora_config
257
+
258
self.padding_idx = config.pad_token_id
259
lora_vocab = (lora_config.lora_extra_vocab_size *
260
(lora_config.max_loras or 1)) if lora_config else 0
@@ -333,25 +332,17 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
}
333
embedding_padding_modules = ["lm_head"]
334
336
- def __init__(
337
- self,
338
- vllm_config: VllmConfig,
339
- prefix: str = "",
340
- ) -> None:
335
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
336
super().__init__()
337
config = vllm_config.model_config.hf_config
343
- cache_config = vllm_config.cache_config
338
quant_config = vllm_config.quant_config
339
lora_config = vllm_config.lora_config
340
341
self.config = config
342
self.lora_config = lora_config
343
350
- self.model = GraniteMoeModel(config,
351
- cache_config,
352
- quant_config,
353
- lora_config=lora_config,
354
- prefix="model")
344
+ self.model = GraniteMoeModel(vllm_config=vllm_config,
345
+ prefix=maybe_prefix(prefix, "model"))
346
self.unpadded_vocab_size = config.vocab_size
347
if lora_config:
348
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/idefics3.py
+12
-17
@@ -22,17 +22,15 @@ import torch.utils.checkpoint
22
from PIL import Image
23
from torch import nn
24
# Temporary solution for transformers below 4.46.0.
25
-from transformers import PretrainedConfig as Idefics3Config
25
from transformers import ProcessorMixin as Idefics3ImageProcessor
26
27
from vllm.attention import AttentionMetadata
29
-from vllm.config import CacheConfig, VllmConfig
28
+from vllm.config import VllmConfig
29
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30
InputContext, token_inputs)
31
from vllm.logger import init_logger
32
from vllm.model_executor.layers.linear import ReplicatedLinear
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
-from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -48,7 +46,8 @@ from .idefics2_vision_model import (
46
# yapf: enable
47
from .interfaces import SupportsMultiModal
48
from .llama import LlamaModel
51
-from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
49
+from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
50
+ merge_multimodal_embeddings)
51
52
logger = init_logger(__name__)
53
@@ -417,13 +416,13 @@ class Idefics3Connector(nn.Module):
416
417
class Idefics3Model(nn.Module):
418
420
- def __init__(
421
- self,
422
- config: Idefics3Config,
423
- cache_config: Optional[CacheConfig] = None,
424
- quant_config: Optional[QuantizationConfig] = None,
425
- ):
419
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
420
super().__init__()
421
+
422
+ config = vllm_config.model_config.hf_config
423
+ cache_config = vllm_config.cache_config
424
+ quant_config = vllm_config.quant_config
425
+
426
self.config = config
427
self.padding_idx = self.config.text_config.pad_token_id
428
self.vocab_size = self.config.text_config.vocab_size
@@ -613,22 +612,18 @@ class Idefics3Model(nn.Module):
612
@INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
613
class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
614
616
- def __init__(
617
- self,
618
- vllm_config: VllmConfig,
619
- prefix: str = "",
620
- ) -> None:
615
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
616
super().__init__()
617
618
config = vllm_config.model_config.hf_config
624
- cache_config = vllm_config.cache_config
619
quant_config = vllm_config.quant_config
620
multimodal_config = vllm_config.model_config.multimodal_config
621
622
self.config = config
623
self.multimodal_config = multimodal_config
624
631
- self.model = Idefics3Model(config, cache_config, quant_config)
625
+ self.model = Idefics3Model(vllm_config=vllm_config,
626
+ prefix=maybe_prefix(prefix, "model"))
627
self.image_token_id = self.config.image_token_id
628
629
self.lm_head = ParallelLMHead(
vllm/model_executor/models/internlm2.py
+8
-16
@@ -250,14 +250,13 @@ class InternLMDecoderLayer(nn.Module):
250
@support_torch_compile
251
class InternLM2Model(nn.Module):
252
253
- def __init__(
254
- self,
255
- config: PretrainedConfig,
256
- cache_config: Optional[CacheConfig] = None,
257
- quant_config: Optional[QuantizationConfig] = None,
258
- prefix: str = "",
259
- ) -> None:
253
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
254
super().__init__()
255
+
256
+ config = vllm_config.model_config.hf_config
257
+ cache_config = vllm_config.cache_config
258
+ quant_config = vllm_config.quant_config
259
+
260
self.config = config
261
self.padding_idx = config.pad_token_id
262
self.vocab_size = config.vocab_size
@@ -317,20 +316,13 @@ class InternLM2Model(nn.Module):
316
317
class InternLM2ForCausalLM(nn.Module, SupportsPP):
318
320
- def __init__(
321
- self,
322
- vllm_config: VllmConfig,
323
- prefix: str = "",
324
- ) -> None:
319
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
320
super().__init__()
321
config = vllm_config.model_config.hf_config
327
- cache_config = vllm_config.cache_config
322
quant_config = vllm_config.quant_config
323
self.config = config
324
self.quant_config = quant_config
331
- self.model = InternLM2Model(config,
332
- cache_config,
333
- quant_config,
325
+ self.model = InternLM2Model(vllm_config=vllm_config,
326
prefix=maybe_prefix(prefix, "model"))
327
self.output = ParallelLMHead(config.vocab_size,
328
config.hidden_size,
vllm/model_executor/models/internlm2_ve.py
+9
-14
@@ -104,14 +104,13 @@ class InternLM2VEDecoderLayer(nn.Module):
104
105
class InternLM2VEModel(InternLM2Model):
106
107
- def __init__(
108
- self,
109
- config: PretrainedConfig,
110
- cache_config: Optional[CacheConfig] = None,
111
- quant_config: Optional[QuantizationConfig] = None,
112
- prefix: str = "",
113
- ) -> None:
114
- super().__init__(config, cache_config, quant_config)
107
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
108
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
109
+
110
+ config = vllm_config.model_config.hf_config
111
+ cache_config = vllm_config.cache_config
112
+ quant_config = vllm_config.quant_config
113
+
114
self.start_layer, self.end_layer, self.layers = make_layers(
115
config.num_hidden_layers,
116
lambda prefix: InternLM2VEDecoderLayer(
@@ -159,12 +158,8 @@ class InternLM2VEModel(InternLM2Model):
158
159
class InternLM2VEForCausalLM(InternLM2ForCausalLM):
160
162
- def __init__(
163
- self,
164
- vllm_config: VllmConfig,
165
- prefix: str = "",
166
- ) -> None:
167
- super().__init__(vllm_config, prefix=prefix)
161
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
162
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
163
164
config = vllm_config.model_config.hf_config
165
cache_config = vllm_config.cache_config
vllm/model_executor/models/internvl.py
+3
-3
@@ -35,7 +35,7 @@ from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
35
get_clip_num_patches)
36
from .interfaces import SupportsMultiModal, SupportsPP
37
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38
- merge_multimodal_embeddings)
38
+ maybe_prefix, merge_multimodal_embeddings)
39
40
IMG_START = '<img>'
41
IMG_END = '</img>'
@@ -435,13 +435,13 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
435
config,
436
quant_config=quant_config,
437
is_mono=self.is_mono,
438
- prefix="vision_model",
438
+ prefix=maybe_prefix(prefix, "vision_model"),
439
)
440
441
self.language_model = init_vllm_registered_model(
442
config.text_config,
443
vllm_config=vllm_config,
444
- prefix="language_model")
444
+ prefix=maybe_prefix(prefix, "language_model"))
445
446
self.mlp1 = self._init_mlp1(config)
447
vllm/model_executor/models/jais.py
+11
-10
@@ -44,7 +44,8 @@ from vllm.transformers_utils.configs import JAISConfig
44
45
from .interfaces import SupportsPP
46
from .utils import (is_pp_missing_parameter,
47
- make_empty_intermediate_tensors_factory, make_layers)
47
+ make_empty_intermediate_tensors_factory, make_layers,
48
+ maybe_prefix)
49
50
51
class SwiGLUActivation(nn.Module):
@@ -215,14 +216,13 @@ class JAISBlock(nn.Module):
216
@support_torch_compile
217
class JAISModel(nn.Module):
218
218
- def __init__(
219
- self,
220
- config: JAISConfig,
221
- cache_config: Optional[CacheConfig] = None,
222
- quant_config: Optional[QuantizationConfig] = None,
223
- prefix: str = "",
224
- ):
219
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
220
super().__init__()
221
+
222
+ config = vllm_config.model_config.hf_config
223
+ cache_config = vllm_config.cache_config
224
+ quant_config = vllm_config.quant_config
225
+
226
self.config = config
227
assert not config.add_cross_attention
228
assert not config.scale_attn_by_inverse_layer_idx
@@ -293,11 +293,12 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
293
):
294
super().__init__()
295
config = vllm_config.model_config.hf_config
296
- cache_config = vllm_config.cache_config
296
quant_config = vllm_config.quant_config
297
self.config = config
298
self.quant_config = quant_config
300
- self.transformer = JAISModel(config, cache_config, quant_config)
299
+ self.transformer = JAISModel(vllm_config=vllm_config,
300
+ prefix=maybe_prefix(
301
+ prefix, "transformer"))
302
if self.config.tie_word_embeddings:
303
self.lm_head = self.transformer.wte
304
else:
vllm/model_executor/models/jamba.py
+12
-18
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
from vllm.attention.layer import Attention
10
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
10
+from vllm.config import CacheConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
12
from vllm.model_executor.layers.fused_moe import FusedMoE
13
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -29,6 +29,7 @@ from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
29
_get_graph_batch_size)
30
31
from .interfaces import HasInnerState, SupportsLoRA
32
+from .utils import maybe_prefix
33
34
KVCache = Tuple[torch.Tensor, torch.Tensor]
35
@@ -258,14 +259,14 @@ ALL_DECODER_LAYER_TYPES = {
259
260
class JambaModel(nn.Module):
261
261
- def __init__(
262
- self,
263
- config: JambaConfig,
264
- quant_config: Optional[QuantizationConfig] = None,
265
- cache_config: Optional[CacheConfig] = None,
266
- lora_config: Optional[LoRAConfig] = None,
267
- ) -> None:
262
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
263
super().__init__()
264
+
265
+ config = vllm_config.model_config.hf_config
266
+ cache_config = vllm_config.cache_config
267
+ quant_config = vllm_config.quant_config
268
+ lora_config = vllm_config.lora_config
269
+
270
self.config = config
271
self.padding_idx = config.pad_token_id
272
lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -348,14 +349,9 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
349
}
350
embedding_padding_modules = ["lm_head"]
351
351
- def __init__(
352
- self,
353
- vllm_config: VllmConfig,
354
- prefix: str = "",
355
- ) -> None:
352
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
353
config = vllm_config.model_config.hf_config
354
cache_config = vllm_config.cache_config
358
- quant_config = vllm_config.quant_config
355
lora_config = vllm_config.lora_config
356
scheduler_config = vllm_config.scheduler_config
357
assert not cache_config.enable_prefix_caching, \
@@ -364,10 +360,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
360
super().__init__()
361
self.config = config
362
self.scheduler_config = scheduler_config
367
- self.model = JambaModel(config,
368
- cache_config=cache_config,
369
- quant_config=quant_config,
370
- lora_config=lora_config)
363
+ self.model = JambaModel(vllm_config=vllm_config,
364
+ prefix=maybe_prefix(prefix, "model"))
365
self.unpadded_vocab_size = config.vocab_size
366
if lora_config:
367
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/llama.py
+12
-32
@@ -28,7 +28,7 @@ from transformers import LlamaConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -271,15 +271,14 @@ class LlamaDecoderLayer(nn.Module):
271
@support_torch_compile
272
class LlamaModel(nn.Module):
273
274
- def __init__(
275
- self,
276
- config: LlamaConfig,
277
- cache_config: Optional[CacheConfig] = None,
278
- quant_config: Optional[QuantizationConfig] = None,
279
- lora_config: Optional[LoRAConfig] = None,
280
- prefix: str = "",
281
- ) -> None:
274
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
275
super().__init__()
276
+
277
+ config = vllm_config.model_config.hf_config
278
+ cache_config = vllm_config.cache_config
279
+ quant_config = vllm_config.quant_config
280
+ lora_config = vllm_config.lora_config
281
+
282
self.config = config
283
self.padding_idx = config.pad_token_id
284
lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -492,24 +491,16 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
491
"norm": "model.norm"
492
}
493
495
- def __init__(
496
- self,
497
- vllm_config: VllmConfig,
498
- prefix: str = "",
499
- ) -> None:
494
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
495
super().__init__()
496
config = vllm_config.model_config.hf_config
502
- cache_config = vllm_config.cache_config
497
quant_config = vllm_config.quant_config
498
lora_config = vllm_config.lora_config
499
pooler_config = vllm_config.model_config.pooler_config
500
self.config = config
501
self.lora_config = lora_config
502
509
- self.model = LlamaModel(config,
510
- cache_config,
511
- quant_config,
512
- lora_config=lora_config,
503
+ self.model = LlamaModel(vllm_config=vllm_config,
504
prefix=maybe_prefix(prefix, "model"))
505
if get_pp_group().is_last_rank:
506
self.unpadded_vocab_size = config.vocab_size
@@ -652,23 +643,12 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
643
}
644
embedding_padding_modules = []
645
655
- def __init__(
656
- self,
657
- vllm_config: VllmConfig,
658
- prefix: str = "",
659
- ) -> None:
646
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
647
super().__init__()
648
662
- config = vllm_config.model_config.hf_config
663
- cache_config = vllm_config.cache_config
664
- quant_config = vllm_config.quant_config
665
- lora_config = vllm_config.lora_config
649
pooler_config = vllm_config.model_config.pooler_config
650
668
- self.model = LlamaModel(config,
669
- cache_config,
670
- quant_config,
671
- lora_config,
651
+ self.model = LlamaModel(vllm_config=vllm_config,
652
prefix=maybe_prefix(prefix, "model"))
653
self._pooler = Pooler.from_config_with_defaults(
654
pooler_config,
vllm/model_executor/models/llava.py
+3
-3
@@ -32,7 +32,7 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
32
dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
33
input_processor_for_siglip)
34
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
35
- merge_multimodal_embeddings)
35
+ maybe_prefix, merge_multimodal_embeddings)
36
37
38
class LlavaImagePixelInputs(TypedDict):
@@ -282,7 +282,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
282
config,
283
quant_config,
284
require_post_norm=False,
285
- prefix="vision_tower")
285
+ prefix=maybe_prefix(prefix, "vision_tower"))
286
self.multi_modal_projector = LlavaMultiModalProjector(
287
vision_hidden_size=config.vision_config.hidden_size,
288
text_hidden_size=config.text_config.hidden_size,
@@ -291,7 +291,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
291
self.language_model = init_vllm_registered_model(
292
config.text_config,
293
vllm_config=vllm_config,
294
- prefix="language_model")
294
+ prefix=maybe_prefix(prefix, "language_model"))
295
296
self.make_empty_intermediate_tensors = (
297
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_next.py
+3
-3
@@ -31,7 +31,7 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
32
get_siglip_patch_grid_length, input_processor_for_siglip)
33
from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
34
- init_vllm_registered_model)
34
+ init_vllm_registered_model, maybe_prefix)
35
36
37
class LlavaNextImagePixelInputs(TypedDict):
@@ -296,7 +296,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
296
config,
297
quant_config,
298
require_post_norm=False,
299
- prefix="vision_tower")
299
+ prefix=maybe_prefix(prefix, "vision_tower"))
300
self.image_newline = nn.Parameter(
301
torch.empty(config.text_config.hidden_size))
302
self.multi_modal_projector = LlavaMultiModalProjector(
@@ -307,7 +307,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
307
self.language_model = init_vllm_registered_model(
308
config.text_config,
309
vllm_config=vllm_config,
310
- prefix="language_model")
310
+ prefix=maybe_prefix(prefix, "language_model"))
311
312
# The same model class supports both language generation and embedding
313
# because the architecture name is the same
vllm/model_executor/models/llava_next_video.py
+3
-3
@@ -29,7 +29,7 @@ from .llava import init_vision_tower_for_llava
29
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30
dummy_seq_data_for_siglip)
31
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
32
- merge_multimodal_embeddings)
32
+ maybe_prefix, merge_multimodal_embeddings)
33
34
# For profile run
35
_MAX_FRAMES_PER_VIDEO = 32
@@ -267,7 +267,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
267
config,
268
quant_config,
269
require_post_norm=False,
270
- prefix="vision_tower")
270
+ prefix=maybe_prefix(prefix, "vision_tower"))
271
self.vision_resampler = LlavaNextVideoPooler(config)
272
self.multi_modal_projector = LlavaNextMultiModalProjector(
273
vision_hidden_size=config.vision_config.hidden_size,
@@ -276,7 +276,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
276
self.language_model = init_vllm_registered_model(
277
config.text_config,
278
vllm_config=vllm_config,
279
- prefix="language_model")
279
+ prefix=maybe_prefix(prefix, "language_model"))
280
281
self.make_empty_intermediate_tensors = (
282
self.language_model.model.make_empty_intermediate_tensors)
vllm/model_executor/models/llava_onevision.py
+3
-3
@@ -35,7 +35,7 @@ from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
35
dummy_video_for_siglip, get_siglip_image_feature_size,
36
get_siglip_patch_grid_length, input_processor_for_siglip)
37
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38
- merge_multimodal_embeddings)
38
+ maybe_prefix, merge_multimodal_embeddings)
39
40
# Result in the max possible feature size (2x2 grid of 336x336px tiles)
41
MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
@@ -418,12 +418,12 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
418
config,
419
quant_config,
420
require_post_norm=False,
421
- prefix="vision_tower")
421
+ prefix=maybe_prefix(prefix, "vision_tower"))
422
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
423
self.language_model = init_vllm_registered_model(
424
config.text_config,
425
vllm_config=vllm_config,
426
- prefix="language_model")
426
+ prefix=maybe_prefix(prefix, "language_model"))
427
self.image_newline = nn.Parameter(
428
torch.empty(config.text_config.hidden_size))
429
vllm/model_executor/models/mamba.py
+13
-18
@@ -6,7 +6,7 @@ from torch import nn
6
from transformers import MambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
9
+from vllm.config import CacheConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -26,6 +26,8 @@ from vllm.sequence import IntermediateTensors
26
from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
27
_get_graph_batch_size)
28
29
+from .utils import maybe_prefix
30
+
31
KVCache = Tuple[torch.Tensor, torch.Tensor]
32
33
@@ -73,14 +75,14 @@ class MambaDecoderLayer(nn.Module):
75
76
class MambaModel(nn.Module):
77
76
- def __init__(
77
- self,
78
- config: MambaConfig,
79
- quant_config: Optional[QuantizationConfig] = None,
80
- cache_config: Optional[CacheConfig] = None,
81
- lora_config: Optional[LoRAConfig] = None,
82
- ) -> None:
78
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
79
super().__init__()
80
+
81
+ config = vllm_config.model_config.hf_config
82
+ cache_config = vllm_config.cache_config
83
+ quant_config = vllm_config.quant_config
84
+ lora_config = vllm_config.lora_config
85
+
86
self.config = config
87
self.padding_idx = config.pad_token_id
88
lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -130,14 +132,9 @@ class MambaModel(nn.Module):
132
133
class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
134
133
- def __init__(
134
- self,
135
- vllm_config: VllmConfig,
136
- prefix: str = "",
137
- ) -> None:
135
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
136
config = vllm_config.model_config.hf_config
137
cache_config = vllm_config.cache_config
140
- quant_config = vllm_config.quant_config
138
lora_config = vllm_config.lora_config
139
scheduler_config = vllm_config.scheduler_config
140
assert not cache_config.enable_prefix_caching, \
@@ -146,10 +143,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
143
super().__init__()
144
self.config = config
145
self.scheduler_config = scheduler_config
149
- self.backbone = MambaModel(config,
150
- cache_config=cache_config,
151
- quant_config=quant_config,
152
- lora_config=lora_config)
146
+ self.backbone = MambaModel(vllm_config=vllm_config,
147
+ prefix=maybe_prefix(prefix, "backbone"))
148
self.unpadded_vocab_size = config.vocab_size
149
if lora_config:
150
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/minicpm.py
+17
-21
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
+from vllm.config import CacheConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
@@ -53,7 +53,8 @@ from vllm.sequence import IntermediateTensors
53
54
from .interfaces import SupportsLoRA, SupportsPP
55
from .utils import (is_pp_missing_parameter,
56
- make_empty_intermediate_tensors_factory, make_layers)
56
+ make_empty_intermediate_tensors_factory, make_layers,
57
+ maybe_prefix)
58
59
60
class MiniCPMMoE(nn.Module):
@@ -351,15 +352,14 @@ class MiniCPMDecoderLayer(nn.Module):
352
@support_torch_compile
353
class MiniCPMModel(nn.Module):
354
354
- def __init__(
355
- self,
356
- config: PretrainedConfig,
357
- cache_config: Optional[CacheConfig] = None,
358
- quant_config: Optional[QuantizationConfig] = None,
359
- lora_config: Optional[LoRAConfig] = None,
360
- prefix: str = "",
361
- ) -> None:
355
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
356
super().__init__()
357
+
358
+ config = vllm_config.model_config.hf_config
359
+ cache_config = vllm_config.cache_config
360
+ quant_config = vllm_config.quant_config
361
+ lora_config = vllm_config.lora_config
362
+
363
self.config = config
364
self.cache_config = cache_config
365
self.quant_config = quant_config
@@ -461,24 +461,22 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
461
}
462
embedding_padding_modules = ["lm_head"]
463
464
- def __init__(
465
- self,
466
- vllm_config: VllmConfig,
467
- prefix: str = "",
468
- ) -> None:
464
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
465
super().__init__()
466
config = vllm_config.model_config.hf_config
467
cache_config = vllm_config.cache_config
468
quant_config = vllm_config.quant_config
469
lora_config = vllm_config.lora_config
470
471
+ self.prefix = prefix
472
+ self.vllm_config = vllm_config
473
self.config = config
474
self.lora_config = lora_config
475
self.cache_config = cache_config
476
self.quant_config = quant_config
477
478
self.num_experts = getattr(self.config, "num_experts", 0)
481
- self._init_model()
479
+ self._init_model(vllm_config=vllm_config, prefix=prefix)
480
unpadded_vocab_size = config.vocab_size
481
if lora_config:
482
unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -502,11 +500,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
500
self.make_empty_intermediate_tensors = (
501
self.model.make_empty_intermediate_tensors)
502
505
- def _init_model(self):
506
- self.model = MiniCPMModel(config=self.config,
507
- cache_config=self.cache_config,
508
- quant_config=self.quant_config,
509
- lora_config=self.lora_config)
503
+ def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
504
+ self.model = MiniCPMModel(vllm_config=vllm_config,
505
+ prefix=maybe_prefix(prefix, "model"))
506
507
def forward(
508
self,
vllm/model_executor/models/minicpm3.py
+5
-7
@@ -28,7 +28,7 @@ from torch import nn
28
from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
-from vllm.config import CacheConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -40,7 +40,7 @@ from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer,
40
MiniCPMForCausalLM,
41
MiniCPMModel)
42
43
-from .utils import make_layers
43
+from .utils import make_layers, maybe_prefix
44
45
46
class MiniCPM3Attention(nn.Module):
@@ -238,8 +238,6 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
238
# `embedding_modules` and `embedding_padding_modules`
239
# are inherited from MiniCPMForCausalLM
240
241
- def _init_model(self):
242
- self.model = MiniCPM3Model(config=self.config,
243
- cache_config=self.cache_config,
244
- quant_config=self.quant_config,
245
- lora_config=self.lora_config)
241
+ def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
242
+ self.model = MiniCPM3Model(vllm_config=vllm_config,
243
+ prefix=maybe_prefix(prefix, "model"))
vllm/model_executor/models/minicpmv.py
+19
-37
@@ -34,7 +34,7 @@ from transformers import PretrainedConfig
34
from typing_extensions import NotRequired
35
36
from vllm.attention import AttentionMetadata
37
-from vllm.config import CacheConfig, VllmConfig
37
+from vllm.config import VllmConfig
38
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39
InputContext, token_inputs)
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -59,7 +59,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
59
60
from .idefics2_vision_model import Idefics2VisionTransformer
61
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
62
-from .utils import is_pp_missing_parameter
62
+from .utils import is_pp_missing_parameter, maybe_prefix
63
64
_KEYS_TO_MODIFY_MAPPING = {
65
"llm.lm_head": "lm_head",
@@ -390,7 +390,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
390
):
391
config = vllm_config.model_config.hf_config
392
multimodal_config = vllm_config.model_config.multimodal_config
393
- cache_config = vllm_config.cache_config
393
quant_config = vllm_config.quant_config
394
super().__init__()
395
# All MiniCPM-V models disable `tie_word_embeddings` but
@@ -401,11 +400,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
400
self.multimodal_config = multimodal_config
401
402
self.version = get_version_by_config(self.config)
404
- self.llm = self.init_llm(config,
405
- cache_config,
406
- quant_config,
407
- prefix="llm")
408
- self.vpm = self.init_vision_module(config, quant_config, prefix="vpm")
403
+ self.llm = self.init_llm(vllm_config=vllm_config,
404
+ prefix=maybe_prefix(prefix, "llm"))
405
+ self.vpm = self.init_vision_module(config,
406
+ quant_config,
407
+ prefix=maybe_prefix(prefix, "vpm"))
408
param_dtype = torch.get_default_dtype()
409
self.vpm.to(dtype=param_dtype)
410
self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
@@ -414,13 +413,15 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
413
self.resampler = self.init_resampler(self.embed_dim,
414
self.vision_dim,
415
quant_config=quant_config,
417
- prefix="resampler")
416
+ prefix=maybe_prefix(
417
+ prefix, "resampler"))
418
self.resampler.to(device="cuda", dtype=param_dtype)
419
# TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
420
self.lm_head = ParallelLMHead(config.vocab_size,
421
config.hidden_size,
422
quant_config=quant_config,
423
- prefix="llm.lm_head")
423
+ prefix=maybe_prefix(
424
+ prefix, "llm.lm_head"))
425
self.logits_processor = LogitsProcessor(config.vocab_size)
426
self.sampler = get_sampler()
427
@@ -661,9 +662,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
662
663
def init_llm(
664
self,
664
- config: PretrainedConfig,
665
- cache_config: Optional[CacheConfig] = None,
666
- quant_config: Optional[QuantizationConfig] = None,
665
+ vllm_config: VllmConfig,
666
prefix: str = "",
667
) -> nn.Module:
668
raise NotImplementedError
@@ -711,16 +710,10 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
710
711
def init_llm(
712
self,
714
- config: PretrainedConfig,
715
- cache_config: Optional[CacheConfig] = None,
716
- quant_config: Optional[QuantizationConfig] = None,
713
+ vllm_config: VllmConfig,
714
prefix: str = "",
715
) -> nn.Module:
719
-
720
- return LLMWrapper(MiniCPMModel(config,
721
- cache_config=cache_config,
722
- quant_config=quant_config,
723
- prefix=prefix),
716
+ return LLMWrapper(MiniCPMModel(vllm_config=vllm_config, prefix=prefix),
717
name="model")
718
719
def init_vision_module(
@@ -875,15 +868,10 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
868
869
def init_llm(
870
self,
878
- config: PretrainedConfig,
879
- cache_config: Optional[CacheConfig] = None,
880
- quant_config: Optional[QuantizationConfig] = None,
871
+ vllm_config: VllmConfig,
872
prefix: str = "",
873
) -> nn.Module:
883
- return LLMWrapper(LlamaModel(config,
884
- cache_config=cache_config,
885
- quant_config=quant_config,
886
- prefix=prefix),
874
+ return LLMWrapper(LlamaModel(vllm_config=vllm_config, prefix=prefix),
875
name="model")
876
877
def init_vision_module(
@@ -1022,16 +1010,10 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1010
1011
def init_llm(
1012
self,
1025
- config: PretrainedConfig,
1026
- cache_config: Optional[CacheConfig] = None,
1027
- quant_config: Optional[QuantizationConfig] = None,
1013
+ vllm_config: VllmConfig,
1014
prefix: str = "",
1015
) -> nn.Module:
1030
-
1031
- return LLMWrapper(Qwen2Model(config,
1032
- cache_config=cache_config,
1033
- quant_config=quant_config,
1034
- prefix=prefix),
1016
+ return LLMWrapper(Qwen2Model(vllm_config=vllm_config, prefix=prefix),
1017
name="model")
1018
1019
def init_vision_module(
@@ -1151,4 +1133,4 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1133
if instance_class is None:
1134
raise ValueError(
1135
"Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1154
- return instance_class(vllm_config, prefix=prefix)
1136
+ return instance_class(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/mixtral.py
+13
-21
@@ -28,7 +28,7 @@ from transformers import MixtralConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -48,7 +48,8 @@ from vllm.sequence import IntermediateTensors
48
49
from .interfaces import SupportsLoRA, SupportsPP
50
from .utils import (is_pp_missing_parameter,
51
- make_empty_intermediate_tensors_factory, make_layers)
51
+ make_empty_intermediate_tensors_factory, make_layers,
52
+ maybe_prefix)
53
54
55
class MixtralMoE(nn.Module):
@@ -248,15 +249,14 @@ class MixtralDecoderLayer(nn.Module):
249
@support_torch_compile
250
class MixtralModel(nn.Module):
251
251
- def __init__(
252
- self,
253
- config: MixtralConfig,
254
- cache_config: Optional[CacheConfig] = None,
255
- quant_config: Optional[QuantizationConfig] = None,
256
- lora_config: Optional[LoRAConfig] = None,
257
- prefix: str = "",
258
- ) -> None:
252
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
253
super().__init__()
254
+
255
+ config = vllm_config.model_config.hf_config
256
+ cache_config = vllm_config.cache_config
257
+ quant_config = vllm_config.quant_config
258
+ lora_config = vllm_config.lora_config
259
+
260
self.padding_idx = config.pad_token_id
261
lora_vocab = (lora_config.lora_extra_vocab_size *
262
(lora_config.max_loras or 1)) if lora_config else 0
@@ -332,24 +332,16 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
}
333
embedding_padding_modules = ["lm_head"]
334
335
- def __init__(
336
- self,
337
- vllm_config: VllmConfig,
338
- prefix: str = "",
339
- ) -> None:
335
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
336
super().__init__()
337
config = vllm_config.model_config.hf_config
342
- cache_config = vllm_config.cache_config
338
quant_config = vllm_config.quant_config
339
lora_config = vllm_config.lora_config
340
self.config = config
341
self.lora_config = lora_config
342
348
- self.model = MixtralModel(config,
349
- cache_config,
350
- quant_config,
351
- lora_config=lora_config,
352
- prefix="model")
343
+ self.model = MixtralModel(vllm_config=vllm_config,
344
+ prefix=maybe_prefix(prefix, "model"))
345
self.unpadded_vocab_size = config.vocab_size
346
if lora_config:
347
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/mixtral_quant.py
+11
-15
@@ -49,7 +49,8 @@ from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsPP
51
from .utils import (is_pp_missing_parameter,
52
- make_empty_intermediate_tensors_factory, make_layers)
52
+ make_empty_intermediate_tensors_factory, make_layers,
53
+ maybe_prefix)
54
55
56
class MixtralMLP(nn.Module):
@@ -293,14 +294,13 @@ class MixtralDecoderLayer(nn.Module):
294
295
class MixtralModel(nn.Module):
296
296
- def __init__(
297
- self,
298
- config: MixtralConfig,
299
- cache_config: Optional[CacheConfig] = None,
300
- quant_config: Optional[QuantizationConfig] = None,
301
- prefix: str = "",
302
- ) -> None:
297
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
298
super().__init__()
299
+
300
+ config = vllm_config.model_config.hf_config
301
+ cache_config = vllm_config.cache_config
302
+ quant_config = vllm_config.quant_config
303
+
304
self.padding_idx = config.pad_token_id
305
self.vocab_size = config.vocab_size
306
@@ -350,18 +350,14 @@ class MixtralModel(nn.Module):
350
class MixtralForCausalLM(nn.Module, SupportsPP):
351
fall_back_to_pt_during_load = False
352
353
- def __init__(
354
- self,
355
- vllm_config: VllmConfig,
356
- prefix: str = "",
357
- ) -> None:
353
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
354
super().__init__()
355
config = vllm_config.model_config.hf_config
360
- cache_config = vllm_config.cache_config
356
quant_config = vllm_config.quant_config
357
self.config = config
358
self.quant_config = quant_config
364
- self.model = MixtralModel(config, cache_config, quant_config)
359
+ self.model = MixtralModel(vllm_config=vllm_config,
360
+ prefix=maybe_prefix(prefix, "model"))
361
self.lm_head = ParallelLMHead(config.vocab_size,
362
config.hidden_size,
363
quant_config=quant_config)
vllm/model_executor/models/mllama.py
+19
-30
@@ -33,7 +33,7 @@ from transformers.models.mllama.processing_mllama import (
33
import vllm.distributed.parallel_state as ps
34
from vllm.attention import Attention, AttentionMetadata, AttentionType
35
from vllm.attention.ops.paged_attn import PagedAttention
36
-from vllm.config import CacheConfig, VllmConfig
36
+from vllm.config import VllmConfig
37
from vllm.distributed import get_tensor_model_parallel_world_size
38
from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
39
InputContext, TokenInputs, token_inputs)
@@ -56,6 +56,7 @@ from vllm.utils import is_list_of
56
from .clip import CLIPMLP
57
from .interfaces import SupportsMultiModal
58
from .llama import LlamaDecoderLayer, LlamaMLP
59
+from .utils import maybe_prefix
60
61
logger = init_logger(__name__)
62
MLLAMA_IMAGE_TOKEN_ID = 128256
@@ -939,15 +940,13 @@ class MllamaTextModel(nn.Module):
940
config_class = config_mllama.MllamaTextConfig
941
base_model_prefix = "model"
942
942
- def __init__(
943
- self,
944
- config: config_mllama.MllamaTextConfig,
945
- cache_config: Optional[CacheConfig],
946
- quant_config: Optional[QuantizationConfig],
947
- prefix: str = "",
948
- ) -> None:
943
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
944
super().__init__()
945
946
+ config = vllm_config.model_config.hf_config.text_config
947
+ cache_config = vllm_config.cache_config
948
+ quant_config = vllm_config.quant_config
949
+
950
self.padding_idx = config.pad_token_id
951
self.vocab_size = config.vocab_size
952
self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
@@ -1029,18 +1028,14 @@ class MllamaForCausalLM(nn.Module):
1028
"MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
1029
]
1030
1032
- def __init__(
1033
- self,
1034
- config: config_mllama.MllamaTextConfig,
1035
- cache_config: Optional[CacheConfig],
1036
- quant_config: Optional[QuantizationConfig],
1037
- prefix: str = "",
1038
- ) -> None:
1031
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1032
super().__init__()
1033
+
1034
+ config = vllm_config.model_config.hf_config.text_config
1035
+ quant_config = vllm_config.quant_config
1036
+
1037
self.vocab_size = config.vocab_size
1041
- self.model = MllamaTextModel(config,
1042
- cache_config,
1043
- quant_config,
1038
+ self.model = MllamaTextModel(vllm_config=vllm_config,
1039
prefix=f"{prefix}.model")
1040
self.lm_head = ParallelLMHead(
1041
config.vocab_size,
@@ -1108,14 +1103,9 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1103
"up_proj": ("gate_up_proj", 1),
1104
}
1105
1111
- def __init__(
1112
- self,
1113
- vllm_config: VllmConfig,
1114
- prefix: str = "",
1115
- ) -> None:
1106
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1107
super().__init__()
1108
config = vllm_config.model_config.hf_config
1118
- cache_config = vllm_config.cache_config
1109
quant_config = vllm_config.quant_config
1110
self.vocab_size = config.text_config.vocab_size
1111
self.hidden_size = config.text_config.hidden_size
@@ -1127,12 +1117,11 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1117
1118
self.vision_model = MllamaVisionModel(config.vision_config,
1119
quant_config,
1130
- prefix="vision_model")
1120
+ prefix=maybe_prefix(
1121
+ prefix, "vision_model"))
1122
self.language_model = MllamaForCausalLM(
1132
- config.text_config,
1133
- cache_config=cache_config,
1134
- quant_config=quant_config,
1135
- prefix="language_model",
1123
+ vllm_config=vllm_config,
1124
+ prefix=maybe_prefix(prefix, "language_model"),
1125
)
1126
self.multi_modal_projector = ColumnParallelLinear(
1127
config.vision_config.vision_output_dim,
@@ -1140,7 +1129,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1129
bias=True,
1130
quant_config=quant_config,
1131
gather_output=True,
1143
- prefix="multi_modal_projector",
1132
+ prefix=maybe_prefix(prefix, "multi_modal_projector"),
1133
)
1134
self.logits_processor = LogitsProcessor(config.output_hidden_states,
1135
config.text_config.vocab_size)
vllm/model_executor/models/molmo.py
+11
-15
@@ -44,7 +44,8 @@ from vllm.transformers_utils.processor import get_processor
44
45
from .interfaces import SupportsMultiModal, SupportsPP
46
from .utils import (get_vit_attn_backend,
47
- make_empty_intermediate_tensors_factory, make_layers)
47
+ make_empty_intermediate_tensors_factory, make_layers,
48
+ maybe_prefix)
49
50
# TODO: hard-coded for now. Consider making it configurable.
51
VIT_LAYERS = [-2, -9]
@@ -716,14 +717,13 @@ class MolmoVisionBackbone(nn.Module):
717
@support_torch_compile
718
class MolmoModel(nn.Module):
719
719
- def __init__(
720
- self,
721
- config: PretrainedConfig,
722
- cache_config: Optional[CacheConfig] = None,
723
- quant_config: Optional[QuantizationConfig] = None,
724
- prefix: str = "",
725
- ) -> None:
720
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
721
super().__init__()
722
+
723
+ config = vllm_config.model_config.hf_config
724
+ cache_config = vllm_config.cache_config
725
+ quant_config = vllm_config.quant_config
726
+
727
self.config = config
728
729
self.embedding_size = config.embedding_size or config.vocab_size
@@ -1024,14 +1024,9 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1024
@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1025
class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1026
1027
- def __init__(
1028
- self,
1029
- vllm_config: VllmConfig,
1030
- prefix: str = "",
1031
- ) -> None:
1027
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1028
super().__init__()
1029
config = vllm_config.model_config.hf_config
1034
- cache_config = vllm_config.cache_config
1030
quant_config = vllm_config.quant_config
1031
multimodal_config = vllm_config.model_config.multimodal_config
1032
self.config = config
@@ -1040,7 +1035,8 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1035
vision_config = VisionBackboneConfig()
1036
self.vision_backbone = MolmoVisionBackbone(config, vision_config,
1037
quant_config)
1043
- self.model = MolmoModel(config, cache_config, quant_config)
1038
+ self.model = MolmoModel(vllm_config=vllm_config,
1039
+ prefix=maybe_prefix(prefix, "model"))
1040
1041
if self.config.weight_tying:
1042
self.lm_head = self.model.transformer.wte
vllm/model_executor/models/mpt.py
+11
-15
@@ -26,7 +26,8 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
26
27
from .interfaces import SupportsPP
28
from .utils import (is_pp_missing_parameter,
29
- make_empty_intermediate_tensors_factory, make_layers)
29
+ make_empty_intermediate_tensors_factory, make_layers,
30
+ maybe_prefix)
31
32
33
def _get_alibi_slopes(
@@ -207,14 +208,13 @@ class MPTBlock(nn.Module):
208
@support_torch_compile
209
class MPTModel(nn.Module):
210
210
- def __init__(
211
- self,
212
- config: MPTConfig,
213
- cache_config: Optional[CacheConfig] = None,
214
- quant_config: Optional[QuantizationConfig] = None,
215
- prefix: str = "",
216
- ):
211
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
212
super().__init__()
213
+
214
+ config = vllm_config.model_config.hf_config
215
+ cache_config = vllm_config.cache_config
216
+ quant_config = vllm_config.quant_config
217
+
218
assert config.embedding_fraction == 1.0
219
assert config.norm_type == "low_precision_layernorm"
220
@@ -267,20 +267,16 @@ class MPTModel(nn.Module):
267
268
class MPTForCausalLM(nn.Module, SupportsPP):
269
270
- def __init__(
271
- self,
272
- vllm_config: VllmConfig,
273
- prefix: str = "",
274
- ) -> None:
270
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271
super().__init__()
272
config = vllm_config.model_config.hf_config
277
- cache_config = vllm_config.cache_config
273
quant_config = vllm_config.quant_config
274
self.config = config
275
assert config.tie_word_embeddings
276
self.quant_config = quant_config
277
283
- self.transformer = MPTModel(config, cache_config, quant_config)
278
+ self.transformer = MPTModel(vllm_config=vllm_config,
279
+ prefix=maybe_prefix(prefix, "transformer"))
280
self.lm_head = self.transformer.wte
281
self.logits_processor = LogitsProcessor(config.vocab_size)
282
self.sampler = get_sampler()
vllm/model_executor/models/nemotron.py
+13
-21
@@ -27,7 +27,7 @@ from torch import nn
27
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.compilation.decorators import support_torch_compile
30
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
30
+from vllm.config import CacheConfig, VllmConfig
31
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -47,7 +47,8 @@ from vllm.transformers_utils.configs import NemotronConfig
47
48
from .interfaces import SupportsLoRA, SupportsPP
49
from .utils import (PPMissingLayer, is_pp_missing_parameter,
50
- make_empty_intermediate_tensors_factory, make_layers)
50
+ make_empty_intermediate_tensors_factory, make_layers,
51
+ maybe_prefix)
52
53
# The architecture is pretty similar to Llama, with these changes:
54
# - There is no gate_proj, just up_proj
@@ -293,15 +294,14 @@ class NemotronDecoderLayer(nn.Module):
294
@support_torch_compile
295
class NemotronModel(nn.Module):
296
296
- def __init__(
297
- self,
298
- config: NemotronConfig,
299
- cache_config: Optional[CacheConfig] = None,
300
- quant_config: Optional[QuantizationConfig] = None,
301
- lora_config: Optional[LoRAConfig] = None,
302
- prefix: str = "",
303
- ) -> None:
297
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
298
super().__init__()
299
+
300
+ config = vllm_config.model_config.hf_config
301
+ cache_config = vllm_config.cache_config
302
+ quant_config = vllm_config.quant_config
303
+ lora_config = vllm_config.lora_config
304
+
305
self.config = config
306
self.padding_idx = config.pad_token_id
307
lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -401,14 +401,9 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
401
"v_proj": ("qkv_proj", 2),
402
}
403
404
- def __init__(
405
- self,
406
- vllm_config: VllmConfig,
407
- prefix: str = "",
408
- ) -> None:
404
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
405
super().__init__()
406
config = vllm_config.model_config.hf_config
411
- cache_config = vllm_config.cache_config
407
quant_config = vllm_config.quant_config
408
lora_config = vllm_config.lora_config
409
assert isinstance(config, NemotronConfig)
@@ -416,11 +411,8 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411
self.config = config
412
self.lora_config = lora_config
413
419
- self.model = NemotronModel(config,
420
- cache_config,
421
- quant_config,
422
- lora_config=lora_config,
423
- prefix="model")
414
+ self.model = NemotronModel(vllm_config=vllm_config,
415
+ prefix=maybe_prefix(prefix, "model"))
416
if get_pp_group().is_last_rank:
417
self.unpadded_vocab_size = config.vocab_size
418
if lora_config:
vllm/model_executor/models/olmo.py
+11
-13
@@ -46,7 +46,8 @@ from vllm.sequence import IntermediateTensors
46
47
from .interfaces import SupportsPP
48
from .utils import (is_pp_missing_parameter,
49
- make_empty_intermediate_tensors_factory, make_layers)
49
+ make_empty_intermediate_tensors_factory, make_layers,
50
+ maybe_prefix)
51
52
53
class OlmoAttention(nn.Module):
@@ -224,12 +225,13 @@ class OlmoDecoderLayer(nn.Module):
225
@support_torch_compile
226
class OlmoModel(nn.Module):
227
227
- def __init__(self,
228
- config: OlmoConfig,
229
- cache_config: Optional[CacheConfig] = None,
230
- quant_config: Optional[QuantizationConfig] = None,
231
- prefix: str = ""):
228
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
229
super().__init__()
230
+
231
+ config = vllm_config.model_config.hf_config
232
+ cache_config = vllm_config.cache_config
233
+ quant_config = vllm_config.quant_config
234
+
235
self.config = config
236
237
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
@@ -291,17 +293,13 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
293
Extremely barebones HF model wrapper.
294
"""
295
294
- def __init__(
295
- self,
296
- vllm_config: VllmConfig,
297
- prefix: str = "",
298
- ) -> None:
296
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
297
super().__init__()
298
config = vllm_config.model_config.hf_config
301
- cache_config = vllm_config.cache_config
299
quant_config = vllm_config.quant_config
300
self.config = config
304
- self.model = OlmoModel(config, cache_config, quant_config)
301
+ self.model = OlmoModel(vllm_config=vllm_config,
302
+ prefix=maybe_prefix(prefix, "model"))
303
if config.tie_word_embeddings:
304
self.lm_head = self.model.embed_tokens
305
else:
vllm/model_executor/models/olmoe.py
+11
-15
@@ -38,7 +38,8 @@ from vllm.utils import print_warning_once
38
39
from .interfaces import SupportsPP
40
from .utils import (is_pp_missing_parameter,
41
- make_empty_intermediate_tensors_factory, make_layers)
41
+ make_empty_intermediate_tensors_factory, make_layers,
42
+ maybe_prefix)
43
44
45
class OlmoeMoE(nn.Module):
@@ -243,14 +244,13 @@ class OlmoeDecoderLayer(nn.Module):
244
@support_torch_compile
245
class OlmoeModel(nn.Module):
246
246
- def __init__(
247
- self,
248
- config: PretrainedConfig,
249
- cache_config: Optional[CacheConfig] = None,
250
- quant_config: Optional[QuantizationConfig] = None,
251
- prefix: str = "",
252
- ) -> None:
247
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
248
super().__init__()
249
+
250
+ config = vllm_config.model_config.hf_config
251
+ cache_config = vllm_config.cache_config
252
+ quant_config = vllm_config.quant_config
253
+
254
self.padding_idx = config.pad_token_id
255
self.vocab_size = config.vocab_size
256
@@ -309,18 +309,14 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
309
310
fall_back_to_pt_during_load = False
311
312
- def __init__(
313
- self,
314
- vllm_config: VllmConfig,
315
- prefix: str = "",
316
- ) -> None:
312
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
313
super().__init__()
314
config = vllm_config.model_config.hf_config
319
- cache_config = vllm_config.cache_config
315
quant_config = vllm_config.quant_config
316
self.config = config
317
self.quant_config = quant_config
323
- self.model = OlmoeModel(config, cache_config, quant_config)
318
+ self.model = OlmoeModel(vllm_config=vllm_config,
319
+ prefix=maybe_prefix(prefix, "model"))
320
self.lm_head = ParallelLMHead(config.vocab_size,
321
config.hidden_size,
322
quant_config=quant_config)
vllm/model_executor/models/opt.py
+8
-16
@@ -293,14 +293,13 @@ class OPTDecoder(nn.Module):
293
@support_torch_compile
294
class OPTModel(nn.Module):
295
296
- def __init__(
297
- self,
298
- config: OPTConfig,
299
- cache_config: Optional[CacheConfig] = None,
300
- quant_config: Optional[QuantizationConfig] = None,
301
- prefix: str = "",
302
- ):
296
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
297
super().__init__()
298
+
299
+ config = vllm_config.model_config.hf_config
300
+ cache_config = vllm_config.cache_config
301
+ quant_config = vllm_config.quant_config
302
+
303
self.decoder = OPTDecoder(config,
304
cache_config,
305
quant_config,
@@ -342,21 +341,14 @@ class OPTForCausalLM(nn.Module, SupportsPP):
341
".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
342
]
343
345
- def __init__(
346
- self,
347
- vllm_config: VllmConfig,
348
- prefix: str = "",
349
- ) -> None:
344
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
345
super().__init__()
346
config = vllm_config.model_config.hf_config
352
- cache_config = vllm_config.cache_config
347
quant_config = vllm_config.quant_config
348
super().__init__()
349
self.config = config
350
self.quant_config = quant_config
357
- self.model = OPTModel(config,
358
- cache_config,
359
- quant_config,
351
+ self.model = OPTModel(vllm_config=vllm_config,
352
prefix=maybe_prefix(prefix, "model"))
353
if self.config.tie_word_embeddings:
354
self.lm_head = self.model.decoder.embed_tokens
vllm/model_executor/models/orion.py
+11
-15
@@ -29,7 +29,8 @@ from vllm.sequence import IntermediateTensors
29
30
from .interfaces import SupportsPP
31
from .utils import (is_pp_missing_parameter,
32
- make_empty_intermediate_tensors_factory, make_layers)
32
+ make_empty_intermediate_tensors_factory, make_layers,
33
+ maybe_prefix)
34
35
36
class OrionMLP(nn.Module):
@@ -208,14 +209,13 @@ class OrionDecoderLayer(nn.Module):
209
@support_torch_compile
210
class OrionModel(nn.Module):
211
211
- def __init__(
212
- self,
213
- config: PretrainedConfig,
214
- cache_config: Optional[CacheConfig] = None,
215
- quant_config: Optional[QuantizationConfig] = None,
216
- prefix: str = "",
217
- ) -> None:
212
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
213
super().__init__()
214
+
215
+ config = vllm_config.model_config.hf_config
216
+ cache_config = vllm_config.cache_config
217
+ quant_config = vllm_config.quant_config
218
+
219
self.config = config
220
self.padding_idx = config.pad_token_id
221
self.vocab_size = config.vocab_size
@@ -268,18 +268,14 @@ class OrionModel(nn.Module):
268
269
class OrionForCausalLM(nn.Module, SupportsPP):
270
271
- def __init__(
272
- self,
273
- vllm_config: VllmConfig,
274
- prefix: str = "",
275
- ) -> None:
271
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
272
super().__init__()
273
config = vllm_config.model_config.hf_config
278
- cache_config = vllm_config.cache_config
274
quant_config = vllm_config.quant_config
275
self.config = config
276
self.quant_config = quant_config
282
- self.model = OrionModel(config, cache_config, quant_config)
277
+ self.model = OrionModel(vllm_config=vllm_config,
278
+ prefix=maybe_prefix(prefix, "model"))
279
self.lm_head = ParallelLMHead(config.vocab_size,
280
config.hidden_size,
281
quant_config=quant_config)
vllm/model_executor/models/paligemma.py
+5
-8
@@ -20,7 +20,7 @@ from .interfaces import SupportsMultiModal, SupportsPP
20
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
21
dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
22
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
23
- merge_multimodal_embeddings)
23
+ maybe_prefix, merge_multimodal_embeddings)
24
25
logger = init_logger(__name__)
26
@@ -131,11 +131,7 @@ class PaliGemmaMultiModalProjector(nn.Module):
131
class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
132
SupportsPP):
133
134
- def __init__(
135
- self,
136
- vllm_config: VllmConfig,
137
- prefix: str = "",
138
- ) -> None:
134
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
135
super().__init__()
136
config = vllm_config.model_config.hf_config
137
quant_config = vllm_config.quant_config
@@ -145,7 +141,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
141
142
self.vision_tower = SiglipVisionModel(config.vision_config,
143
quant_config,
148
- prefix="vision_tower")
144
+ prefix=maybe_prefix(
145
+ prefix, "vision_tower"))
146
self.multi_modal_projector = PaliGemmaMultiModalProjector(
147
vision_hidden_size=config.vision_config.hidden_size,
148
projection_dim=config.vision_config.projection_dim)
@@ -155,7 +152,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
152
self.language_model = init_vllm_registered_model(
153
config.text_config,
154
vllm_config=vllm_config,
158
- prefix="language_model")
155
+ prefix=maybe_prefix(prefix, "language_model"))
156
logit_scale = getattr(config, "logit_scale", 1.0)
157
self.language_model.logits_processor.scale *= logit_scale
158
vllm/model_executor/models/persimmon.py
+11
-16
@@ -45,7 +45,8 @@ from vllm.sequence import IntermediateTensors
45
46
from .interfaces import SupportsPP
47
from .utils import (is_pp_missing_parameter,
48
- make_empty_intermediate_tensors_factory, make_layers)
48
+ make_empty_intermediate_tensors_factory, make_layers,
49
+ maybe_prefix)
50
51
52
class PersimmonMLP(nn.Module):
@@ -212,12 +213,13 @@ class PersimmonDecoderLayer(nn.Module):
213
@support_torch_compile
214
class PersimmonModel(nn.Module):
215
215
- def __init__(self,
216
- config: PersimmonConfig,
217
- cache_config: Optional[CacheConfig] = None,
218
- quant_config: Optional[QuantizationConfig] = None,
219
- prefix: str = ""):
216
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
217
super().__init__()
218
+
219
+ config = vllm_config.model_config.hf_config
220
+ cache_config = vllm_config.cache_config
221
+ quant_config = vllm_config.quant_config
222
+
223
self.vocab_size = config.vocab_size
224
225
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
@@ -265,20 +267,13 @@ class PersimmonModel(nn.Module):
267
268
class PersimmonForCausalLM(nn.Module, SupportsPP):
269
268
- def __init__(
269
- self,
270
- vllm_config: VllmConfig,
271
- prefix: str = "",
272
- ) -> None:
270
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271
super().__init__()
272
config = vllm_config.model_config.hf_config
275
- cache_config = vllm_config.cache_config
276
- quant_config = vllm_config.quant_config
273
self.config = config
274
self.vocab_size = config.vocab_size
279
- self.model = PersimmonModel(config,
280
- cache_config=cache_config,
281
- quant_config=quant_config)
275
+ self.model = PersimmonModel(vllm_config=vllm_config,
276
+ prefix=maybe_prefix(prefix, "model"))
277
self.lm_head = ParallelLMHead(config.vocab_size,
278
config.hidden_size,
279
bias=False)
vllm/model_executor/models/phi.py
+11
-13
@@ -60,7 +60,8 @@ from vllm.sequence import IntermediateTensors
60
61
from .interfaces import SupportsLoRA, SupportsPP
62
from .utils import (is_pp_missing_parameter,
63
- make_empty_intermediate_tensors_factory, make_layers)
63
+ make_empty_intermediate_tensors_factory, make_layers,
64
+ maybe_prefix)
65
66
67
class PhiAttention(nn.Module):
@@ -196,12 +197,13 @@ class PhiLayer(nn.Module):
197
@support_torch_compile
198
class PhiModel(nn.Module):
199
199
- def __init__(self,
200
- config: PhiConfig,
201
- cache_config: Optional[CacheConfig] = None,
202
- quant_config: Optional[QuantizationConfig] = None,
203
- prefix: str = ""):
200
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
201
super().__init__()
202
+
203
+ config = vllm_config.model_config.hf_config
204
+ cache_config = vllm_config.cache_config
205
+ quant_config = vllm_config.quant_config
206
+
207
self.config = config
208
self.quant_config = quant_config
209
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
@@ -277,14 +279,9 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
279
embedding_modules = {}
280
embedding_padding_modules = []
281
280
- def __init__(
281
- self,
282
- vllm_config: VllmConfig,
283
- prefix: str = "",
284
- ) -> None:
282
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
283
super().__init__()
284
config = vllm_config.model_config.hf_config
287
- cache_config = vllm_config.cache_config
285
quant_config = vllm_config.quant_config
286
lora_config = vllm_config.lora_config
287
self.config = config
@@ -294,7 +291,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
291
292
self.quant_config = quant_config
293
297
- self.model = PhiModel(config, cache_config, quant_config)
294
+ self.model = PhiModel(vllm_config=vllm_config,
295
+ prefix=maybe_prefix(prefix, "model"))
296
297
self.lm_head = ParallelLMHead(config.vocab_size,
298
config.hidden_size,
vllm/model_executor/models/phi3_small.py
+11
-15
@@ -24,7 +24,8 @@ from vllm.sequence import IntermediateTensors
24
25
from .interfaces import SupportsPP
26
from .utils import (is_pp_missing_parameter,
27
- make_empty_intermediate_tensors_factory, make_layers)
27
+ make_empty_intermediate_tensors_factory, make_layers,
28
+ maybe_prefix)
29
30
31
def load_column_parallel_weight(param: torch.nn.Parameter,
@@ -299,14 +300,13 @@ class Phi3SmallDecoderLayer(nn.Module):
300
301
class Phi3SmallModel(nn.Module):
302
302
- def __init__(
303
- self,
304
- config: PretrainedConfig,
305
- cache_config: Optional[CacheConfig] = None,
306
- quant_config: Optional[QuantizationConfig] = None,
307
- prefix: str = "",
308
- ):
303
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
304
super().__init__()
305
+
306
+ config = vllm_config.model_config.hf_config
307
+ cache_config = vllm_config.cache_config
308
+ quant_config = vllm_config.quant_config
309
+
310
self.config = config
311
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
312
config.hidden_size)
@@ -363,18 +363,14 @@ class Phi3SmallModel(nn.Module):
363
class Phi3SmallForCausalLM(nn.Module, SupportsPP):
364
_tied_weights_keys = ["lm_head.weight"]
365
366
- def __init__(
367
- self,
368
- vllm_config: VllmConfig,
369
- prefix: str = "",
370
- ) -> None:
366
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
367
super().__init__()
368
config = vllm_config.model_config.hf_config
373
- cache_config = vllm_config.cache_config
369
quant_config = vllm_config.quant_config
370
self.config = config
371
self.quant_config = quant_config
377
- self.model = Phi3SmallModel(config, cache_config, quant_config)
372
+ self.model = Phi3SmallModel(vllm_config=vllm_config,
373
+ prefix=maybe_prefix(prefix, "model"))
374
self.vocab_size = config.vocab_size
375
self.mup_width_multiplier = config.mup_width_multiplier
376
self.lm_head = ParallelLMHead(
vllm/model_executor/models/phi3v.py
+6
-8
@@ -45,7 +45,7 @@ from vllm.utils import is_list_of
45
46
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
47
from .interfaces import SupportsMultiModal, SupportsPP
48
-from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
48
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,
49
merge_multimodal_embeddings)
50
51
logger = init_logger(__name__)
@@ -525,11 +525,7 @@ def input_processor_for_phi3v(ctx: InputContext,
525
@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
526
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
527
528
- def __init__(
529
- self,
530
- vllm_config: VllmConfig,
531
- prefix: str = "",
532
- ) -> None:
528
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
529
super().__init__()
530
config = vllm_config.model_config.hf_config
531
quant_config = vllm_config.quant_config
@@ -544,12 +540,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
540
config.hidden_size,
541
org_num_embeddings=config.vocab_size,
542
quant_config=quant_config,
547
- prefix="model.embed_tokens",
543
+ prefix=maybe_prefix(prefix, "model.embed_tokens"),
544
)
545
546
# TODO: Optionally initializes this for supporting input embeddings.
547
self.vision_embed_tokens = Phi3HDImageEmbedding(
552
- config, quant_config, prefix="model.vision_embed_tokens")
548
+ config,
549
+ quant_config,
550
+ prefix=maybe_prefix(prefix, "model.vision_embed_tokens"))
551
552
# The prefix is empty intentionally because default prefix of
553
# LlamaForCausalLM is "model"
vllm/model_executor/models/phimoe.py
+13
-21
@@ -28,7 +28,7 @@ from transformers.configuration_utils import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -48,7 +48,8 @@ from vllm.sequence import IntermediateTensors
48
49
from .interfaces import SupportsLoRA, SupportsPP
50
from .utils import (is_pp_missing_parameter,
51
- make_empty_intermediate_tensors_factory, make_layers)
51
+ make_empty_intermediate_tensors_factory, make_layers,
52
+ maybe_prefix)
53
54
55
class PhiMoEConfig(PretrainedConfig):
@@ -432,15 +433,14 @@ class PhiMoEDecoderLayer(nn.Module):
433
@support_torch_compile
434
class PhiMoEModel(nn.Module):
435
435
- def __init__(
436
- self,
437
- config: PhiMoEConfig,
438
- cache_config: Optional[CacheConfig] = None,
439
- quant_config: Optional[QuantizationConfig] = None,
440
- lora_config: Optional[LoRAConfig] = None,
441
- prefix: str = "",
442
- ) -> None:
436
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
437
super().__init__()
438
+
439
+ config = vllm_config.model_config.hf_config
440
+ cache_config = vllm_config.cache_config
441
+ quant_config = vllm_config.quant_config
442
+ lora_config = vllm_config.lora_config
443
+
444
self.padding_idx = config.pad_token_id
445
lora_vocab = ((lora_config.lora_extra_vocab_size *
446
(lora_config.max_loras or 1)) if lora_config else 0)
@@ -529,23 +529,15 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
529
}
530
embedding_padding_modules = ["lm_head"]
531
532
- def __init__(
533
- self,
534
- vllm_config: VllmConfig,
535
- prefix: str = "",
536
- ) -> None:
532
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
533
super().__init__()
534
config = vllm_config.model_config.hf_config
539
- cache_config = vllm_config.cache_config
540
- quant_config = vllm_config.quant_config
535
lora_config = vllm_config.lora_config
536
self.config = config
537
self.lora_config = lora_config
538
545
- self.model = PhiMoEModel(config,
546
- cache_config,
547
- quant_config,
548
- lora_config=lora_config)
539
+ self.model = PhiMoEModel(vllm_config=vllm_config,
540
+ prefix=maybe_prefix(prefix, "model"))
541
self.unpadded_vocab_size = config.vocab_size
542
if lora_config:
543
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/pixtral.py
+3
-7
@@ -38,7 +38,7 @@ from vllm.transformers_utils.processor import cached_get_processor
38
from vllm.utils import is_list_of
39
40
from .interfaces import SupportsMultiModal, SupportsPP
41
-from .utils import init_vllm_registered_model
41
+from .utils import init_vllm_registered_model, maybe_prefix
42
43
try:
44
from xformers import ops as xops
@@ -152,11 +152,7 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
152
class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
153
SupportsPP):
154
155
- def __init__(
156
- self,
157
- vllm_config: VllmConfig,
158
- prefix: str = "",
159
- ) -> None:
155
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
156
super().__init__()
157
config = vllm_config.model_config.hf_config
158
multimodal_config = vllm_config.model_config.multimodal_config
@@ -176,7 +172,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
172
self.language_model = init_vllm_registered_model(
173
config.text_config,
174
vllm_config=vllm_config,
179
- prefix="language_model")
175
+ prefix=maybe_prefix(prefix, "language_model"))
176
177
self.vision_encoder = VisionTransformer(self.vision_args)
178
self.vision_language_adapter = VisionLanguageAdapter(
vllm/model_executor/models/qwen.py
+12
-15
@@ -50,7 +50,8 @@ from vllm.utils import is_list_of
50
51
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
52
from .utils import (flatten_bn, is_pp_missing_parameter,
53
- make_empty_intermediate_tensors_factory, make_layers)
53
+ make_empty_intermediate_tensors_factory, make_layers,
54
+ maybe_prefix)
55
56
logger = init_logger(__name__)
57
@@ -552,14 +553,13 @@ class QWenBlock(nn.Module):
553
@support_torch_compile
554
class QWenModel(nn.Module):
555
555
- def __init__(
556
- self,
557
- config: PretrainedConfig,
558
- cache_config: Optional[CacheConfig] = None,
559
- quant_config: Optional[QuantizationConfig] = None,
560
- prefix: str = "",
561
- ):
556
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
557
super().__init__()
558
+
559
+ config = vllm_config.model_config.hf_config
560
+ cache_config = vllm_config.cache_config
561
+ quant_config = vllm_config.quant_config
562
+
563
self.config = config
564
self.vocab_size = config.vocab_size
565
@@ -865,20 +865,17 @@ def dummy_data_for_qwen(
865
866
class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
867
868
- def __init__(
869
- self,
870
- vllm_config: VllmConfig,
871
- prefix: str = "",
872
- ) -> None:
868
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
869
super().__init__()
870
config = vllm_config.model_config.hf_config
875
- cache_config = vllm_config.cache_config
871
quant_config = vllm_config.quant_config
872
multimodal_config = vllm_config.model_config.multimodal_config
873
self.config = config
874
self.multimodal_config = multimodal_config
875
self.quant_config = quant_config
881
- self.transformer = QWenModel(config, cache_config, quant_config)
876
+ self.transformer = QWenModel(vllm_config=vllm_config,
877
+ prefix=maybe_prefix(
878
+ prefix, "transformer"))
879
self.lm_head = ParallelLMHead(config.vocab_size,
880
config.hidden_size,
881
quant_config=quant_config)
vllm/model_executor/models/qwen2.py
+8
-15
@@ -240,14 +240,13 @@ class Qwen2DecoderLayer(nn.Module):
240
@support_torch_compile
241
class Qwen2Model(nn.Module):
242
243
- def __init__(
244
- self,
245
- config: Qwen2Config,
246
- cache_config: Optional[CacheConfig] = None,
247
- quant_config: Optional[QuantizationConfig] = None,
248
- prefix: str = "",
249
- ) -> None:
243
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
244
super().__init__()
245
+
246
+ config = vllm_config.model_config.hf_config
247
+ cache_config = vllm_config.cache_config
248
+ quant_config = vllm_config.quant_config
249
+
250
self.config = config
251
self.padding_idx = config.pad_token_id
252
self.vocab_size = config.vocab_size
@@ -403,11 +402,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
402
"up_proj": ("gate_up_proj", 1),
403
}
404
406
- def __init__(
407
- self,
408
- vllm_config: VllmConfig,
409
- prefix: str = "",
410
- ) -> None:
405
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
406
super().__init__()
407
config = vllm_config.model_config.hf_config
408
cache_config = vllm_config.cache_config
@@ -429,9 +424,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
424
self.lora_config = lora_config
425
426
self.quant_config = quant_config
432
- self.model = Qwen2Model(config,
433
- cache_config,
434
- quant_config,
427
+ self.model = Qwen2Model(vllm_config=vllm_config,
428
prefix=maybe_prefix(prefix, "model"))
429
430
if config.tie_word_embeddings:
vllm/model_executor/models/qwen2_audio.py
+4
-8
@@ -264,14 +264,9 @@ def input_mapper_for_qwen2_audio(
264
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
265
SupportsPP):
266
267
- def __init__(
268
- self,
269
- vllm_config: VllmConfig,
270
- prefix: str = "",
271
- ) -> None:
267
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
268
super().__init__()
269
config = vllm_config.model_config.hf_config
274
- cache_config = vllm_config.cache_config
270
quant_config = vllm_config.quant_config
271
multimodal_config = vllm_config.model_config.multimodal_config
272
self.config = config
@@ -283,8 +278,9 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
278
279
self.quant_config = quant_config
280
286
- self.language_model = Qwen2Model(config.text_config, cache_config,
287
- quant_config)
281
+ self.language_model = Qwen2Model(
282
+ vllm_config=vllm_config.with_hf_config(config.text_config),
283
+ prefix=prefix)
284
self.unpadded_vocab_size = config.text_config.vocab_size
285
if config.text_config.tie_word_embeddings:
286
self.lm_head = self.language_model.embed_tokens
vllm/model_executor/models/qwen2_cls.py
+4
-7
@@ -17,7 +17,7 @@ from vllm.model_executor.models.qwen2 import Qwen2Model
17
from vllm.model_executor.pooling_metadata import PoolingMetadata
18
from vllm.sequence import IntermediateTensors, PoolerOutput
19
20
-from .utils import AutoWeightsLoader
20
+from .utils import AutoWeightsLoader, maybe_prefix
21
22
23
class Qwen2ForSequenceClassification(nn.Module):
@@ -43,11 +43,7 @@ class Qwen2ForSequenceClassification(nn.Module):
43
embedding_modules = {}
44
embedding_padding_modules = []
45
46
- def __init__(
47
- self,
48
- vllm_config: VllmConfig,
49
- prefix: str = "",
50
- ) -> None:
46
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
47
super().__init__()
48
config = vllm_config.model_config.hf_config
49
cache_config = vllm_config.cache_config
@@ -70,7 +66,8 @@ class Qwen2ForSequenceClassification(nn.Module):
66
self.lora_config = lora_config
67
68
self.quant_config = quant_config
73
- self.model = Qwen2Model(config, cache_config, quant_config)
69
+ self.model = Qwen2Model(vllm_config=vllm_config,
70
+ prefix=maybe_prefix(prefix, "model"))
71
72
self.score = RowParallelLinear(config.hidden_size,
73
config.num_labels,
vllm/model_executor/models/qwen2_moe.py
+11
-15
@@ -54,7 +54,8 @@ from vllm.utils import print_warning_once
54
55
from .interfaces import SupportsPP
56
from .utils import (is_pp_missing_parameter,
57
- make_empty_intermediate_tensors_factory, make_layers)
57
+ make_empty_intermediate_tensors_factory, make_layers,
58
+ maybe_prefix)
59
60
61
class Qwen2MoeMLP(nn.Module):
@@ -315,14 +316,13 @@ class Qwen2MoeDecoderLayer(nn.Module):
316
@support_torch_compile
317
class Qwen2MoeModel(nn.Module):
318
318
- def __init__(
319
- self,
320
- config: PretrainedConfig,
321
- cache_config: Optional[CacheConfig] = None,
322
- quant_config: Optional[QuantizationConfig] = None,
323
- prefix: str = "",
324
- ) -> None:
319
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
320
super().__init__()
321
+
322
+ config = vllm_config.model_config.hf_config
323
+ cache_config = vllm_config.cache_config
324
+ quant_config = vllm_config.quant_config
325
+
326
self.padding_idx = config.pad_token_id
327
self.vocab_size = config.vocab_size
328
@@ -377,18 +377,14 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
377
378
fall_back_to_pt_during_load = False
379
380
- def __init__(
381
- self,
382
- vllm_config: VllmConfig,
383
- prefix: str = "",
384
- ) -> None:
380
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
381
super().__init__()
382
config = vllm_config.model_config.hf_config
387
- cache_config = vllm_config.cache_config
383
quant_config = vllm_config.quant_config
384
self.config = config
385
self.quant_config = quant_config
391
- self.model = Qwen2MoeModel(config, cache_config, quant_config)
386
+ self.model = Qwen2MoeModel(vllm_config=vllm_config,
387
+ prefix=maybe_prefix(prefix, "model"))
388
self.lm_head = ParallelLMHead(config.vocab_size,
389
config.hidden_size,
390
quant_config=quant_config)
vllm/model_executor/models/qwen2_rm.py
+4
-7
@@ -18,7 +18,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
18
19
from .interfaces import SupportsPP
20
from .qwen2 import Qwen2Model
21
-from .utils import AutoWeightsLoader
21
+from .utils import AutoWeightsLoader, maybe_prefix
22
23
24
class ReLU(nn.Module):
@@ -55,11 +55,7 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
55
embedding_modules = {}
56
embedding_padding_modules = []
57
58
- def __init__(
59
- self,
60
- vllm_config: VllmConfig,
61
- prefix: str = "",
62
- ) -> None:
58
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
59
super().__init__()
60
config = vllm_config.model_config.hf_config
61
cache_config = vllm_config.cache_config
@@ -82,7 +78,8 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
78
self.lora_config = lora_config
79
80
self.quant_config = quant_config
85
- self.model = Qwen2Model(config, cache_config, quant_config)
81
+ self.model = Qwen2Model(vllm_config=vllm_config,
82
+ prefix=maybe_prefix(prefix, "model"))
83
84
self.score = nn.Sequential(
85
ColumnParallelLinear(config.hidden_size,
vllm/model_executor/models/qwen2_vl.py
+7
-12
@@ -70,7 +70,7 @@ from vllm.transformers_utils.processor import cached_get_processor
70
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
71
from .utils import (PPMissingLayer, get_vit_attn_backend,
72
is_pp_missing_parameter,
73
- make_empty_intermediate_tensors_factory)
73
+ make_empty_intermediate_tensors_factory, maybe_prefix)
74
75
logger = init_logger(__name__)
76
@@ -966,11 +966,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
966
embedding_modules = {}
967
embedding_padding_modules = []
968
969
- def __init__(
970
- self,
971
- vllm_config: VllmConfig,
972
- prefix: str = "",
973
- ) -> None:
969
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
970
super().__init__()
971
config = vllm_config.model_config.hf_config
972
cache_config = vllm_config.cache_config
@@ -986,13 +982,11 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
982
config.vision_config,
983
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
984
quant_config=self._maybe_ignore_quant_config(quant_config),
989
- prefix="visual",
985
+ prefix=maybe_prefix(prefix, "visual"),
986
)
987
992
- self.model = Qwen2Model(config,
993
- cache_config,
994
- quant_config,
995
- prefix="model")
988
+ self.model = Qwen2Model(vllm_config=vllm_config,
989
+ prefix=maybe_prefix(prefix, "model"))
990
991
if get_pp_group().is_last_rank:
992
if config.tie_word_embeddings:
@@ -1001,7 +995,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
995
self.lm_head = ParallelLMHead(config.vocab_size,
996
config.hidden_size,
997
quant_config=quant_config,
1004
- prefix="lm_head")
998
+ prefix=maybe_prefix(
999
+ prefix, "lm_head"))
1000
else:
1001
self.lm_head = PPMissingLayer()
1002
vllm/model_executor/models/solar.py
+13
-21
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
+from vllm.config import CacheConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -53,7 +53,8 @@ from vllm.sequence import IntermediateTensors
53
54
from .interfaces import SupportsLoRA, SupportsPP
55
from .utils import (PPMissingLayer, is_pp_missing_parameter,
56
- make_empty_intermediate_tensors_factory, make_layers)
56
+ make_empty_intermediate_tensors_factory, make_layers,
57
+ maybe_prefix)
58
59
60
class SolarMLP(nn.Module):
@@ -266,15 +267,14 @@ class SolarDecoderLayer(nn.Module):
267
@support_torch_compile
268
class SolarModel(nn.Module):
269
269
- def __init__(
270
- self,
271
- config: PretrainedConfig,
272
- cache_config: Optional[CacheConfig] = None,
273
- quant_config: Optional[QuantizationConfig] = None,
274
- lora_config: Optional[LoRAConfig] = None,
275
- prefix: str = "",
276
- ) -> None:
270
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271
super().__init__()
272
+
273
+ config = vllm_config.model_config.hf_config
274
+ cache_config = vllm_config.cache_config
275
+ quant_config = vllm_config.quant_config
276
+ lora_config = vllm_config.lora_config
277
+
278
self.config = config
279
self.padding_idx = config.pad_token_id
280
lora_vocab = ((lora_config.lora_extra_vocab_size *
@@ -409,25 +409,17 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
409
"up_proj": ("gate_up_proj", 1),
410
}
411
412
- def __init__(
413
- self,
414
- vllm_config: VllmConfig,
415
- prefix: str = "",
416
- ) -> None:
412
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
413
super().__init__()
414
config = vllm_config.model_config.hf_config
419
- cache_config = vllm_config.cache_config
415
quant_config = vllm_config.quant_config
416
lora_config = vllm_config.lora_config
417
self.config = config
418
self.lora_config = lora_config
419
420
self.model = SolarModel(
426
- config,
427
- cache_config,
428
- quant_config,
429
- lora_config=lora_config,
430
- prefix="model",
421
+ vllm_config=vllm_config,
422
+ prefix=maybe_prefix(prefix, "model"),
423
)
424
if get_pp_group().is_last_rank:
425
self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/stablelm.py
+11
-13
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44
from .interfaces import SupportsPP
45
from .utils import (is_pp_missing_parameter,
46
- make_empty_intermediate_tensors_factory, make_layers)
46
+ make_empty_intermediate_tensors_factory, make_layers,
47
+ maybe_prefix)
48
49
50
class StablelmMLP(nn.Module):
@@ -193,12 +194,13 @@ class StablelmDecoderLayer(nn.Module):
194
195
class StableLMEpochModel(nn.Module):
196
196
- def __init__(self,
197
- config: PretrainedConfig,
198
- cache_config: Optional[CacheConfig] = None,
199
- quant_config: Optional[QuantizationConfig] = None,
200
- prefix: str = '') -> None:
197
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
198
super().__init__()
199
+
200
+ config = vllm_config.model_config.hf_config
201
+ cache_config = vllm_config.cache_config
202
+ quant_config = vllm_config.quant_config
203
+
204
self.embed_tokens = VocabParallelEmbedding(
205
config.vocab_size,
206
config.hidden_size,
@@ -245,18 +247,14 @@ class StableLMEpochModel(nn.Module):
247
248
class StablelmForCausalLM(nn.Module, SupportsPP):
249
248
- def __init__(
249
- self,
250
- vllm_config: VllmConfig,
251
- prefix: str = "",
252
- ) -> None:
250
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
251
super().__init__()
252
config = vllm_config.model_config.hf_config
255
- cache_config = vllm_config.cache_config
253
quant_config = vllm_config.quant_config
254
self.config = config
255
self.quant_config = quant_config
259
- self.model = StableLMEpochModel(config, cache_config, quant_config)
256
+ self.model = StableLMEpochModel(vllm_config=vllm_config,
257
+ prefix=maybe_prefix(prefix, "model"))
258
self.lm_head = ParallelLMHead(config.vocab_size,
259
config.hidden_size,
260
quant_config=quant_config)
vllm/model_executor/models/starcoder2.py
+11
-15
@@ -43,7 +43,8 @@ from vllm.sequence import IntermediateTensors
43
44
from .interfaces import SupportsPP
45
from .utils import (is_pp_missing_parameter,
46
- make_empty_intermediate_tensors_factory, make_layers)
46
+ make_empty_intermediate_tensors_factory, make_layers,
47
+ maybe_prefix)
48
49
50
class Starcoder2Attention(nn.Module):
@@ -195,12 +196,13 @@ class Starcoder2DecoderLayer(nn.Module):
196
@support_torch_compile
197
class Starcoder2Model(nn.Module):
198
198
- def __init__(self,
199
- config: Starcoder2Config,
200
- cache_config: Optional[CacheConfig] = None,
201
- quant_config: Optional[QuantizationConfig] = None,
202
- prefix: str = ""):
199
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
200
super().__init__()
201
+
202
+ config = vllm_config.model_config.hf_config
203
+ cache_config = vllm_config.cache_config
204
+ quant_config = vllm_config.quant_config
205
+
206
self.config = config
207
self.padding_idx = config.pad_token_id
208
self.vocab_size = config.vocab_size
@@ -245,19 +247,13 @@ class Starcoder2Model(nn.Module):
247
248
class Starcoder2ForCausalLM(nn.Module, SupportsPP):
249
248
- def __init__(
249
- self,
250
- vllm_config: VllmConfig,
251
- prefix: str = "",
252
- ) -> None:
250
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
251
super().__init__()
252
config = vllm_config.model_config.hf_config
255
- cache_config = vllm_config.cache_config
253
quant_config = vllm_config.quant_config
254
self.config = config
258
- self.model = Starcoder2Model(config,
259
- cache_config,
260
- quant_config=quant_config)
255
+ self.model = Starcoder2Model(vllm_config=vllm_config,
256
+ prefix=maybe_prefix(prefix, "model"))
257
self.vocab_size = config.vocab_size
258
self.unpadded_vocab_size = config.vocab_size
259
if config.tie_word_embeddings:
vllm/model_executor/models/ultravox.py
+9
-7
@@ -34,7 +34,7 @@ from vllm.utils import is_list_of
34
35
from .interfaces import SupportsMultiModal, SupportsPP
36
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
37
- init_vllm_registered_model,
37
+ init_vllm_registered_model, maybe_prefix,
38
merge_multimodal_embeddings_from_map)
39
40
_AUDIO_PLACEHOLDER_TOKEN = 128002
@@ -339,11 +339,7 @@ class ModifiedWhisperEncoder(WhisperEncoder):
339
@INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox)
340
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
341
342
- def __init__(
343
- self,
344
- vllm_config: VllmConfig,
345
- prefix: str = "",
346
- ) -> None:
342
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
343
super().__init__()
344
config = vllm_config.model_config.hf_config
345
multimodal_config = vllm_config.model_config.multimodal_config
@@ -354,6 +350,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
350
self.secondary_weights = []
351
self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
352
if config.audio_model_id is not None:
353
+ # this prefix is not for initialization, but for loading weights
354
+ # note the trailing dot
355
self.secondary_weights.append(
356
DefaultModelLoader.Source(
357
model_or_path=config.audio_model_id,
@@ -362,8 +360,12 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
360
))
361
self.multi_modal_projector = UltravoxProjector(config)
362
self.language_model = init_vllm_registered_model(
365
- config.text_config, vllm_config, prefix="language_model")
363
+ config.text_config,
364
+ vllm_config=vllm_config,
365
+ prefix=maybe_prefix(prefix, "language_model"))
366
if config.text_model_id is not None:
367
+ # this prefix is not for initialization, but for loading weights
368
+ # note the trailing dot
369
self.secondary_weights.append(
370
DefaultModelLoader.Source(model_or_path=config.text_model_id,
371
revision=None,
vllm/model_executor/models/xverse.py
+6
-13
@@ -46,7 +46,8 @@ from vllm.sequence import IntermediateTensors
46
47
from .interfaces import SupportsLoRA, SupportsPP
48
from .utils import (is_pp_missing_parameter,
49
- make_empty_intermediate_tensors_factory, make_layers)
49
+ make_empty_intermediate_tensors_factory, make_layers,
50
+ maybe_prefix)
51
52
53
class XverseMLP(nn.Module):
@@ -223,11 +224,7 @@ class XverseDecoderLayer(nn.Module):
224
@support_torch_compile
225
class XverseModel(nn.Module):
226
226
- def __init__(
227
- self,
228
- vllm_config: VllmConfig,
229
- prefix: str = "",
230
- ) -> None:
227
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
228
super().__init__()
229
config = vllm_config.model_config.hf_config
230
cache_config = vllm_config.cache_config
@@ -315,15 +312,10 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
312
}
313
embedding_padding_modules = ["lm_head"]
314
318
- def __init__(
319
- self,
320
- vllm_config: VllmConfig,
321
- prefix: str = "",
322
- ) -> None:
315
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
316
super().__init__()
317
318
config = vllm_config.model_config.hf_config
326
- cache_config = vllm_config.cache_config
319
quant_config = vllm_config.quant_config
320
lora_config = vllm_config.lora_config
321
@@ -331,7 +323,8 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
323
self.lora_config = lora_config
324
325
self.quant_config = quant_config
334
- self.model = XverseModel(config, cache_config, quant_config)
326
+ self.model = XverseModel(vllm_config=vllm_config,
327
+ prefix=maybe_prefix(prefix, "model"))
328
self.lm_head = ParallelLMHead(config.vocab_size,
329
config.hidden_size,
330
quant_config=quant_config)