A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[5/N] pass the whole config to model (#9983)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 8, 2024 at 22:17 UTC
1a95f10ee7d2ffa538a6d210b53bf363e039feee
75 files changed
+583
-654
vllm/model_executor/model_loader/loader.py
+7
-93
@@ -9,8 +9,7 @@ import math
9
import os
10
from abc import ABC, abstractmethod
11
from contextlib import contextmanager
12
-from typing import (Any, Dict, Generator, Iterable, List, Optional, Tuple,
13
- Type, cast)
12
+from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, cast
13
14
import gguf
15
import huggingface_hub
@@ -18,20 +17,17 @@ import numpy as np
17
import torch
18
from huggingface_hub import HfApi, hf_hub_download
19
from torch import nn
21
-from transformers import AutoModelForCausalLM, PretrainedConfig
20
+from transformers import AutoModelForCausalLM
21
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
22
24
-from vllm.config import (CacheConfig, LoadConfig, LoadFormat, LoRAConfig,
25
- ModelConfig, MultiModalConfig, ParallelConfig,
26
- PoolerConfig, SchedulerConfig, VllmConfig)
23
+from vllm.config import (LoadConfig, LoadFormat, ModelConfig, ParallelConfig,
24
+ VllmConfig)
25
from vllm.distributed import (get_tensor_model_parallel_rank,
26
get_tensor_model_parallel_world_size)
27
from vllm.envs import VLLM_USE_MODELSCOPE
28
from vllm.logger import init_logger
29
from vllm.model_executor.layers.linear import (ReplicatedLinear,
30
RowParallelLinear)
33
-from vllm.model_executor.layers.quantization.base_config import (
34
- QuantizationConfig)
31
from vllm.model_executor.model_loader.tensorizer import (
32
TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
33
serialize_vllm_model, tensorizer_weights_iterator)
@@ -43,8 +39,6 @@ from vllm.model_executor.model_loader.weight_utils import (
39
get_gguf_extra_tensor_names, gguf_quant_weights_iterator,
40
initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
41
safetensors_weights_iterator)
46
-from vllm.model_executor.models import (has_inner_state, supports_lora,
47
- supports_multimodal)
42
from vllm.model_executor.utils import set_weight_attrs
43
from vllm.platforms import current_platform
44
from vllm.utils import is_pin_memory_available
@@ -94,85 +88,11 @@ def device_loading_context(module: torch.nn.Module,
88
logger = init_logger(__name__)
89
90
97
-def _get_model_initialization_kwargs(
98
- model_class: Type[nn.Module],
99
- lora_config: Optional[LoRAConfig],
100
- multimodal_config: Optional[MultiModalConfig],
101
- scheduler_config: Optional[SchedulerConfig] = None,
102
- pooler_config: Optional[PoolerConfig] = None) -> Dict[str, Any]:
103
- """Get extra kwargs for model initialization."""
104
- extra_kwargs: Dict[str, Any] = {}
105
-
106
- if supports_lora(model_class):
107
- # lora_config=None is used to disable LoRA
108
- extra_kwargs["lora_config"] = lora_config
109
- elif lora_config:
110
- raise ValueError(
111
- f"Model {model_class.__name__} does not support LoRA, "
112
- "but LoRA is enabled. Support for this model may "
113
- "be added in the future. If this is important to you, "
114
- "please open an issue on github.")
115
-
116
- if supports_multimodal(model_class):
117
- assert multimodal_config is not None
118
-
119
- extra_kwargs["multimodal_config"] = multimodal_config
120
-
121
- if has_inner_state(model_class) and scheduler_config:
122
- extra_kwargs["scheduler_config"] = scheduler_config
123
- if pooler_config:
124
- extra_kwargs["pooler_config"] = pooler_config
125
- return extra_kwargs
126
-
127
-
128
-def build_model(model_class: Type[nn.Module],
129
- vllm_config: Optional[VllmConfig],
130
- hf_config: PretrainedConfig,
131
- cache_config: Optional[CacheConfig],
132
- quant_config: Optional[QuantizationConfig],
133
- *,
134
- lora_config: Optional[LoRAConfig],
135
- multimodal_config: Optional[MultiModalConfig],
136
- scheduler_config: Optional[SchedulerConfig],
137
- prefix: Optional[str] = None,
138
- pooler_config: Optional[PoolerConfig] = None) -> nn.Module:
139
- extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
140
- multimodal_config,
141
- scheduler_config,
142
- pooler_config)
143
- if prefix:
144
- extra_kwargs["prefix"] = prefix
145
-
146
- # TODO: unify all the module initialization code
147
- # to only take the `VllmConfig` object as input
148
- from vllm.plugins import set_vllm_config
149
- set_vllm_config(vllm_config)
150
-
151
- return model_class(config=hf_config,
152
- cache_config=cache_config,
153
- quant_config=quant_config,
154
- **extra_kwargs)
155
-
156
-
91
def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
92
"""Initialize a model with the given configurations."""
93
model_config = vllm_config.model_config
160
- lora_config = vllm_config.lora_config
161
- scheduler_config = vllm_config.scheduler_config
162
- cache_config = vllm_config.cache_config
94
model_class, _ = get_model_architecture(model_config)
164
-
165
- return build_model(
166
- model_class,
167
- vllm_config,
168
- model_config.hf_config,
169
- cache_config=cache_config,
170
- quant_config=vllm_config.quant_config,
171
- lora_config=lora_config,
172
- multimodal_config=model_config.multimodal_config,
173
- scheduler_config=scheduler_config,
174
- pooler_config=model_config.pooler_config,
175
- )
95
+ return model_class(vllm_config=vllm_config)
96
97
98
class BaseModelLoader(ABC):
@@ -486,24 +406,18 @@ class TensorizerLoader(BaseModelLoader):
406
407
device_config = vllm_config.device_config
408
model_config = vllm_config.model_config
489
- lora_config = vllm_config.lora_config
490
- cache_config = vllm_config.cache_config
409
410
with set_default_torch_dtype(model_config.dtype):
411
with torch.device(device_config.device):
412
model_class = get_model_architecture(model_config)[0]
495
- quant_config = vllm_config.quant_config
496
- extra_kwargs = _get_model_initialization_kwargs(
497
- model_class, lora_config, model_config.multimodal_config)
498
- extra_kwargs["quant_config"] = quant_config
499
- extra_kwargs["cache_config"] = cache_config
413
414
tensorizer_config = copy.copy(self.tensorizer_config)
415
tensorizer_config.model_class = model_class
416
tensorizer_config.hf_config = model_config.hf_config
417
tensorizer_config.dtype = model_config.dtype
418
506
- model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
419
+ model = load_with_tensorizer(tensorizer_config,
420
+ vllm_config=vllm_config)
421
return model.eval()
422
423
def download_model(self, model_config: ModelConfig) -> None:
vllm/model_executor/model_loader/tensorizer.py
+3
-12
@@ -17,8 +17,6 @@ from vllm.config import ModelConfig, ParallelConfig
17
from vllm.engine.arg_utils import EngineArgs
18
from vllm.engine.llm_engine import LLMEngine
19
from vllm.logger import init_logger
20
-from vllm.model_executor.layers.quantization.base_config import (
21
- QuantizationConfig)
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
VocabParallelEmbedding)
22
from vllm.utils import FlexibleArgumentParser
@@ -268,8 +266,7 @@ class TensorizerAgent:
266
in vllm/model_executor/model_loader/weight_utils.py
267
"""
268
271
- def __init__(self, tensorizer_config: TensorizerConfig,
272
- quant_config: QuantizationConfig, **extra_kwargs):
269
+ def __init__(self, tensorizer_config: TensorizerConfig, vllm_config):
270
if tensorizer_error_msg is not None:
271
raise ImportError(
272
"Tensorizer is not installed. Please install tensorizer "
@@ -279,11 +276,7 @@ class TensorizerAgent:
276
self.tensorizer_config = tensorizer_config
277
self.tensorizer_args = (
278
self.tensorizer_config._construct_tensorizer_args())
282
- self.extra_kwargs = extra_kwargs
283
- if extra_kwargs.get("quant_config") is not None:
284
- self.quant_config = extra_kwargs["quant_config"]
285
- else:
286
- self.quant_config = quant_config
279
+ self.vllm_config = vllm_config
280
self.model = self._init_model()
281
282
def _init_model(self):
@@ -293,9 +286,7 @@ class TensorizerAgent:
286
assert self.tensorizer_config.model_class is not None
287
with no_init_or_tensor():
288
return self.tensorizer_config.model_class(
296
- config=model_args,
297
- quant_config=self.quant_config,
298
- **self.extra_kwargs)
289
+ vllm_config=self.vllm_config, )
290
291
def _resize_lora_embeddings(self):
292
"""Modify LoRA embedding layers to use bigger tensors
vllm/model_executor/models/arctic.py
+9
-7
@@ -6,7 +6,7 @@ from torch import nn
6
7
from vllm.attention import Attention, AttentionMetadata
8
from vllm.compilation.decorators import support_torch_compile
9
-from vllm.config import CacheConfig
9
+from vllm.config import CacheConfig, VllmConfig
10
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size,
12
tensor_model_parallel_all_reduce)
@@ -415,14 +415,16 @@ class ArcticModel(nn.Module):
415
416
class ArcticForCausalLM(nn.Module, SupportsPP):
417
418
- def __init__(self,
419
- config: ArcticConfig,
420
- cache_config: Optional[CacheConfig] = None,
421
- quant_config: Optional[QuantizationConfig] = None,
422
- **kwargs) -> None:
418
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
419
super().__init__()
420
+ config = vllm_config.model_config.hf_config
421
+ cache_config = vllm_config.cache_config
422
+ quant_config = vllm_config.quant_config
423
self.config = config
425
- self.model = ArcticModel(config, cache_config, quant_config)
424
+ self.model = ArcticModel(config,
425
+ cache_config,
426
+ quant_config,
427
+ prefix=prefix)
428
self.vocab_size = config.vocab_size
429
self.lm_head = ParallelLMHead(
430
self.vocab_size,
vllm/model_executor/models/baichuan.py
+16
-21
@@ -26,7 +26,7 @@ from transformers import PretrainedConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
from vllm.compilation.decorators import support_torch_compile
29
-from vllm.config import CacheConfig, LoRAConfig
29
+from vllm.config import CacheConfig, VllmConfig
30
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
31
get_tensor_model_parallel_world_size)
32
from vllm.model_executor.layers.activation import SiluAndMul
@@ -332,14 +332,15 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
333
def __init__(
334
self,
335
- config: PretrainedConfig,
336
- position_embedding: str,
337
- cache_config: Optional[CacheConfig] = None,
338
- quant_config: Optional[QuantizationConfig] = None,
339
- lora_config: Optional[LoRAConfig] = None,
335
+ vllm_config: VllmConfig,
336
+ prefix: str = "",
337
+ position_embedding: str = "ROPE",
338
):
339
super().__init__()
342
-
340
+ config = vllm_config.model_config.hf_config
341
+ cache_config = vllm_config.cache_config
342
+ quant_config = vllm_config.quant_config
343
+ lora_config = vllm_config.lora_config
344
self.config = config
345
self.lora_config = lora_config
346
@@ -439,17 +440,14 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
440
441
def __init__(
442
self,
442
- config: PretrainedConfig,
443
- cache_config: Optional[CacheConfig] = None,
444
- quant_config: Optional[QuantizationConfig] = None,
445
- lora_config: Optional[LoRAConfig] = None,
443
+ vllm_config: VllmConfig,
444
+ prefix: str = "",
445
):
446
+ config = vllm_config.model_config.hf_config
447
if config.hidden_size == 4096: # baichuan2 7b
448
- super().__init__(config, "ROPE", cache_config, quant_config,
449
- lora_config)
448
+ super().__init__(vllm_config, prefix, "ROPE")
449
else: # baichuan 13b, baichuan2 13b
451
- super().__init__(config, "ALIBI", cache_config, quant_config,
452
- lora_config)
450
+ super().__init__(vllm_config, prefix, "ALIBI")
451
452
453
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -459,10 +457,7 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
457
458
def __init__(
459
self,
462
- config: PretrainedConfig,
463
- cache_config: Optional[CacheConfig] = None,
464
- quant_config: Optional[QuantizationConfig] = None,
465
- lora_config: Optional[LoRAConfig] = None,
460
+ vllm_config: VllmConfig,
461
+ prefix: str = "",
462
):
467
- super().__init__(config, "ROPE", cache_config, quant_config,
468
- lora_config)
463
+ super().__init__(vllm_config, prefix, "ROPE")
vllm/model_executor/models/bart.py
+6
-6
@@ -25,7 +25,7 @@ from transformers import BartConfig
25
from transformers.utils import logging
26
27
from vllm.attention import Attention, AttentionMetadata, AttentionType
28
-from vllm.config import CacheConfig, LoRAConfig
28
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
29
from vllm.distributed import get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -810,13 +810,13 @@ class BartModel(nn.Module):
810
class BartForConditionalGeneration(nn.Module):
811
base_model_prefix = "model"
812
813
- def __init__(self,
814
- config: BartConfig,
815
- cache_config: Optional[CacheConfig] = None,
816
- quant_config: Optional[QuantizationConfig] = None,
817
- lora_config: Optional[LoRAConfig] = None):
813
+ def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
814
815
super().__init__()
816
+ config = vllm_config.model_config.hf_config
817
+ cache_config = vllm_config.cache_config
818
+ quant_config = vllm_config.quant_config
819
+ lora_config = vllm_config.lora_config
820
# currently all existing BART models have `tie_word_embeddings` enabled
821
assert config.tie_word_embeddings
822
self.config = config
vllm/model_executor/models/bert.py
+7
-5
@@ -6,7 +6,7 @@ from transformers import BertConfig
6
7
from vllm.attention import Attention, AttentionMetadata, AttentionType
8
from vllm.attention.backends.xformers import XFormersImpl
9
-from vllm.config import CacheConfig, PoolerConfig
9
+from vllm.config import CacheConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -384,12 +384,14 @@ class BertEmbeddingModel(nn.Module):
384
385
def __init__(
386
self,
387
- config: BertConfig,
388
- cache_config: Optional[CacheConfig] = None,
389
- quant_config: Optional[QuantizationConfig] = None,
390
- pooler_config: Optional[PoolerConfig] = None,
387
+ vllm_config: VllmConfig,
388
+ prefix: str = "",
389
) -> None:
390
super().__init__()
391
+ config = vllm_config.model_config.hf_config
392
+ cache_config = vllm_config.cache_config
393
+ quant_config = vllm_config.quant_config
394
+ pooler_config = vllm_config.model_config.pooler_config
395
self.model = BertModel(config, cache_config, quant_config)
396
self._pooler = Pooler.from_config_with_defaults(
397
pooler_config,
vllm/model_executor/models/blip2.py
+11
-9
@@ -8,7 +8,7 @@ from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
8
apply_chunking_to_forward)
9
10
from vllm.attention import AttentionMetadata
11
-from vllm.config import CacheConfig, MultiModalConfig
11
+from vllm.config import CacheConfig, VllmConfig
12
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13
InputContext, token_inputs)
14
from vllm.model_executor.layers.activation import get_act_fn
@@ -483,14 +483,17 @@ def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
483
@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
484
class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
485
486
- def __init__(self,
487
- config: Blip2Config,
488
- multimodal_config: MultiModalConfig,
489
- cache_config: Optional[CacheConfig] = None,
490
- quant_config: Optional[QuantizationConfig] = None) -> None:
486
+ def __init__(
487
+ self,
488
+ vllm_config: VllmConfig,
489
+ prefix: str = "",
490
+ ) -> None:
491
492
super().__init__()
493
-
493
+ config = vllm_config.model_config.hf_config
494
+ cache_config = vllm_config.cache_config
495
+ quant_config = vllm_config.quant_config
496
+ multimodal_config = vllm_config.model_config.multimodal_config
497
self.config = config
498
self.multimodal_config = multimodal_config
499
@@ -513,8 +516,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
516
517
self.language_model = init_vllm_registered_model(
518
config.text_config,
516
- cache_config,
517
- quant_config,
519
+ vllm_config=vllm_config,
520
prefix="language_model")
521
522
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/bloom.py
+6
-4
@@ -24,7 +24,7 @@ from transformers import BloomConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
from vllm.compilation.decorators import support_torch_compile
27
-from vllm.config import CacheConfig
27
+from vllm.config import CacheConfig, VllmConfig
28
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
29
get_tensor_model_parallel_world_size)
30
from vllm.model_executor.layers.activation import get_act_fn
@@ -283,11 +283,13 @@ class BloomForCausalLM(nn.Module, SupportsPP):
283
284
def __init__(
285
self,
286
- config: BloomConfig,
287
- cache_config: Optional[CacheConfig] = None,
288
- quant_config: Optional[QuantizationConfig] = None,
286
+ vllm_config: VllmConfig,
287
+ prefix: str = "",
288
):
289
super().__init__()
290
+ config = vllm_config.model_config.hf_config
291
+ cache_config = vllm_config.cache_config
292
+ quant_config = vllm_config.quant_config
293
self.config = config
294
self.quant_config = quant_config
295
self.transformer = BloomModel(config, cache_config, quant_config)
vllm/model_executor/models/chameleon.py
+7
-5
@@ -9,7 +9,7 @@ from torch import nn
9
from transformers import ChameleonConfig, ChameleonVQVAEConfig
10
11
from vllm.attention import Attention, AttentionMetadata
12
-from vllm.config import CacheConfig, MultiModalConfig
12
+from vllm.config import CacheConfig, VllmConfig
13
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
InputContext, token_inputs)
@@ -926,12 +926,14 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
926
927
def __init__(
928
self,
929
- config: ChameleonConfig,
930
- multimodal_config: MultiModalConfig,
931
- cache_config: Optional[CacheConfig] = None,
932
- quant_config: Optional[QuantizationConfig] = None,
929
+ vllm_config: VllmConfig,
930
+ prefix: str = "",
931
) -> None:
932
super().__init__()
933
+ config = vllm_config.model_config.hf_config
934
+ cache_config = vllm_config.cache_config
935
+ quant_config = vllm_config.quant_config
936
+ multimodal_config = vllm_config.model_config.multimodal_config
937
self.config = config
938
self.multimodal_config = multimodal_config
939
self.model = ChameleonModel(config, cache_config, quant_config)
vllm/model_executor/models/chatglm.py
+8
-7
@@ -11,7 +11,7 @@ from torch import nn
11
from torch.nn import LayerNorm
12
13
from vllm.attention import Attention, AttentionMetadata
14
-from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
14
+from vllm.config import CacheConfig, VllmConfig
15
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
17
InputContext, token_inputs)
@@ -595,14 +595,15 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
595
596
def __init__(
597
self,
598
- config: ChatGLMConfig,
599
- multimodal_config: MultiModalConfig,
600
- cache_config: Optional[CacheConfig] = None,
601
- quant_config: Optional[QuantizationConfig] = None,
602
- lora_config: Optional[LoRAConfig] = None,
598
+ vllm_config: VllmConfig,
599
+ prefix: str = "",
600
):
601
super().__init__()
605
-
602
+ config = vllm_config.model_config.hf_config
603
+ cache_config = vllm_config.cache_config
604
+ quant_config = vllm_config.quant_config
605
+ lora_config = vllm_config.lora_config
606
+ multimodal_config = vllm_config.model_config.multimodal_config
607
self.config = config
608
self.lora_config = lora_config
609
self.multimodal_config = multimodal_config
vllm/model_executor/models/commandr.py
+7
-5
@@ -28,7 +28,7 @@ from transformers import CohereConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -334,12 +334,14 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
334
335
def __init__(
336
self,
337
- config: CohereConfig,
338
- cache_config: Optional[CacheConfig] = None,
339
- quant_config: Optional[QuantizationConfig] = None,
340
- lora_config: Optional[LoRAConfig] = None,
337
+ vllm_config: VllmConfig,
338
+ prefix: str = "",
339
) -> None:
340
super().__init__()
341
+ config = vllm_config.model_config.hf_config
342
+ cache_config = vllm_config.cache_config
343
+ quant_config = vllm_config.quant_config
344
+ lora_config = vllm_config.lora_config
345
self.config = config
346
# currently all existing command R models have `tie_word_embeddings`
347
# enabled
vllm/model_executor/models/dbrx.py
+6
-4
@@ -4,7 +4,7 @@ import torch
4
import torch.nn as nn
5
6
from vllm.attention import Attention, AttentionMetadata
7
-from vllm.config import CacheConfig
7
+from vllm.config import CacheConfig, VllmConfig
8
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
9
get_tensor_model_parallel_world_size)
10
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -352,11 +352,13 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
352
353
def __init__(
354
self,
355
- config: DbrxConfig,
356
- cache_config: Optional[CacheConfig] = None,
357
- quant_config: Optional[QuantizationConfig] = None,
355
+ vllm_config: VllmConfig,
356
+ prefix: str = "",
357
):
358
super().__init__()
359
+ config = vllm_config.model_config.hf_config
360
+ cache_config = vllm_config.cache_config
361
+ quant_config = vllm_config.quant_config
362
self.config = config
363
if config.tie_word_embeddings:
364
raise ValueError(
vllm/model_executor/models/decilm.py
+6
-12
@@ -22,13 +22,11 @@
22
# limitations under the License.
23
"""Inference-only DeciLM model compatible with HuggingFace weights."""
24
25
-from typing import Iterable, Optional, Tuple
25
+from typing import Iterable, Tuple
26
27
import torch
28
-from transformers import LlamaConfig
28
30
-from vllm.config import CacheConfig, LoRAConfig
31
-from vllm.model_executor.layers.quantization import QuantizationConfig
29
+from vllm.config import VllmConfig
30
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31
from vllm.model_executor.models.llama import LlamaForCausalLM
32
@@ -55,17 +53,13 @@ class DeciLMForCausalLM(LlamaForCausalLM):
53
54
def __init__(
55
self,
58
- config: LlamaConfig,
59
- cache_config: Optional[CacheConfig] = None,
60
- quant_config: Optional[QuantizationConfig] = None,
61
- lora_config: Optional[LoRAConfig] = None,
56
+ vllm_config: VllmConfig,
57
+ prefix: str = "",
58
) -> None:
59
+ config = vllm_config.model_config.hf_config
60
config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
61
delattr(config, "num_key_value_heads_per_layer")
65
- super().__init__(config=config,
66
- cache_config=cache_config,
67
- quant_config=quant_config,
68
- lora_config=lora_config)
62
+ super().__init__(vllm_config=vllm_config)
63
64
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
65
stacked_params_mapping = [
vllm/model_executor/models/deepseek.py
+6
-4
@@ -27,7 +27,7 @@ from torch import nn
27
from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
-from vllm.config import CacheConfig
30
+from vllm.config import CacheConfig, VllmConfig
31
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32
get_tensor_model_parallel_world_size,
33
tensor_model_parallel_all_reduce)
@@ -385,11 +385,13 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
385
386
def __init__(
387
self,
388
- config: PretrainedConfig,
389
- cache_config: Optional[CacheConfig] = None,
390
- quant_config: Optional[QuantizationConfig] = None,
388
+ vllm_config: VllmConfig,
389
+ prefix: str = "",
390
) -> None:
391
super().__init__()
392
+ config = vllm_config.model_config.hf_config
393
+ cache_config = vllm_config.cache_config
394
+ quant_config = vllm_config.quant_config
395
self.config = config
396
self.quant_config = quant_config
397
self.model = DeepseekModel(config, cache_config, quant_config)
vllm/model_executor/models/deepseek_v2.py
+6
-4
@@ -28,7 +28,7 @@ from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import (get_pp_group,
33
get_tensor_model_parallel_world_size,
34
tensor_model_parallel_all_reduce)
@@ -481,11 +481,13 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
481
482
def __init__(
483
self,
484
- config: PretrainedConfig,
485
- cache_config: Optional[CacheConfig] = None,
486
- quant_config: Optional[QuantizationConfig] = None,
484
+ vllm_config: VllmConfig,
485
+ prefix: str = "",
486
) -> None:
487
super().__init__()
488
+ config = vllm_config.model_config.hf_config
489
+ cache_config = vllm_config.cache_config
490
+ quant_config = vllm_config.quant_config
491
self.config = config
492
self.quant_config = quant_config
493
self.model = DeepseekV2Model(config,
vllm/model_executor/models/eagle.py
+4
-3
@@ -4,6 +4,7 @@ import torch
4
import torch.nn as nn
5
6
from vllm.attention.backends.abstract import AttentionMetadata
7
+from vllm.config import VllmConfig
8
from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
from vllm.model_executor.layers.sampler import SamplerOutput
10
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -12,7 +13,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
from vllm.model_executor.models import ModelRegistry
14
from vllm.model_executor.sampling_metadata import SamplingMetadata
15
from vllm.sequence import IntermediateTensors
15
-from vllm.transformers_utils.configs.eagle import EAGLEConfig
16
17
18
class EAGLE(nn.Module):
@@ -34,14 +34,15 @@ class EAGLE(nn.Module):
34
in the draft checkpoint (using key token_map). Also, the draft config
35
needs to have truncated_vocab_size (=k) as an attribute."""
36
37
- def __init__(self, config: EAGLEConfig, *args, **kwargs) -> None:
37
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
38
super().__init__()
39
+ config = vllm_config.model_config.hf_config
40
self.config = config
41
42
architectures = getattr(self.config.model, "architectures", [])
43
model_cls, _ = ModelRegistry.resolve_model_cls(architectures)
44
44
- self.model = model_cls(self.config.model, *args, **kwargs)
45
+ self.model = model_cls(vllm_config, prefix)
46
self.fc = nn.Linear(config.model.hidden_size * 2,
47
config.model.hidden_size,
48
bias=getattr(self.config, "eagle_fc_bias", False))
vllm/model_executor/models/exaone.py
+7
-5
@@ -29,7 +29,7 @@ from torch import nn
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -440,12 +440,14 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
440
441
def __init__(
442
self,
443
- config: ExaoneConfig,
444
- cache_config: Optional[CacheConfig] = None,
445
- quant_config: Optional[QuantizationConfig] = None,
446
- lora_config: Optional[LoRAConfig] = None,
443
+ vllm_config: VllmConfig,
444
+ prefix: str = "",
445
) -> None:
446
super().__init__()
447
+ config = vllm_config.model_config.hf_config
448
+ cache_config = vllm_config.cache_config
449
+ quant_config = vllm_config.quant_config
450
+ lora_config = vllm_config.lora_config
451
452
self.config = config
453
self.lora_config = lora_config
vllm/model_executor/models/falcon.py
+6
-4
@@ -27,7 +27,7 @@ from transformers import FalconConfig as HF_FalconConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.compilation.decorators import support_torch_compile
30
-from vllm.config import CacheConfig
30
+from vllm.config import CacheConfig, VllmConfig
31
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32
get_tensor_model_parallel_world_size,
33
tensor_model_parallel_all_reduce)
@@ -403,11 +403,13 @@ class FalconForCausalLM(nn.Module, SupportsPP):
403
404
def __init__(
405
self,
406
- config: FalconConfig,
407
- cache_config: Optional[CacheConfig] = None,
408
- quant_config: Optional[QuantizationConfig] = None,
406
+ vllm_config: VllmConfig,
407
+ prefix: str = "",
408
):
409
super().__init__()
410
+ config = vllm_config.model_config.hf_config
411
+ cache_config = vllm_config.cache_config
412
+ quant_config = vllm_config.quant_config
413
self.config = config
414
self.quant_config = quant_config
415
self.transformer = FalconModel(config, cache_config, quant_config)
vllm/model_executor/models/florence2.py
+5
-5
@@ -6,7 +6,7 @@ import torch.nn as nn
6
from transformers import PretrainedConfig
7
8
from vllm.attention import AttentionMetadata
9
-from vllm.config import CacheConfig
9
+from vllm.config import CacheConfig, VllmConfig
10
from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
@@ -189,11 +189,11 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
189
190
class Florence2ForConditionalGeneration(nn.Module):
191
192
- def __init__(self,
193
- config: PretrainedConfig,
194
- cache_config: Optional[CacheConfig] = None,
195
- quant_config: Optional[QuantizationConfig] = None):
192
+ def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
193
super().__init__()
194
+ config = vllm_config.model_config.hf_config
195
+ cache_config = vllm_config.cache_config
196
+ quant_config = vllm_config.quant_config
197
198
# TODO(Isotr0py): Add vision backbone
199
self.language_model = Florence2LanguageForConditionalGeneration(
vllm/model_executor/models/fuyu.py
+7
-8
@@ -22,14 +22,13 @@ import torch
22
import torch.nn as nn
23
import torch.utils.checkpoint
24
from PIL import Image
25
-from transformers import FuyuConfig, FuyuImageProcessor
25
+from transformers import FuyuImageProcessor
26
27
from vllm.attention import AttentionMetadata
28
-from vllm.config import CacheConfig, MultiModalConfig
28
+from vllm.config import VllmConfig
29
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30
InputContext, token_inputs)
31
from vllm.model_executor.layers.linear import ColumnParallelLinear
32
-from vllm.model_executor.layers.quantization import QuantizationConfig
32
from vllm.model_executor.layers.sampler import SamplerOutput
33
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -227,12 +226,12 @@ def input_mapper_for_fuyu(ctx: InputContext, data: object):
226
@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
227
class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
228
230
- def __init__(self,
231
- config: FuyuConfig,
232
- multimodal_config: MultiModalConfig,
233
- cache_config: Optional[CacheConfig] = None,
234
- quant_config: Optional[QuantizationConfig] = None) -> None:
229
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
230
super().__init__()
231
+ config = vllm_config.model_config.hf_config
232
+ cache_config = vllm_config.cache_config
233
+ quant_config = vllm_config.quant_config
234
+ multimodal_config = vllm_config.model_config.multimodal_config
235
self.config = config
236
self.multimodal_config = multimodal_config
237
vllm/model_executor/models/gemma.py
+6
-5
@@ -22,7 +22,7 @@ from transformers import GemmaConfig
22
23
from vllm.attention import Attention, AttentionMetadata
24
from vllm.compilation.decorators import support_torch_compile
25
-from vllm.config import CacheConfig, LoRAConfig
25
+from vllm.config import CacheConfig, VllmConfig
26
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import GeluAndMul
@@ -374,13 +374,14 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
374
375
def __init__(
376
self,
377
- config: GemmaConfig,
378
- cache_config: Optional[CacheConfig] = None,
379
- quant_config: Optional[QuantizationConfig] = None,
380
- lora_config: Optional[LoRAConfig] = None,
377
+ vllm_config: VllmConfig,
378
prefix: str = "",
379
) -> None:
380
super().__init__()
381
+ config = vllm_config.model_config.hf_config
382
+ cache_config = vllm_config.cache_config
383
+ quant_config = vllm_config.quant_config
384
+ lora_config = vllm_config.lora_config
385
386
self.config = config
387
# currently all existing Gemma models have `tie_word_embeddings` enabled
vllm/model_executor/models/gemma2.py
+15
-12
@@ -21,7 +21,7 @@ from transformers import Gemma2Config
21
22
from vllm.attention import Attention, AttentionMetadata
23
from vllm.compilation.decorators import support_torch_compile
24
-from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
24
+from vllm.config import CacheConfig, VllmConfig
25
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
26
from vllm.logger import init_logger
27
from vllm.model_executor.layers.activation import GeluAndMul
@@ -245,12 +245,13 @@ class Gemma2Model(nn.Module):
245
246
def __init__(
247
self,
248
- config: Gemma2Config,
249
- cache_config: Optional[CacheConfig] = None,
250
- quant_config: Optional[QuantizationConfig] = None,
248
+ vllm_config: VllmConfig,
249
prefix: str = "",
250
) -> None:
251
super().__init__()
252
+ config = vllm_config.model_config.hf_config
253
+ cache_config = vllm_config.cache_config
254
+ quant_config = vllm_config.quant_config
255
self.config = config
256
257
self.embed_tokens = VocabParallelEmbedding(
@@ -400,11 +401,13 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
401
402
def __init__(
403
self,
403
- config: Gemma2Config,
404
- cache_config: Optional[CacheConfig] = None,
405
- quant_config: Optional[QuantizationConfig] = None,
406
- lora_config: Optional[LoRAConfig] = None,
404
+ vllm_config: VllmConfig,
405
+ prefix: str = "",
406
) -> None:
407
+ config = vllm_config.model_config.hf_config
408
+ cache_config = vllm_config.cache_config
409
+ quant_config = vllm_config.quant_config
410
+ lora_config = vllm_config.lora_config
411
del lora_config # Unused.
412
super().__init__()
413
self.config = config
@@ -470,14 +473,14 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
473
474
def __init__(
475
self,
473
- pooler_config: Optional[PoolerConfig] = None,
474
- **kwargs,
476
+ vllm_config: VllmConfig,
477
+ prefix: str = "",
478
) -> None:
479
super().__init__()
480
478
- self.model = Gemma2Model(**kwargs)
481
+ self.model = Gemma2Model(vllm_config, prefix)
482
self._pooler = Pooler.from_config_with_defaults(
480
- pooler_config,
483
+ vllm_config.model_config.pooler_config,
484
pooling_type=PoolingType.LAST,
485
normalize=True,
486
softmax=False)
vllm/model_executor/models/gpt2.py
+6
-4
@@ -24,7 +24,7 @@ from transformers import GPT2Config
24
25
from vllm.attention import Attention, AttentionMetadata
26
from vllm.compilation.decorators import support_torch_compile
27
-from vllm.config import CacheConfig
27
+from vllm.config import CacheConfig, VllmConfig
28
from vllm.distributed.parallel_state import (
29
get_pp_group, get_tensor_model_parallel_world_size)
30
from vllm.model_executor.layers.activation import get_act_fn
@@ -242,11 +242,13 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
242
243
def __init__(
244
self,
245
- config: GPT2Config,
246
- cache_config: Optional[CacheConfig] = None,
247
- quant_config: Optional[QuantizationConfig] = None,
245
+ vllm_config: VllmConfig,
246
+ prefix: str = "",
247
):
248
super().__init__()
249
+ config = vllm_config.model_config.hf_config
250
+ cache_config = vllm_config.cache_config
251
+ quant_config = vllm_config.quant_config
252
self.config = config
253
self.quant_config = quant_config
254
self.transformer = GPT2Model(config,
vllm/model_executor/models/gpt_bigcode.py
+7
-5
@@ -25,7 +25,7 @@ from transformers import GPTBigCodeConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
from vllm.compilation.decorators import support_torch_compile
28
-from vllm.config import CacheConfig, LoRAConfig
28
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
29
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -260,12 +260,14 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
260
261
def __init__(
262
self,
263
- config: GPTBigCodeConfig,
264
- cache_config: Optional[CacheConfig] = None,
265
- quant_config: Optional[QuantizationConfig] = None,
266
- lora_config: Optional[LoRAConfig] = None,
263
+ vllm_config: VllmConfig,
264
+ prefix: str = "",
265
):
266
super().__init__()
267
+ config = vllm_config.model_config.hf_config
268
+ cache_config = vllm_config.cache_config
269
+ quant_config = vllm_config.quant_config
270
+ lora_config = vllm_config.lora_config
271
272
self.config = config
273
self.lora_config = lora_config
vllm/model_executor/models/gpt_j.py
+6
-4
@@ -23,7 +23,7 @@ from transformers import GPTJConfig
23
24
from vllm.attention import Attention, AttentionMetadata
25
from vllm.compilation.decorators import support_torch_compile
26
-from vllm.config import CacheConfig
26
+from vllm.config import CacheConfig, VllmConfig
27
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -231,11 +231,13 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
231
232
def __init__(
233
self,
234
- config: GPTJConfig,
235
- cache_config: Optional[CacheConfig] = None,
236
- quant_config: Optional[QuantizationConfig] = None,
234
+ vllm_config: VllmConfig,
235
+ prefix: str = "",
236
):
237
super().__init__()
238
+ config = vllm_config.model_config.hf_config
239
+ cache_config = vllm_config.cache_config
240
+ quant_config = vllm_config.quant_config
241
self.config = config
242
self.quant_config = quant_config
243
assert not config.tie_word_embeddings
vllm/model_executor/models/gpt_neox.py
+6
-4
@@ -23,7 +23,7 @@ from transformers import GPTNeoXConfig
23
24
from vllm.attention import Attention, AttentionMetadata
25
from vllm.compilation.decorators import support_torch_compile
26
-from vllm.config import CacheConfig
26
+from vllm.config import CacheConfig, VllmConfig
27
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -244,11 +244,13 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
244
245
def __init__(
246
self,
247
- config: GPTNeoXConfig,
248
- cache_config: Optional[CacheConfig] = None,
249
- quant_config: Optional[QuantizationConfig] = None,
247
+ vllm_config: VllmConfig,
248
+ prefix: str = "",
249
):
250
super().__init__()
251
+ config = vllm_config.model_config.hf_config
252
+ cache_config = vllm_config.cache_config
253
+ quant_config = vllm_config.quant_config
254
self.config = config
255
self.quant_config = quant_config
256
self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config)
vllm/model_executor/models/granite.py
+7
-5
@@ -28,7 +28,7 @@ from transformers import GraniteConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -372,12 +372,14 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
372
373
def __init__(
374
self,
375
- config: GraniteConfig,
376
- cache_config: Optional[CacheConfig] = None,
377
- quant_config: Optional[QuantizationConfig] = None,
378
- lora_config: Optional[LoRAConfig] = None,
375
+ vllm_config: VllmConfig,
376
+ prefix: str = "",
377
) -> None:
378
super().__init__()
379
+ config = vllm_config.model_config.hf_config
380
+ cache_config = vllm_config.cache_config
381
+ quant_config = vllm_config.quant_config
382
+ lora_config = vllm_config.lora_config
383
384
self.config = config
385
self.lora_config = lora_config
vllm/model_executor/models/granitemoe.py
+7
-5
@@ -28,7 +28,7 @@ from transformers.models.granitemoe import GraniteMoeConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -335,12 +335,14 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
335
336
def __init__(
337
self,
338
- config: GraniteMoeConfig,
339
- cache_config: Optional[CacheConfig] = None,
340
- quant_config: Optional[QuantizationConfig] = None,
341
- lora_config: Optional[LoRAConfig] = None,
338
+ vllm_config: VllmConfig,
339
+ prefix: str = "",
340
) -> None:
341
super().__init__()
342
+ config = vllm_config.model_config.hf_config
343
+ cache_config = vllm_config.cache_config
344
+ quant_config = vllm_config.quant_config
345
+ lora_config = vllm_config.lora_config
346
347
self.config = config
348
self.lora_config = lora_config
vllm/model_executor/models/idefics3.py
+8
-5
@@ -26,7 +26,7 @@ from transformers import PretrainedConfig as Idefics3Config
26
from transformers import ProcessorMixin as Idefics3ImageProcessor
27
28
from vllm.attention import AttentionMetadata
29
-from vllm.config import CacheConfig, MultiModalConfig
29
+from vllm.config import CacheConfig, VllmConfig
30
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
31
InputContext, token_inputs)
32
from vllm.logger import init_logger
@@ -615,13 +615,16 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
615
616
def __init__(
617
self,
618
- config: Idefics3Config,
619
- multimodal_config: MultiModalConfig,
620
- cache_config: Optional[CacheConfig] = None,
621
- quant_config: Optional[QuantizationConfig] = None,
618
+ vllm_config: VllmConfig,
619
+ prefix: str = "",
620
) -> None:
621
super().__init__()
622
623
+ config = vllm_config.model_config.hf_config
624
+ cache_config = vllm_config.cache_config
625
+ quant_config = vllm_config.quant_config
626
+ multimodal_config = vllm_config.model_config.multimodal_config
627
+
628
self.config = config
629
self.multimodal_config = multimodal_config
630
vllm/model_executor/models/interfaces_base.py
+4
-20
@@ -11,9 +11,8 @@ from vllm.utils import supports_kw
11
12
if TYPE_CHECKING:
13
from vllm.attention import AttentionMetadata
14
- from vllm.config import CacheConfig
14
+ from vllm.config import VllmConfig
15
from vllm.model_executor.layers.pooler import PoolerOutput
16
- from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.layers.sampler import SamplerOutput
17
from vllm.model_executor.pooling_metadata import PoolingMetadata
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -39,10 +38,8 @@ class VllmModel(Protocol[C_co, T_co]):
38
39
def __init__(
40
self,
42
- config: C_co,
43
- *,
44
- cache_config: Optional["CacheConfig"],
45
- quant_config: Optional["QuantizationConfig"],
41
+ vllm_config: "VllmConfig",
42
+ prefix: str = "",
43
) -> None:
44
...
45
@@ -58,20 +55,7 @@ class VllmModel(Protocol[C_co, T_co]):
55
56
def _check_vllm_model_init(model: Union[Type[object], object]) -> bool:
57
model_init = model.__init__
61
- vllm_kws = ("cache_config", "quant_config")
62
- missing_kws = tuple(kw for kw in vllm_kws
63
- if not supports_kw(model_init, kw))
64
-
65
- if missing_kws and (isinstance(model, type)
66
- and issubclass(model, nn.Module)):
67
- logger.warning(
68
- "The model (%s) is missing "
69
- "vLLM-specific keywords from its initializer: %s",
70
- model,
71
- missing_kws,
72
- )
73
-
74
- return len(missing_kws) == 0
58
+ return supports_kw(model_init, "vllm_config")
59
60
61
def _check_vllm_model_forward(model: Union[Type[object], object]) -> bool:
vllm/model_executor/models/internlm2.py
+5
-4
@@ -7,7 +7,7 @@ from transformers import PretrainedConfig
7
8
from vllm.attention import Attention, AttentionMetadata
9
from vllm.compilation.decorators import support_torch_compile
10
-from vllm.config import CacheConfig
10
+from vllm.config import CacheConfig, VllmConfig
11
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
12
get_tensor_model_parallel_world_size,
13
split_tensor_along_last_dim,
@@ -319,12 +319,13 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
319
320
def __init__(
321
self,
322
- config: PretrainedConfig,
323
- cache_config: Optional[CacheConfig] = None,
324
- quant_config: Optional[QuantizationConfig] = None,
322
+ vllm_config: VllmConfig,
323
prefix: str = "",
324
) -> None:
325
super().__init__()
326
+ config = vllm_config.model_config.hf_config
327
+ cache_config = vllm_config.cache_config
328
+ quant_config = vllm_config.quant_config
329
self.config = config
330
self.quant_config = quant_config
331
self.model = InternLM2Model(config,
vllm/model_executor/models/internlm2_ve.py
+5
-4
@@ -5,7 +5,7 @@ from torch import nn
5
from transformers import PretrainedConfig
6
7
from vllm.attention import AttentionMetadata
8
-from vllm.config import CacheConfig
8
+from vllm.config import CacheConfig, VllmConfig
9
from vllm.distributed import get_pp_group
10
from vllm.model_executor.layers.layernorm import RMSNorm
11
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -161,11 +161,12 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
161
162
def __init__(
163
self,
164
- config: PretrainedConfig,
165
- cache_config: Optional[CacheConfig] = None,
166
- quant_config: Optional[QuantizationConfig] = None,
164
+ vllm_config: VllmConfig,
165
prefix: str = "",
166
) -> None:
167
+ config = vllm_config.model_config.hf_config
168
+ cache_config = vllm_config.cache_config
169
+ quant_config = vllm_config.quant_config
170
super().__init__(config, cache_config, quant_config)
171
self.model = InternLM2VEModel(config,
172
cache_config,
vllm/model_executor/models/internvl.py
+7
-8
@@ -16,7 +16,7 @@ from PIL import Image
16
from transformers import PretrainedConfig
17
18
from vllm.attention import AttentionMetadata
19
-from vllm.config import CacheConfig, MultiModalConfig
19
+from vllm.config import VllmConfig
20
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
InputContext, token_inputs)
22
from vllm.model_executor.layers.quantization import (AWQConfig,
@@ -410,13 +410,13 @@ input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
410
@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
411
class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
412
413
- def __init__(self,
414
- config: PretrainedConfig,
415
- multimodal_config: MultiModalConfig,
416
- cache_config: Optional[CacheConfig] = None,
417
- quant_config: Optional[QuantizationConfig] = None) -> None:
413
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
414
super().__init__()
415
416
+ config = vllm_config.model_config.hf_config
417
+ quant_config = vllm_config.quant_config
418
+ multimodal_config = vllm_config.model_config.multimodal_config
419
+
420
self.config = config
421
self.multimodal_config = multimodal_config
422
self._patch_quant_config(config, quant_config)
@@ -440,8 +440,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
440
441
self.language_model = init_vllm_registered_model(
442
config.text_config,
443
- cache_config,
444
- quant_config,
443
+ vllm_config=vllm_config,
444
prefix="language_model")
445
446
self.mlp1 = self._init_mlp1(config)
vllm/model_executor/models/jais.py
+6
-4
@@ -26,7 +26,7 @@ from torch import nn
26
27
from vllm.attention import Attention, AttentionMetadata
28
from vllm.compilation.decorators import support_torch_compile
29
-from vllm.config import CacheConfig
29
+from vllm.config import CacheConfig, VllmConfig
30
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
31
get_tensor_model_parallel_world_size)
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -288,11 +288,13 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
288
289
def __init__(
290
self,
291
- config: JAISConfig,
292
- cache_config: Optional[CacheConfig] = None,
293
- quant_config: Optional[QuantizationConfig] = None,
291
+ vllm_config: VllmConfig,
292
+ prefix: str = "",
293
):
294
super().__init__()
295
+ config = vllm_config.model_config.hf_config
296
+ cache_config = vllm_config.cache_config
297
+ quant_config = vllm_config.quant_config
298
self.config = config
299
self.quant_config = quant_config
300
self.transformer = JAISModel(config, cache_config, quant_config)
vllm/model_executor/models/jamba.py
+8
-6
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
from vllm.attention.layer import Attention
10
-from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
12
from vllm.model_executor.layers.fused_moe import FusedMoE
13
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -350,12 +350,14 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
350
351
def __init__(
352
self,
353
- config: JambaConfig,
354
- cache_config: Optional[CacheConfig] = None,
355
- quant_config: Optional[QuantizationConfig] = None,
356
- lora_config: Optional[LoRAConfig] = None,
357
- scheduler_config: Optional[SchedulerConfig] = None,
353
+ vllm_config: VllmConfig,
354
+ prefix: str = "",
355
) -> None:
356
+ config = vllm_config.model_config.hf_config
357
+ cache_config = vllm_config.cache_config
358
+ quant_config = vllm_config.quant_config
359
+ lora_config = vllm_config.lora_config
360
+ scheduler_config = vllm_config.scheduler_config
361
assert not cache_config.enable_prefix_caching, \
362
"Jamba currently does not support prefix caching"
363
vllm/model_executor/models/llama.py
+20
-10
@@ -28,7 +28,7 @@ from transformers import LlamaConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
31
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -494,15 +494,15 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
494
495
def __init__(
496
self,
497
- config: LlamaConfig,
498
- cache_config: Optional[CacheConfig] = None,
499
- quant_config: Optional[QuantizationConfig] = None,
500
- lora_config: Optional[LoRAConfig] = None,
497
+ vllm_config: VllmConfig,
498
prefix: str = "",
502
- pooler_config: Optional[PoolerConfig] = None,
499
) -> None:
500
super().__init__()
505
-
501
+ config = vllm_config.model_config.hf_config
502
+ cache_config = vllm_config.cache_config
503
+ quant_config = vllm_config.quant_config
504
+ lora_config = vllm_config.lora_config
505
+ pooler_config = vllm_config.model_config.pooler_config
506
self.config = config
507
self.lora_config = lora_config
508
@@ -654,12 +654,22 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
654
655
def __init__(
656
self,
657
- pooler_config: Optional[PoolerConfig] = None,
658
- **kwargs,
657
+ vllm_config: VllmConfig,
658
+ prefix: str = "",
659
) -> None:
660
super().__init__()
661
662
- self.model = LlamaModel(**kwargs)
662
+ config = vllm_config.model_config.hf_config
663
+ cache_config = vllm_config.cache_config
664
+ quant_config = vllm_config.quant_config
665
+ lora_config = vllm_config.lora_config
666
+ pooler_config = vllm_config.model_config.pooler_config
667
+
668
+ self.model = LlamaModel(config,
669
+ cache_config,
670
+ quant_config,
671
+ lora_config,
672
+ prefix=maybe_prefix(prefix, "model"))
673
self._pooler = Pooler.from_config_with_defaults(
674
pooler_config,
675
pooling_type=PoolingType.LAST,
vllm/model_executor/models/llava.py
+7
-8
@@ -9,7 +9,7 @@ from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9
PretrainedConfig, SiglipVisionConfig)
10
11
from vllm.attention import AttentionMetadata
12
-from vllm.config import CacheConfig, MultiModalConfig
12
+from vllm.config import VllmConfig
13
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14
InputContext)
15
from vllm.model_executor.layers.activation import get_act_fn
@@ -258,13 +258,13 @@ def init_vision_tower_for_llava(
258
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
259
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
260
261
- def __init__(self,
262
- config: LlavaConfig,
263
- multimodal_config: MultiModalConfig,
264
- cache_config: Optional[CacheConfig] = None,
265
- quant_config: Optional[QuantizationConfig] = None) -> None:
261
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
262
super().__init__()
263
264
+ config = vllm_config.model_config.hf_config
265
+ quant_config = vllm_config.quant_config
266
+ multimodal_config = vllm_config.model_config.multimodal_config
267
+
268
self.config = config
269
self.multimodal_config = multimodal_config
270
@@ -290,8 +290,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
290
291
self.language_model = init_vllm_registered_model(
292
config.text_config,
293
- cache_config,
294
- quant_config,
293
+ vllm_config=vllm_config,
294
prefix="language_model")
295
296
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/llava_next.py
+7
-10
@@ -11,11 +11,10 @@ from transformers.models.llava_next.modeling_llava_next import (
11
from typing_extensions import NotRequired
12
13
from vllm.attention import AttentionMetadata
14
-from vllm.config import CacheConfig, MultiModalConfig, PoolerConfig
14
+from vllm.config import VllmConfig
15
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16
InputContext)
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
-from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19
from vllm.model_executor.pooling_metadata import PoolingMetadata
20
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -282,13 +281,12 @@ def input_processor_for_llava_next(ctx: InputContext,
281
class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
282
SupportsPP):
283
285
- def __init__(self,
286
- config: LlavaNextConfig,
287
- multimodal_config: MultiModalConfig,
288
- cache_config: Optional[CacheConfig] = None,
289
- quant_config: Optional[QuantizationConfig] = None,
290
- pooler_config: Optional[PoolerConfig] = None) -> None:
284
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
285
super().__init__()
286
+ config = vllm_config.model_config.hf_config
287
+ quant_config = vllm_config.quant_config
288
+ pooler_config = vllm_config.model_config.pooler_config
289
+ multimodal_config = vllm_config.model_config.multimodal_config
290
291
self.config = config
292
self.multimodal_config = multimodal_config
@@ -308,8 +306,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
306
307
self.language_model = init_vllm_registered_model(
308
config.text_config,
311
- cache_config,
312
- quant_config,
309
+ vllm_config=vllm_config,
310
prefix="language_model")
311
312
# The same model class supports both language generation and embedding
vllm/model_executor/models/llava_next_video.py
+6
-9
@@ -10,11 +10,10 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
10
SiglipVisionConfig)
11
12
from vllm.attention import AttentionMetadata
13
-from vllm.config import CacheConfig, MultiModalConfig
13
+from vllm.config import VllmConfig
14
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
InputContext, token_inputs)
16
from vllm.model_executor.layers.activation import get_act_fn
17
-from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.models.clip import CLIPVisionModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -254,12 +253,11 @@ class LlavaNextMultiModalProjector(nn.Module):
253
class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
254
SupportsPP):
255
257
- def __init__(self,
258
- config: LlavaNextVideoConfig,
259
- multimodal_config: MultiModalConfig,
260
- cache_config: Optional[CacheConfig] = None,
261
- quant_config: Optional[QuantizationConfig] = None) -> None:
256
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
257
super().__init__()
258
+ config = vllm_config.model_config.hf_config
259
+ quant_config = vllm_config.quant_config
260
+ multimodal_config = vllm_config.model_config.multimodal_config
261
262
self.config = config
263
self.multimodal_config = multimodal_config
@@ -277,8 +275,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
275
projector_hidden_act=config.projector_hidden_act)
276
self.language_model = init_vllm_registered_model(
277
config.text_config,
280
- cache_config,
281
- quant_config,
278
+ vllm_config=vllm_config,
279
prefix="language_model")
280
281
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/llava_onevision.py
+6
-9
@@ -14,11 +14,10 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
14
from typing_extensions import NotRequired
15
16
from vllm.attention import AttentionMetadata
17
-from vllm.config import CacheConfig, MultiModalConfig
17
+from vllm.config import VllmConfig
18
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19
InputContext, token_inputs)
20
from vllm.model_executor.layers.activation import get_act_fn
21
-from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -405,12 +404,11 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
404
class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
405
SupportsPP):
406
408
- def __init__(self,
409
- config: LlavaOnevisionConfig,
410
- multimodal_config: MultiModalConfig,
411
- cache_config: Optional[CacheConfig] = None,
412
- quant_config: Optional[QuantizationConfig] = None) -> None:
407
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
408
super().__init__()
409
+ config = vllm_config.model_config.hf_config
410
+ quant_config = vllm_config.quant_config
411
+ multimodal_config = vllm_config.model_config.multimodal_config
412
413
self.config = config
414
self.multimodal_config = multimodal_config
@@ -424,8 +422,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
422
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
423
self.language_model = init_vllm_registered_model(
424
config.text_config,
427
- cache_config,
428
- quant_config,
425
+ vllm_config=vllm_config,
426
prefix="language_model")
427
self.image_newline = nn.Parameter(
428
torch.empty(config.text_config.hidden_size))
vllm/model_executor/models/mamba.py
+8
-6
@@ -6,7 +6,7 @@ from torch import nn
6
from transformers import MambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
-from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -132,12 +132,14 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
132
133
def __init__(
134
self,
135
- config: MambaConfig,
136
- cache_config: Optional[CacheConfig] = None,
137
- quant_config: Optional[QuantizationConfig] = None,
138
- lora_config: Optional[LoRAConfig] = None,
139
- scheduler_config: Optional[SchedulerConfig] = None,
135
+ vllm_config: VllmConfig,
136
+ prefix: str = "",
137
) -> None:
138
+ config = vllm_config.model_config.hf_config
139
+ cache_config = vllm_config.cache_config
140
+ quant_config = vllm_config.quant_config
141
+ lora_config = vllm_config.lora_config
142
+ scheduler_config = vllm_config.scheduler_config
143
assert not cache_config.enable_prefix_caching, \
144
"Mamba does not support prefix caching"
145
vllm/model_executor/models/medusa.py
+3
-2
@@ -3,13 +3,13 @@ from typing import Iterable, List, Optional, Tuple
3
import torch
4
import torch.nn as nn
5
6
+from vllm.config import VllmConfig
7
from vllm.model_executor.layers.logits_processor import LogitsProcessor
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.model_executor.layers.vocab_parallel_embedding import (
10
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
11
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
12
from vllm.model_executor.sampling_metadata import SamplingMetadata
12
-from vllm.transformers_utils.configs.medusa import MedusaConfig
13
14
15
class ResidualBlock(nn.Module):
@@ -44,7 +44,8 @@ class Medusa(nn.Module):
44
in the draft checkpoint (using key token_map). Also, the draft config
45
needs to have truncated_vocab_size (=k) as an attribute."""
46
47
- def __init__(self, config: MedusaConfig, **_) -> None:
47
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
48
+ config = vllm_config.model_config.hf_config
49
super().__init__()
50
self.config = config
51
self.blocks = nn.ModuleList([
vllm/model_executor/models/minicpm.py
+7
-5
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
@@ -463,12 +463,14 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
463
464
def __init__(
465
self,
466
- config: PretrainedConfig,
467
- cache_config: Optional[CacheConfig] = None,
468
- quant_config: Optional[QuantizationConfig] = None,
469
- lora_config: Optional[LoRAConfig] = None,
466
+ vllm_config: VllmConfig,
467
+ prefix: str = "",
468
) -> None:
469
super().__init__()
470
+ config = vllm_config.model_config.hf_config
471
+ cache_config = vllm_config.cache_config
472
+ quant_config = vllm_config.quant_config
473
+ lora_config = vllm_config.lora_config
474
475
self.config = config
476
self.lora_config = lora_config
vllm/model_executor/models/minicpmv.py
+19
-29
@@ -34,7 +34,7 @@ from transformers import PretrainedConfig
34
from typing_extensions import NotRequired
35
36
from vllm.attention import AttentionMetadata
37
-from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
37
+from vllm.config import CacheConfig, VllmConfig
38
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39
InputContext, token_inputs)
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -385,11 +385,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
385
386
def __init__(
387
self,
388
- config: PretrainedConfig,
389
- multimodal_config: MultiModalConfig,
390
- cache_config: Optional[CacheConfig] = None,
391
- quant_config: Optional[QuantizationConfig] = None,
388
+ vllm_config: VllmConfig,
389
+ prefix: str = "",
390
):
391
+ config = vllm_config.model_config.hf_config
392
+ multimodal_config = vllm_config.model_config.multimodal_config
393
+ cache_config = vllm_config.cache_config
394
+ quant_config = vllm_config.quant_config
395
super().__init__()
396
# All MiniCPM-V models disable `tie_word_embeddings` but
397
# `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
@@ -701,12 +703,10 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
703
704
def __init__(
705
self,
704
- config: PretrainedConfig,
705
- multimodal_config: MultiModalConfig,
706
- cache_config: Optional[CacheConfig] = None,
707
- quant_config: Optional[QuantizationConfig] = None,
706
+ vllm_config: VllmConfig,
707
+ prefix: str = "",
708
):
709
- super().__init__(config, multimodal_config, cache_config, quant_config)
709
+ super().__init__(vllm_config)
710
assert self.version == (2, 0)
711
712
def init_llm(
@@ -867,13 +867,10 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
867
868
def __init__(
869
self,
870
- config: PretrainedConfig,
871
- multimodal_config: MultiModalConfig,
872
- cache_config: Optional[CacheConfig] = None,
873
- quant_config: Optional[QuantizationConfig] = None,
874
- lora_config: Optional[LoRAConfig] = None,
870
+ vllm_config: VllmConfig,
871
+ prefix: str = "",
872
):
876
- super().__init__(config, multimodal_config, cache_config, quant_config)
873
+ super().__init__(vllm_config)
874
assert self.version == (2, 5)
875
876
def init_llm(
@@ -1017,12 +1014,10 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1014
1015
def __init__(
1016
self,
1020
- config: PretrainedConfig,
1021
- multimodal_config: MultiModalConfig,
1022
- cache_config: Optional[CacheConfig] = None,
1023
- quant_config: Optional[QuantizationConfig] = None,
1017
+ vllm_config: VllmConfig,
1018
+ prefix: str = "",
1019
):
1025
- super().__init__(config, multimodal_config, cache_config, quant_config)
1020
+ super().__init__(vllm_config)
1021
assert self.version == (2, 6)
1022
1023
def init_llm(
@@ -1141,12 +1136,8 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1136
embedding_modules = {}
1137
embedding_padding_modules = []
1138
1144
- def __new__(cls,
1145
- config: PretrainedConfig,
1146
- multimodal_config: MultiModalConfig,
1147
- cache_config: Optional[CacheConfig] = None,
1148
- quant_config: Optional[QuantizationConfig] = None,
1149
- lora_config: Optional[LoRAConfig] = None):
1139
+ def __new__(cls, vllm_config: VllmConfig, prefix: str = ""):
1140
+ config = vllm_config.model_config.hf_config
1141
if not hasattr(config, "version"):
1142
if config.hidden_size == 2304 and config.query_num == 64:
1143
version = (2, 0)
@@ -1160,5 +1151,4 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1151
if instance_class is None:
1152
raise ValueError(
1153
"Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1163
- return instance_class(config, multimodal_config, cache_config,
1164
- quant_config)
1154
+ return instance_class(vllm_config, prefix=prefix)
vllm/model_executor/models/mixtral.py
+7
-6
@@ -28,7 +28,7 @@ from transformers import MixtralConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -334,13 +334,14 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
334
335
def __init__(
336
self,
337
- config: MixtralConfig,
338
- cache_config: Optional[CacheConfig] = None,
339
- quant_config: Optional[QuantizationConfig] = None,
340
- lora_config: Optional[LoRAConfig] = None,
337
+ vllm_config: VllmConfig,
338
+ prefix: str = "",
339
) -> None:
340
super().__init__()
343
-
341
+ config = vllm_config.model_config.hf_config
342
+ cache_config = vllm_config.cache_config
343
+ quant_config = vllm_config.quant_config
344
+ lora_config = vllm_config.lora_config
345
self.config = config
346
self.lora_config = lora_config
347
vllm/model_executor/models/mixtral_quant.py
+6
-4
@@ -29,7 +29,7 @@ from torch import nn
29
from transformers import MixtralConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
-from vllm.config import CacheConfig
32
+from vllm.config import CacheConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
@@ -352,11 +352,13 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
352
353
def __init__(
354
self,
355
- config: MixtralConfig,
356
- cache_config: Optional[CacheConfig] = None,
357
- quant_config: Optional[QuantizationConfig] = None,
355
+ vllm_config: VllmConfig,
356
+ prefix: str = "",
357
) -> None:
358
super().__init__()
359
+ config = vllm_config.model_config.hf_config
360
+ cache_config = vllm_config.cache_config
361
+ quant_config = vllm_config.quant_config
362
self.config = config
363
self.quant_config = quant_config
364
self.model = MixtralModel(config, cache_config, quant_config)
vllm/model_executor/models/mllama.py
+9
-6
@@ -33,7 +33,7 @@ from transformers.models.mllama.processing_mllama import (
33
import vllm.distributed.parallel_state as ps
34
from vllm.attention import Attention, AttentionMetadata, AttentionType
35
from vllm.attention.ops.paged_attn import PagedAttention
36
-from vllm.config import CacheConfig, MultiModalConfig
36
+from vllm.config import CacheConfig, VllmConfig
37
from vllm.distributed import get_tensor_model_parallel_world_size
38
from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
39
InputContext, TokenInputs, token_inputs)
@@ -1108,12 +1108,15 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1108
"up_proj": ("gate_up_proj", 1),
1109
}
1110
1111
- def __init__(self,
1112
- config: config_mllama.MllamaConfig,
1113
- multimodal_config: MultiModalConfig,
1114
- cache_config: Optional[CacheConfig] = None,
1115
- quant_config: Optional[QuantizationConfig] = None):
1111
+ def __init__(
1112
+ self,
1113
+ vllm_config: VllmConfig,
1114
+ prefix: str = "",
1115
+ ) -> None:
1116
super().__init__()
1117
+ config = vllm_config.model_config.hf_config
1118
+ cache_config = vllm_config.cache_config
1119
+ quant_config = vllm_config.quant_config
1120
self.vocab_size = config.text_config.vocab_size
1121
self.hidden_size = config.text_config.hidden_size
1122
self.max_num_tiles = config.vision_config.max_num_tiles
vllm/model_executor/models/molmo.py
+8
-8
@@ -3,8 +3,7 @@ import re
3
from array import array
4
from dataclasses import dataclass
5
from functools import lru_cache, partial
6
-from typing import (Any, Iterable, List, Mapping, Optional, Tuple, TypedDict,
7
- Union)
6
+from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
7
8
import torch
9
from einops import rearrange
@@ -16,7 +15,7 @@ from transformers import PretrainedConfig
15
from vllm.attention import Attention, AttentionMetadata
16
from vllm.attention.selector import _Backend
17
from vllm.compilation.decorators import support_torch_compile
19
-from vllm.config import CacheConfig, MultiModalConfig
18
+from vllm.config import CacheConfig, VllmConfig
19
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
20
get_tensor_model_parallel_world_size,
21
split_tensor_along_last_dim,
@@ -1027,13 +1026,14 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1026
1027
def __init__(
1028
self,
1030
- config: PretrainedConfig,
1031
- multimodal_config: Optional[MultiModalConfig] = None,
1032
- cache_config: Optional[CacheConfig] = None,
1033
- quant_config: Optional[Mapping[str, Any]] = None,
1029
+ vllm_config: VllmConfig,
1030
+ prefix: str = "",
1031
) -> None:
1032
super().__init__()
1036
-
1033
+ config = vllm_config.model_config.hf_config
1034
+ cache_config = vllm_config.cache_config
1035
+ quant_config = vllm_config.quant_config
1036
+ multimodal_config = vllm_config.model_config.multimodal_config
1037
self.config = config
1038
self.multimodal_config = multimodal_config
1039
vllm/model_executor/models/mpt.py
+7
-5
@@ -7,7 +7,7 @@ import torch.nn as nn
7
8
from vllm.attention import Attention, AttentionMetadata
9
from vllm.compilation.decorators import support_torch_compile
10
-from vllm.config import CacheConfig
10
+from vllm.config import CacheConfig, VllmConfig
11
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
12
get_tensor_model_parallel_world_size)
13
from vllm.model_executor.layers.activation import get_act_fn
@@ -269,11 +269,13 @@ class MPTForCausalLM(nn.Module, SupportsPP):
269
270
def __init__(
271
self,
272
- config: MPTConfig,
273
- cache_config: Optional[CacheConfig] = None,
274
- quant_config: Optional[QuantizationConfig] = None,
275
- ):
272
+ vllm_config: VllmConfig,
273
+ prefix: str = "",
274
+ ) -> None:
275
super().__init__()
276
+ config = vllm_config.model_config.hf_config
277
+ cache_config = vllm_config.cache_config
278
+ quant_config = vllm_config.quant_config
279
self.config = config
280
assert config.tie_word_embeddings
281
self.quant_config = quant_config
vllm/model_executor/models/nemotron.py
+7
-6
@@ -27,7 +27,7 @@ from torch import nn
27
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.compilation.decorators import support_torch_compile
30
-from vllm.config import CacheConfig, LoRAConfig
30
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -403,13 +403,14 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
403
404
def __init__(
405
self,
406
- config: NemotronConfig,
407
- cache_config: Optional[CacheConfig] = None,
408
- quant_config: Optional[QuantizationConfig] = None,
409
- lora_config: Optional[LoRAConfig] = None,
406
+ vllm_config: VllmConfig,
407
+ prefix: str = "",
408
) -> None:
409
super().__init__()
412
-
410
+ config = vllm_config.model_config.hf_config
411
+ cache_config = vllm_config.cache_config
412
+ quant_config = vllm_config.quant_config
413
+ lora_config = vllm_config.lora_config
414
assert isinstance(config, NemotronConfig)
415
416
self.config = config
vllm/model_executor/models/olmo.py
+9
-5
@@ -28,7 +28,7 @@ from transformers import OlmoConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig
31
+from vllm.config import CacheConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -291,11 +291,15 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
291
Extremely barebones HF model wrapper.
292
"""
293
294
- def __init__(self,
295
- config: OlmoConfig,
296
- cache_config: Optional[CacheConfig] = None,
297
- quant_config: Optional[QuantizationConfig] = None):
294
+ def __init__(
295
+ self,
296
+ vllm_config: VllmConfig,
297
+ prefix: str = "",
298
+ ) -> None:
299
super().__init__()
300
+ config = vllm_config.model_config.hf_config
301
+ cache_config = vllm_config.cache_config
302
+ quant_config = vllm_config.quant_config
303
self.config = config
304
self.model = OlmoModel(config, cache_config, quant_config)
305
if config.tie_word_embeddings:
vllm/model_executor/models/olmoe.py
+6
-4
@@ -18,7 +18,7 @@ from transformers import PretrainedConfig
18
19
from vllm.attention import Attention, AttentionMetadata
20
from vllm.compilation.decorators import support_torch_compile
21
-from vllm.config import CacheConfig
21
+from vllm.config import CacheConfig, VllmConfig
22
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
23
from vllm.model_executor.layers.fused_moe import FusedMoE
24
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -311,11 +311,13 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
311
312
def __init__(
313
self,
314
- config: PretrainedConfig,
315
- cache_config: Optional[CacheConfig] = None,
316
- quant_config: Optional[QuantizationConfig] = None,
314
+ vllm_config: VllmConfig,
315
+ prefix: str = "",
316
) -> None:
317
super().__init__()
318
+ config = vllm_config.model_config.hf_config
319
+ cache_config = vllm_config.cache_config
320
+ quant_config = vllm_config.quant_config
321
self.config = config
322
self.quant_config = quant_config
323
self.model = OlmoeModel(config, cache_config, quant_config)
vllm/model_executor/models/opt.py
+7
-5
@@ -24,7 +24,7 @@ from transformers import OPTConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
from vllm.compilation.decorators import support_torch_compile
27
-from vllm.config import CacheConfig
27
+from vllm.config import CacheConfig, VllmConfig
28
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -344,11 +344,13 @@ class OPTForCausalLM(nn.Module, SupportsPP):
344
345
def __init__(
346
self,
347
- config: OPTConfig,
348
- cache_config: Optional[CacheConfig] = None,
349
- quant_config: Optional[QuantizationConfig] = None,
347
+ vllm_config: VllmConfig,
348
prefix: str = "",
351
- ):
349
+ ) -> None:
350
+ super().__init__()
351
+ config = vllm_config.model_config.hf_config
352
+ cache_config = vllm_config.cache_config
353
+ quant_config = vllm_config.quant_config
354
super().__init__()
355
self.config = config
356
self.quant_config = quant_config
vllm/model_executor/models/orion.py
+6
-4
@@ -11,7 +11,7 @@ from transformers import PretrainedConfig
11
12
from vllm.attention import Attention, AttentionMetadata
13
from vllm.compilation.decorators import support_torch_compile
14
-from vllm.config import CacheConfig
14
+from vllm.config import CacheConfig, VllmConfig
15
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16
from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -270,11 +270,13 @@ class OrionForCausalLM(nn.Module, SupportsPP):
270
271
def __init__(
272
self,
273
- config: PretrainedConfig,
274
- cache_config: Optional[CacheConfig] = None,
275
- quant_config: Optional[QuantizationConfig] = None,
273
+ vllm_config: VllmConfig,
274
+ prefix: str = "",
275
) -> None:
276
super().__init__()
277
+ config = vllm_config.model_config.hf_config
278
+ cache_config = vllm_config.cache_config
279
+ quant_config = vllm_config.quant_config
280
self.config = config
281
self.quant_config = quant_config
282
self.model = OrionModel(config, cache_config, quant_config)
vllm/model_executor/models/paligemma.py
+16
-14
@@ -6,13 +6,11 @@ from torch import nn
6
from transformers import PaliGemmaConfig
7
8
from vllm.attention import AttentionMetadata
9
-from vllm.config import CacheConfig, MultiModalConfig
9
+from vllm.config import VllmConfig
10
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
11
InputContext, token_inputs)
12
from vllm.logger import init_logger
13
-from vllm.model_executor.layers.quantization import QuantizationConfig
13
from vllm.model_executor.layers.sampler import SamplerOutput
15
-from vllm.model_executor.models.gemma import GemmaForCausalLM
14
from vllm.model_executor.sampling_metadata import SamplingMetadata
15
from vllm.multimodal import MULTIMODAL_REGISTRY
16
from vllm.multimodal.utils import cached_get_tokenizer
@@ -21,7 +19,8 @@ from vllm.sequence import IntermediateTensors
19
from .interfaces import SupportsMultiModal, SupportsPP
20
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
21
dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
24
-from .utils import AutoWeightsLoader, merge_multimodal_embeddings
22
+from .utils import (AutoWeightsLoader, init_vllm_registered_model,
23
+ merge_multimodal_embeddings)
24
25
logger = init_logger(__name__)
26
@@ -132,13 +131,15 @@ class PaliGemmaMultiModalProjector(nn.Module):
131
class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
132
SupportsPP):
133
135
- def __init__(self,
136
- config: PaliGemmaConfig,
137
- multimodal_config: MultiModalConfig,
138
- cache_config: Optional[CacheConfig] = None,
139
- quant_config: Optional[QuantizationConfig] = None) -> None:
134
+ def __init__(
135
+ self,
136
+ vllm_config: VllmConfig,
137
+ prefix: str = "",
138
+ ) -> None:
139
super().__init__()
141
-
140
+ config = vllm_config.model_config.hf_config
141
+ quant_config = vllm_config.quant_config
142
+ multimodal_config = vllm_config.model_config.multimodal_config
143
self.config = config
144
self.multimodal_config = multimodal_config
145
@@ -150,10 +151,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
151
projection_dim=config.vision_config.projection_dim)
152
153
self.quant_config = quant_config
153
- self.language_model = GemmaForCausalLM(config.text_config,
154
- cache_config,
155
- quant_config,
156
- prefix="language_model")
154
+ config.text_config.architectures = ["GemmaForCausalLM"]
155
+ self.language_model = init_vllm_registered_model(
156
+ config.text_config,
157
+ vllm_config=vllm_config,
158
+ prefix="language_model")
159
logit_scale = getattr(config, "logit_scale", 1.0)
160
self.language_model.logits_processor.scale *= logit_scale
161
vllm/model_executor/models/persimmon.py
+9
-5
@@ -27,7 +27,7 @@ from transformers import PersimmonConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.compilation.decorators import support_torch_compile
30
-from vllm.config import CacheConfig
30
+from vllm.config import CacheConfig, VllmConfig
31
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -265,11 +265,15 @@ class PersimmonModel(nn.Module):
265
266
class PersimmonForCausalLM(nn.Module, SupportsPP):
267
268
- def __init__(self,
269
- config: PersimmonConfig,
270
- cache_config: Optional[CacheConfig] = None,
271
- quant_config: Optional[QuantizationConfig] = None):
268
+ def __init__(
269
+ self,
270
+ vllm_config: VllmConfig,
271
+ prefix: str = "",
272
+ ) -> None:
273
super().__init__()
274
+ config = vllm_config.model_config.hf_config
275
+ cache_config = vllm_config.cache_config
276
+ quant_config = vllm_config.quant_config
277
self.config = config
278
self.vocab_size = config.vocab_size
279
self.model = PersimmonModel(config,
vllm/model_executor/models/phi.py
+8
-7
@@ -42,7 +42,7 @@ from transformers import PhiConfig
42
43
from vllm.attention import Attention, AttentionMetadata
44
from vllm.compilation.decorators import support_torch_compile
45
-from vllm.config import CacheConfig, LoRAConfig
45
+from vllm.config import CacheConfig, VllmConfig
46
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
47
from vllm.model_executor.layers.activation import get_act_fn
48
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -279,13 +279,14 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
279
280
def __init__(
281
self,
282
- config: PhiConfig,
283
- cache_config: Optional[CacheConfig] = None,
284
- quant_config: Optional[QuantizationConfig] = None,
285
- lora_config: Optional[LoRAConfig] = None,
286
- ):
282
+ vllm_config: VllmConfig,
283
+ prefix: str = "",
284
+ ) -> None:
285
super().__init__()
288
-
286
+ config = vllm_config.model_config.hf_config
287
+ cache_config = vllm_config.cache_config
288
+ quant_config = vllm_config.quant_config
289
+ lora_config = vllm_config.lora_config
290
self.config = config
291
# lm_head use bias, cannot share word embeddings
292
assert not config.tie_word_embeddings
vllm/model_executor/models/phi3_small.py
+7
-6
@@ -6,7 +6,7 @@ from torch import nn
6
from transformers.configuration_utils import PretrainedConfig
7
8
from vllm.attention import Attention, AttentionMetadata
9
-from vllm.config import CacheConfig, LoRAConfig
9
+from vllm.config import CacheConfig, VllmConfig
10
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size)
12
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -365,12 +365,13 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
365
366
def __init__(
367
self,
368
- config: PretrainedConfig,
369
- cache_config: Optional[CacheConfig] = None,
370
- quant_config: Optional[QuantizationConfig] = None,
371
- lora_config: Optional[LoRAConfig] = None,
372
- ):
368
+ vllm_config: VllmConfig,
369
+ prefix: str = "",
370
+ ) -> None:
371
super().__init__()
372
+ config = vllm_config.model_config.hf_config
373
+ cache_config = vllm_config.cache_config
374
+ quant_config = vllm_config.quant_config
375
self.config = config
376
self.quant_config = quant_config
377
self.model = Phi3SmallModel(config, cache_config, quant_config)
vllm/model_executor/models/phi3v.py
+12
-11
@@ -25,8 +25,7 @@ from PIL import Image
25
from transformers import CLIPVisionConfig, PretrainedConfig
26
27
from vllm.attention import AttentionMetadata
28
-from vllm.config import (CacheConfig, ModelConfig, MultiModalConfig,
29
- PoolerConfig)
28
+from vllm.config import ModelConfig, VllmConfig
29
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30
InputContext, token_inputs)
31
from vllm.logger import init_logger
@@ -526,14 +525,16 @@ def input_processor_for_phi3v(ctx: InputContext,
525
@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
526
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
527
529
- def __init__(self,
530
- config: PretrainedConfig,
531
- multimodal_config: MultiModalConfig,
532
- cache_config: Optional[CacheConfig] = None,
533
- quant_config: Optional[QuantizationConfig] = None,
534
- pooler_config: Optional[PoolerConfig] = None) -> None:
528
+ def __init__(
529
+ self,
530
+ vllm_config: VllmConfig,
531
+ prefix: str = "",
532
+ ) -> None:
533
super().__init__()
536
-
534
+ config = vllm_config.model_config.hf_config
535
+ quant_config = vllm_config.quant_config
536
+ pooler_config = vllm_config.model_config.pooler_config
537
+ multimodal_config = vllm_config.model_config.multimodal_config
538
self.config = config
539
self.multimodal_config = multimodal_config
540
self.image_token_id = _IMAGE_TOKEN_ID
@@ -552,8 +553,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
553
554
# The prefix is empty intentionally because default prefix of
555
# LlamaForCausalLM is "model"
555
- self.language_model = LlamaForCausalLM(config, cache_config,
556
- quant_config)
556
+ self.language_model = LlamaForCausalLM(vllm_config=vllm_config,
557
+ prefix="")
558
559
# The same model class supports both language generation and embedding
560
# because the architecture name is the same
vllm/model_executor/models/phimoe.py
+7
-6
@@ -28,7 +28,7 @@ from transformers.configuration_utils import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
-from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -531,13 +531,14 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
531
532
def __init__(
533
self,
534
- config: PhiMoEConfig,
535
- cache_config: Optional[CacheConfig] = None,
536
- quant_config: Optional[QuantizationConfig] = None,
537
- lora_config: Optional[LoRAConfig] = None,
534
+ vllm_config: VllmConfig,
535
+ prefix: str = "",
536
) -> None:
537
super().__init__()
540
-
538
+ config = vllm_config.model_config.hf_config
539
+ cache_config = vllm_config.cache_config
540
+ quant_config = vllm_config.quant_config
541
+ lora_config = vllm_config.lora_config
542
self.config = config
543
self.lora_config = lora_config
544
vllm/model_executor/models/pixtral.py
+10
-10
@@ -9,14 +9,14 @@ import torch.nn as nn
9
import torch.nn.functional as F
10
from mistral_common.protocol.instruct.messages import ImageChunk
11
from PIL import Image
12
-from transformers import PixtralVisionConfig, PretrainedConfig
12
+from transformers import PixtralVisionConfig
13
from transformers.models.pixtral.image_processing_pixtral import (
14
_num_image_tokens)
15
from transformers.models.pixtral.modeling_pixtral import (
16
PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
17
18
from vllm.attention import AttentionMetadata
19
-from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
19
+from vllm.config import ModelConfig, VllmConfig
20
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
InputContext, token_inputs)
22
from vllm.model_executor.layers.activation import get_act_and_mul_fn
@@ -152,13 +152,14 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
152
class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
153
SupportsPP):
154
155
- def __init__(self,
156
- config: PretrainedConfig,
157
- multimodal_config: MultiModalConfig,
158
- cache_config: Optional[CacheConfig] = None,
159
- quant_config: Optional[QuantizationConfig] = None) -> None:
155
+ def __init__(
156
+ self,
157
+ vllm_config: VllmConfig,
158
+ prefix: str = "",
159
+ ) -> None:
160
super().__init__()
161
-
161
+ config = vllm_config.model_config.hf_config
162
+ multimodal_config = vllm_config.model_config.multimodal_config
163
self.config = config
164
self.multimodal_config = multimodal_config
165
@@ -174,8 +175,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
175
# init MistralForCausalLM
176
self.language_model = init_vllm_registered_model(
177
config.text_config,
177
- cache_config,
178
- quant_config,
178
+ vllm_config=vllm_config,
179
prefix="language_model")
180
181
self.vision_encoder = VisionTransformer(self.vision_args)
vllm/model_executor/models/qwen.py
+14
-17
@@ -20,7 +20,7 @@ from transformers import PretrainedConfig
20
21
from vllm.attention import Attention, AttentionMetadata
22
from vllm.compilation.decorators import support_torch_compile
23
-from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
23
+from vllm.config import CacheConfig, VllmConfig
24
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
26
InputContext, token_inputs)
@@ -867,13 +867,14 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
867
868
def __init__(
869
self,
870
- config: PretrainedConfig,
871
- multimodal_config: MultiModalConfig,
872
- cache_config: Optional[CacheConfig] = None,
873
- quant_config: Optional[QuantizationConfig] = None,
874
- lora_config: Optional[LoRAConfig] = None,
875
- ):
870
+ vllm_config: VllmConfig,
871
+ prefix: str = "",
872
+ ) -> None:
873
super().__init__()
874
+ config = vllm_config.model_config.hf_config
875
+ cache_config = vllm_config.cache_config
876
+ quant_config = vllm_config.quant_config
877
+ multimodal_config = vllm_config.model_config.multimodal_config
878
self.config = config
879
self.multimodal_config = multimodal_config
880
self.quant_config = quant_config
@@ -1064,17 +1065,13 @@ class QWenLMHeadModel(QWenBaseModel, SupportsLoRA):
1065
1066
def __new__(
1067
cls,
1067
- config: PretrainedConfig,
1068
- multimodal_config: MultiModalConfig,
1069
- cache_config: Optional[CacheConfig] = None,
1070
- quant_config: Optional[QuantizationConfig] = None,
1071
- lora_config: Optional[LoRAConfig] = None,
1072
- ):
1068
+ vllm_config: VllmConfig,
1069
+ prefix: str = "",
1070
+ ) -> None:
1071
+ config = vllm_config.model_config.hf_config
1072
# Initialize VL
1073
if hasattr(config, "visual"):
1075
- return QWenVL(config, multimodal_config, cache_config,
1076
- quant_config, lora_config)
1074
+ return QWenVL(vllm_config)
1075
# Initialize LLM
1076
else:
1079
- return QWenLLM(config, multimodal_config, cache_config,
1080
- quant_config, lora_config)
1077
+ return QWenLLM(vllm_config)
vllm/model_executor/models/qwen2.py
+7
-7
@@ -29,7 +29,7 @@ from transformers import Qwen2Config
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.config import CacheConfig, VllmConfig
33
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -405,12 +405,14 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
405
406
def __init__(
407
self,
408
- config: Qwen2Config,
409
- cache_config: Optional[CacheConfig] = None,
410
- quant_config: Optional[QuantizationConfig] = None,
411
- lora_config: Optional[LoRAConfig] = None,
408
+ vllm_config: VllmConfig,
409
prefix: str = "",
410
) -> None:
411
+ super().__init__()
412
+ config = vllm_config.model_config.hf_config
413
+ cache_config = vllm_config.cache_config
414
+ quant_config = vllm_config.quant_config
415
+ lora_config = vllm_config.lora_config
416
# TODO (@robertgshaw2): see if this can be moved out
417
if (cache_config.sliding_window is not None
418
and hasattr(config, "max_window_layers")):
@@ -423,8 +425,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
425
config.num_hidden_layers,
426
))
427
426
- super().__init__()
427
-
428
self.config = config
429
self.lora_config = lora_config
430
vllm/model_executor/models/qwen2_audio.py
+11
-10
@@ -26,16 +26,14 @@ import librosa
26
import numpy as np
27
import torch
28
import torch.nn as nn
29
-from transformers import Qwen2AudioConfig, Qwen2AudioEncoder
29
+from transformers import Qwen2AudioEncoder
30
31
from vllm.attention import AttentionMetadata
32
-from vllm.config import CacheConfig, MultiModalConfig
32
+from vllm.config import VllmConfig
33
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
34
InputContext, token_inputs)
35
from vllm.logger import init_logger
36
from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
-from vllm.model_executor.layers.quantization.base_config import (
38
- QuantizationConfig)
37
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
39
from vllm.model_executor.model_loader.weight_utils import (
@@ -266,13 +264,16 @@ def input_mapper_for_qwen2_audio(
264
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
265
SupportsPP):
266
269
- def __init__(self,
270
- config: Qwen2AudioConfig,
271
- multimodal_config: MultiModalConfig,
272
- cache_config: Optional[CacheConfig] = None,
273
- quant_config: Optional[QuantizationConfig] = None) -> None:
267
+ def __init__(
268
+ self,
269
+ vllm_config: VllmConfig,
270
+ prefix: str = "",
271
+ ) -> None:
272
super().__init__()
275
-
273
+ config = vllm_config.model_config.hf_config
274
+ cache_config = vllm_config.cache_config
275
+ quant_config = vllm_config.quant_config
276
+ multimodal_config = vllm_config.model_config.multimodal_config
277
self.config = config
278
self.multimodal_config = multimodal_config
279
vllm/model_executor/models/qwen2_cls.py
+9
-11
@@ -8,14 +8,11 @@ from typing import Iterable, List, Optional, Tuple
8
9
import torch
10
from torch import nn
11
-from transformers import Qwen2Config
11
12
from vllm.attention import AttentionMetadata
14
-from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
13
+from vllm.config import VllmConfig
14
from vllm.model_executor.layers.linear import RowParallelLinear
15
from vllm.model_executor.layers.pooler import Pooler, PoolingType
17
-from vllm.model_executor.layers.quantization.base_config import (
18
- QuantizationConfig)
16
from vllm.model_executor.models.qwen2 import Qwen2Model
17
from vllm.model_executor.pooling_metadata import PoolingMetadata
18
from vllm.sequence import IntermediateTensors, PoolerOutput
@@ -48,12 +45,15 @@ class Qwen2ForSequenceClassification(nn.Module):
45
46
def __init__(
47
self,
51
- config: Qwen2Config,
52
- cache_config: Optional[CacheConfig] = None,
53
- quant_config: Optional[QuantizationConfig] = None,
54
- lora_config: Optional[LoRAConfig] = None,
55
- pooler_config: Optional[PoolerConfig] = None,
48
+ vllm_config: VllmConfig,
49
+ prefix: str = "",
50
) -> None:
51
+ super().__init__()
52
+ config = vllm_config.model_config.hf_config
53
+ cache_config = vllm_config.cache_config
54
+ quant_config = vllm_config.quant_config
55
+ lora_config = vllm_config.lora_config
56
+ pooler_config = vllm_config.model_config.pooler_config
57
# TODO (@robertgshaw2): see if this can be moved out
58
if (cache_config.sliding_window is not None
59
and hasattr(config, "max_window_layers")):
@@ -66,8 +66,6 @@ class Qwen2ForSequenceClassification(nn.Module):
66
config.num_hidden_layers,
67
))
68
69
- super().__init__()
70
-
69
self.config = config
70
self.lora_config = lora_config
71
vllm/model_executor/models/qwen2_moe.py
+6
-4
@@ -30,7 +30,7 @@ from transformers import PretrainedConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
from vllm.compilation.decorators import support_torch_compile
33
-from vllm.config import CacheConfig
33
+from vllm.config import CacheConfig, VllmConfig
34
from vllm.distributed import (get_pp_group,
35
get_tensor_model_parallel_world_size,
36
tensor_model_parallel_all_reduce)
@@ -379,11 +379,13 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
379
380
def __init__(
381
self,
382
- config: PretrainedConfig,
383
- cache_config: Optional[CacheConfig] = None,
384
- quant_config: Optional[QuantizationConfig] = None,
382
+ vllm_config: VllmConfig,
383
+ prefix: str = "",
384
) -> None:
385
super().__init__()
386
+ config = vllm_config.model_config.hf_config
387
+ cache_config = vllm_config.cache_config
388
+ quant_config = vllm_config.quant_config
389
self.config = config
390
self.quant_config = quant_config
391
self.model = Qwen2MoeModel(config, cache_config, quant_config)
vllm/model_executor/models/qwen2_rm.py
+9
-10
@@ -7,14 +7,12 @@ from typing import Iterable, List, Optional, Tuple, Union
7
8
import torch
9
from torch import nn
10
-from transformers import Qwen2Config
10
11
from vllm.attention import AttentionMetadata
13
-from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
12
+from vllm.config import VllmConfig
13
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
RowParallelLinear)
15
from vllm.model_executor.layers.pooler import Pooler, PoolingType
17
-from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.pooling_metadata import PoolingMetadata
17
from vllm.sequence import IntermediateTensors, PoolerOutput
18
@@ -59,12 +57,15 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
57
58
def __init__(
59
self,
62
- config: Qwen2Config,
63
- cache_config: Optional[CacheConfig] = None,
64
- quant_config: Optional[QuantizationConfig] = None,
65
- lora_config: Optional[LoRAConfig] = None,
66
- pooler_config: Optional[PoolerConfig] = None,
60
+ vllm_config: VllmConfig,
61
+ prefix: str = "",
62
) -> None:
63
+ super().__init__()
64
+ config = vllm_config.model_config.hf_config
65
+ cache_config = vllm_config.cache_config
66
+ quant_config = vllm_config.quant_config
67
+ lora_config = vllm_config.lora_config
68
+ pooler_config = vllm_config.model_config.pooler_config
69
# TODO (@robertgshaw2): see if this can be moved out
70
if (cache_config.sliding_window is not None
71
and hasattr(config, "max_window_layers")):
@@ -77,8 +78,6 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
78
config.num_hidden_layers,
79
))
80
80
- super().__init__()
81
-
81
self.config = config
82
self.lora_config = lora_config
83
vllm/model_executor/models/qwen2_vl.py
+10
-9
@@ -40,7 +40,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
41
from vllm.attention import AttentionMetadata
42
from vllm.attention.selector import _Backend
43
-from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
43
+from vllm.config import VllmConfig
44
from vllm.distributed import get_pp_group, parallel_state
45
from vllm.distributed import utils as dist_utils
46
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
@@ -966,15 +966,16 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
966
embedding_modules = {}
967
embedding_padding_modules = []
968
969
- def __init__(self,
970
- config: Qwen2VLConfig,
971
- multimodal_config: MultiModalConfig,
972
- cache_config: Optional[CacheConfig] = None,
973
- quant_config: Optional[QuantizationConfig] = None,
974
- lora_config: Optional[LoRAConfig] = None) -> None:
975
-
969
+ def __init__(
970
+ self,
971
+ vllm_config: VllmConfig,
972
+ prefix: str = "",
973
+ ) -> None:
974
super().__init__()
977
-
975
+ config = vllm_config.model_config.hf_config
976
+ cache_config = vllm_config.cache_config
977
+ quant_config = vllm_config.quant_config
978
+ multimodal_config = vllm_config.model_config.multimodal_config
979
assert not cache_config.enable_prefix_caching, \
980
"Qwen2-VL currently does not support prefix caching"
981
vllm/model_executor/models/solar.py
+7
-6
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.config import CacheConfig, LoRAConfig, VllmConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -411,13 +411,14 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411
412
def __init__(
413
self,
414
- config: PretrainedConfig,
415
- cache_config: Optional[CacheConfig] = None,
416
- quant_config: Optional[QuantizationConfig] = None,
417
- lora_config: Optional[LoRAConfig] = None,
414
+ vllm_config: VllmConfig,
415
+ prefix: str = "",
416
) -> None:
417
super().__init__()
420
-
418
+ config = vllm_config.model_config.hf_config
419
+ cache_config = vllm_config.cache_config
420
+ quant_config = vllm_config.quant_config
421
+ lora_config = vllm_config.lora_config
422
self.config = config
423
self.lora_config = lora_config
424
vllm/model_executor/models/stablelm.py
+6
-4
@@ -25,7 +25,7 @@ from torch import nn
25
from transformers import PretrainedConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
-from vllm.config import CacheConfig
28
+from vllm.config import CacheConfig, VllmConfig
29
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import SiluAndMul
31
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -247,11 +247,13 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
247
248
def __init__(
249
self,
250
- config: PretrainedConfig,
251
- cache_config: Optional[CacheConfig] = None,
252
- quant_config: Optional[QuantizationConfig] = None,
250
+ vllm_config: VllmConfig,
251
+ prefix: str = "",
252
) -> None:
253
super().__init__()
254
+ config = vllm_config.model_config.hf_config
255
+ cache_config = vllm_config.cache_config
256
+ quant_config = vllm_config.quant_config
257
self.config = config
258
self.quant_config = quant_config
259
self.model = StableLMEpochModel(config, cache_config, quant_config)
vllm/model_executor/models/starcoder2.py
+9
-5
@@ -25,7 +25,7 @@ from transformers import Starcoder2Config
25
26
from vllm.attention import Attention, AttentionMetadata
27
from vllm.compilation.decorators import support_torch_compile
28
-from vllm.config import CacheConfig
28
+from vllm.config import CacheConfig, VllmConfig
29
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -245,11 +245,15 @@ class Starcoder2Model(nn.Module):
245
246
class Starcoder2ForCausalLM(nn.Module, SupportsPP):
247
248
- def __init__(self,
249
- config: Starcoder2Config,
250
- cache_config: Optional[CacheConfig] = None,
251
- quant_config: Optional[QuantizationConfig] = None):
248
+ def __init__(
249
+ self,
250
+ vllm_config: VllmConfig,
251
+ prefix: str = "",
252
+ ) -> None:
253
super().__init__()
254
+ config = vllm_config.model_config.hf_config
255
+ cache_config = vllm_config.cache_config
256
+ quant_config = vllm_config.quant_config
257
self.config = config
258
self.model = Starcoder2Model(config,
259
cache_config,
vllm/model_executor/models/ultravox.py
+9
-11
@@ -15,12 +15,11 @@ from transformers.models.whisper import WhisperFeatureExtractor
15
from transformers.models.whisper.modeling_whisper import WhisperEncoder
16
17
from vllm.attention import AttentionMetadata
18
-from vllm.config import CacheConfig, MultiModalConfig
18
+from vllm.config import VllmConfig
19
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20
InputContext, token_inputs)
21
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
22
from vllm.model_executor.layers.layernorm import RMSNorm
23
-from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24
from vllm.model_executor.model_loader.loader import DefaultModelLoader
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -340,12 +339,14 @@ class ModifiedWhisperEncoder(WhisperEncoder):
339
@INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox)
340
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
341
343
- def __init__(self,
344
- config: UltravoxConfig,
345
- multimodal_config: MultiModalConfig,
346
- cache_config: Optional[CacheConfig] = None,
347
- quant_config: Optional["QuantizationConfig"] = None):
342
+ def __init__(
343
+ self,
344
+ vllm_config: VllmConfig,
345
+ prefix: str = "",
346
+ ) -> None:
347
super().__init__()
348
+ config = vllm_config.model_config.hf_config
349
+ multimodal_config = vllm_config.model_config.multimodal_config
350
self.config = config
351
self.multi_modal_config = multimodal_config
352
assert self.multi_modal_config
@@ -361,10 +362,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
362
))
363
self.multi_modal_projector = UltravoxProjector(config)
364
self.language_model = init_vllm_registered_model(
364
- config.text_config,
365
- cache_config,
366
- quant_config,
367
- prefix="language_model")
365
+ config.text_config, vllm_config, prefix="language_model")
366
if config.text_model_id is not None:
367
self.secondary_weights.append(
368
DefaultModelLoader.Source(model_or_path=config.text_model_id,
vllm/model_executor/models/utils.py
+6
-21
@@ -11,11 +11,8 @@ from transformers import PretrainedConfig
11
import vllm.envs as envs
12
from vllm.attention.selector import (_Backend, backend_name_to_enum,
13
get_global_forced_attn_backend)
14
-from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
15
- SchedulerConfig)
14
+from vllm.config import VllmConfig
15
from vllm.logger import init_logger
17
-from vllm.model_executor.layers.quantization import QuantizationConfig
18
-from vllm.model_executor.model_loader.loader import build_model
16
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.models import ModelRegistry
18
from vllm.multimodal.base import MultiModalPlaceholderMap, NestedTensors
@@ -236,12 +233,7 @@ class AutoWeightsLoader:
233
234
def init_vllm_registered_model(
235
hf_config: PretrainedConfig,
239
- cache_config: Optional[CacheConfig],
240
- quant_config: Optional[QuantizationConfig],
241
- *,
242
- lora_config: Optional[LoRAConfig] = None,
243
- multimodal_config: Optional[MultiModalConfig] = None,
244
- scheduler_config: Optional[SchedulerConfig] = None,
236
+ vllm_config: VllmConfig,
237
prefix: str = "",
238
) -> nn.Module:
239
"""
@@ -249,18 +241,11 @@ def init_vllm_registered_model(
241
based on the arguments passed to the outer vLLM model.
242
"""
243
model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures)
244
+ import copy
245
+ copied_config = copy.deepcopy(vllm_config)
246
+ copied_config.model_config.hf_config = hf_config
247
253
- return build_model(
254
- model_class,
255
- None,
256
- hf_config,
257
- cache_config,
258
- quant_config,
259
- lora_config=lora_config,
260
- multimodal_config=multimodal_config,
261
- scheduler_config=scheduler_config,
262
- prefix=prefix,
263
- )
248
+ return model_class(vllm_config=copied_config, prefix=prefix)
249
250
251
@overload
vllm/model_executor/models/xverse.py
+13
-9
@@ -27,7 +27,7 @@ from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.compilation.decorators import support_torch_compile
30
-from vllm.config import CacheConfig, LoRAConfig
30
+from vllm.config import CacheConfig, VllmConfig
31
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -225,13 +225,14 @@ class XverseModel(nn.Module):
225
226
def __init__(
227
self,
228
- config: PretrainedConfig,
229
- cache_config: Optional[CacheConfig] = None,
230
- quant_config: Optional[QuantizationConfig] = None,
231
- lora_config: Optional[LoRAConfig] = None,
228
+ vllm_config: VllmConfig,
229
prefix: str = "",
230
) -> None:
231
super().__init__()
232
+ config = vllm_config.model_config.hf_config
233
+ cache_config = vllm_config.cache_config
234
+ quant_config = vllm_config.quant_config
235
+ lora_config = vllm_config.lora_config
236
self.config = config
237
self.padding_idx = config.pad_token_id
238
lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -316,13 +317,16 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
317
318
def __init__(
319
self,
319
- config: PretrainedConfig,
320
- cache_config: Optional[CacheConfig] = None,
321
- quant_config: Optional[QuantizationConfig] = None,
322
- lora_config: Optional[LoRAConfig] = None,
320
+ vllm_config: VllmConfig,
321
+ prefix: str = "",
322
) -> None:
323
super().__init__()
324
325
+ config = vllm_config.model_config.hf_config
326
+ cache_config = vllm_config.cache_config
327
+ quant_config = vllm_config.quant_config
328
+ lora_config = vllm_config.lora_config
329
+
330
self.config = config
331
self.lora_config = lora_config
332
vllm/plugins/__init__.py
-12
@@ -61,15 +61,3 @@ def set_compilation_config(config: Optional[CompilationConfig]):
61
62
def get_compilation_config() -> Optional[CompilationConfig]:
63
return _compilation_config
64
-
65
-
66
-_vllm_config: Optional[VllmConfig] = None
67
-
68
-
69
-def set_vllm_config(config: Optional[VllmConfig]):
70
- global _vllm_config
71
- _vllm_config = config
72
-
73
-
74
-def get_vllm_config() -> Optional[VllmConfig]:
75
- return _vllm_config