@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a95f10ee7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[5/N] pass the whole config to model (#9983)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 8, 2024 at 22:17 UTC 1a95f10ee7d2ffa538a6d210b53bf363e039feee
75 files changed +583 -654
vllm/model_executor/model_loader/loader.py
+7 -93
@@ -9,8 +9,7 @@ import math
9 import os
10 from abc import ABC, abstractmethod
11 from contextlib import contextmanager
12 -from typing import (Any, Dict, Generator, Iterable, List, Optional, Tuple,
13 - Type, cast)
12 +from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, cast
13
14 import gguf
15 import huggingface_hub
@@ -18,20 +17,17 @@ import numpy as np
17 import torch
18 from huggingface_hub import HfApi, hf_hub_download
19 from torch import nn
21 -from transformers import AutoModelForCausalLM, PretrainedConfig
20 +from transformers import AutoModelForCausalLM
21 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
22
24 -from vllm.config import (CacheConfig, LoadConfig, LoadFormat, LoRAConfig,
25 - ModelConfig, MultiModalConfig, ParallelConfig,
26 - PoolerConfig, SchedulerConfig, VllmConfig)
23 +from vllm.config import (LoadConfig, LoadFormat, ModelConfig, ParallelConfig,
24 + VllmConfig)
25 from vllm.distributed import (get_tensor_model_parallel_rank,
26 get_tensor_model_parallel_world_size)
27 from vllm.envs import VLLM_USE_MODELSCOPE
28 from vllm.logger import init_logger
29 from vllm.model_executor.layers.linear import (ReplicatedLinear,
30 RowParallelLinear)
33 -from vllm.model_executor.layers.quantization.base_config import (
34 - QuantizationConfig)
31 from vllm.model_executor.model_loader.tensorizer import (
32 TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
33 serialize_vllm_model, tensorizer_weights_iterator)
@@ -43,8 +39,6 @@ from vllm.model_executor.model_loader.weight_utils import (
39 get_gguf_extra_tensor_names, gguf_quant_weights_iterator,
40 initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
41 safetensors_weights_iterator)
46 -from vllm.model_executor.models import (has_inner_state, supports_lora,
47 - supports_multimodal)
42 from vllm.model_executor.utils import set_weight_attrs
43 from vllm.platforms import current_platform
44 from vllm.utils import is_pin_memory_available
@@ -94,85 +88,11 @@ def device_loading_context(module: torch.nn.Module,
88 logger = init_logger(__name__)
89
90
97 -def _get_model_initialization_kwargs(
98 - model_class: Type[nn.Module],
99 - lora_config: Optional[LoRAConfig],
100 - multimodal_config: Optional[MultiModalConfig],
101 - scheduler_config: Optional[SchedulerConfig] = None,
102 - pooler_config: Optional[PoolerConfig] = None) -> Dict[str, Any]:
103 - """Get extra kwargs for model initialization."""
104 - extra_kwargs: Dict[str, Any] = {}
105 -
106 - if supports_lora(model_class):
107 - # lora_config=None is used to disable LoRA
108 - extra_kwargs["lora_config"] = lora_config
109 - elif lora_config:
110 - raise ValueError(
111 - f"Model {model_class.__name__} does not support LoRA, "
112 - "but LoRA is enabled. Support for this model may "
113 - "be added in the future. If this is important to you, "
114 - "please open an issue on github.")
115 -
116 - if supports_multimodal(model_class):
117 - assert multimodal_config is not None
118 -
119 - extra_kwargs["multimodal_config"] = multimodal_config
120 -
121 - if has_inner_state(model_class) and scheduler_config:
122 - extra_kwargs["scheduler_config"] = scheduler_config
123 - if pooler_config:
124 - extra_kwargs["pooler_config"] = pooler_config
125 - return extra_kwargs
126 -
127 -
128 -def build_model(model_class: Type[nn.Module],
129 - vllm_config: Optional[VllmConfig],
130 - hf_config: PretrainedConfig,
131 - cache_config: Optional[CacheConfig],
132 - quant_config: Optional[QuantizationConfig],
133 - *,
134 - lora_config: Optional[LoRAConfig],
135 - multimodal_config: Optional[MultiModalConfig],
136 - scheduler_config: Optional[SchedulerConfig],
137 - prefix: Optional[str] = None,
138 - pooler_config: Optional[PoolerConfig] = None) -> nn.Module:
139 - extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
140 - multimodal_config,
141 - scheduler_config,
142 - pooler_config)
143 - if prefix:
144 - extra_kwargs["prefix"] = prefix
145 -
146 - # TODO: unify all the module initialization code
147 - # to only take the `VllmConfig` object as input
148 - from vllm.plugins import set_vllm_config
149 - set_vllm_config(vllm_config)
150 -
151 - return model_class(config=hf_config,
152 - cache_config=cache_config,
153 - quant_config=quant_config,
154 - **extra_kwargs)
155 -
156 -
91 def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
92 """Initialize a model with the given configurations."""
93 model_config = vllm_config.model_config
160 - lora_config = vllm_config.lora_config
161 - scheduler_config = vllm_config.scheduler_config
162 - cache_config = vllm_config.cache_config
94 model_class, _ = get_model_architecture(model_config)
164 -
165 - return build_model(
166 - model_class,
167 - vllm_config,
168 - model_config.hf_config,
169 - cache_config=cache_config,
170 - quant_config=vllm_config.quant_config,
171 - lora_config=lora_config,
172 - multimodal_config=model_config.multimodal_config,
173 - scheduler_config=scheduler_config,
174 - pooler_config=model_config.pooler_config,
175 - )
95 + return model_class(vllm_config=vllm_config)
96
97
98 class BaseModelLoader(ABC):
@@ -486,24 +406,18 @@ class TensorizerLoader(BaseModelLoader):
406
407 device_config = vllm_config.device_config
408 model_config = vllm_config.model_config
489 - lora_config = vllm_config.lora_config
490 - cache_config = vllm_config.cache_config
409
410 with set_default_torch_dtype(model_config.dtype):
411 with torch.device(device_config.device):
412 model_class = get_model_architecture(model_config)[0]
495 - quant_config = vllm_config.quant_config
496 - extra_kwargs = _get_model_initialization_kwargs(
497 - model_class, lora_config, model_config.multimodal_config)
498 - extra_kwargs["quant_config"] = quant_config
499 - extra_kwargs["cache_config"] = cache_config
413
414 tensorizer_config = copy.copy(self.tensorizer_config)
415 tensorizer_config.model_class = model_class
416 tensorizer_config.hf_config = model_config.hf_config
417 tensorizer_config.dtype = model_config.dtype
418
506 - model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
419 + model = load_with_tensorizer(tensorizer_config,
420 + vllm_config=vllm_config)
421 return model.eval()
422
423 def download_model(self, model_config: ModelConfig) -> None:
vllm/model_executor/model_loader/tensorizer.py
+3 -12
@@ -17,8 +17,6 @@ from vllm.config import ModelConfig, ParallelConfig
17 from vllm.engine.arg_utils import EngineArgs
18 from vllm.engine.llm_engine import LLMEngine
19 from vllm.logger import init_logger
20 -from vllm.model_executor.layers.quantization.base_config import (
21 - QuantizationConfig)
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
22 from vllm.utils import FlexibleArgumentParser
@@ -268,8 +266,7 @@ class TensorizerAgent:
266 in vllm/model_executor/model_loader/weight_utils.py
267 """
268
271 - def __init__(self, tensorizer_config: TensorizerConfig,
272 - quant_config: QuantizationConfig, **extra_kwargs):
269 + def __init__(self, tensorizer_config: TensorizerConfig, vllm_config):
270 if tensorizer_error_msg is not None:
271 raise ImportError(
272 "Tensorizer is not installed. Please install tensorizer "
@@ -279,11 +276,7 @@ class TensorizerAgent:
276 self.tensorizer_config = tensorizer_config
277 self.tensorizer_args = (
278 self.tensorizer_config._construct_tensorizer_args())
282 - self.extra_kwargs = extra_kwargs
283 - if extra_kwargs.get("quant_config") is not None:
284 - self.quant_config = extra_kwargs["quant_config"]
285 - else:
286 - self.quant_config = quant_config
279 + self.vllm_config = vllm_config
280 self.model = self._init_model()
281
282 def _init_model(self):
@@ -293,9 +286,7 @@ class TensorizerAgent:
286 assert self.tensorizer_config.model_class is not None
287 with no_init_or_tensor():
288 return self.tensorizer_config.model_class(
296 - config=model_args,
297 - quant_config=self.quant_config,
298 - **self.extra_kwargs)
289 + vllm_config=self.vllm_config, )
290
291 def _resize_lora_embeddings(self):
292 """Modify LoRA embedding layers to use bigger tensors
vllm/model_executor/models/arctic.py
+9 -7
@@ -6,7 +6,7 @@ from torch import nn
6
7 from vllm.attention import Attention, AttentionMetadata
8 from vllm.compilation.decorators import support_torch_compile
9 -from vllm.config import CacheConfig
9 +from vllm.config import CacheConfig, VllmConfig
10 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
11 get_tensor_model_parallel_world_size,
12 tensor_model_parallel_all_reduce)
@@ -415,14 +415,16 @@ class ArcticModel(nn.Module):
415
416 class ArcticForCausalLM(nn.Module, SupportsPP):
417
418 - def __init__(self,
419 - config: ArcticConfig,
420 - cache_config: Optional[CacheConfig] = None,
421 - quant_config: Optional[QuantizationConfig] = None,
422 - **kwargs) -> None:
418 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
419 super().__init__()
420 + config = vllm_config.model_config.hf_config
421 + cache_config = vllm_config.cache_config
422 + quant_config = vllm_config.quant_config
423 self.config = config
425 - self.model = ArcticModel(config, cache_config, quant_config)
424 + self.model = ArcticModel(config,
425 + cache_config,
426 + quant_config,
427 + prefix=prefix)
428 self.vocab_size = config.vocab_size
429 self.lm_head = ParallelLMHead(
430 self.vocab_size,
vllm/model_executor/models/baichuan.py
+16 -21
@@ -26,7 +26,7 @@ from transformers import PretrainedConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 from vllm.compilation.decorators import support_torch_compile
29 -from vllm.config import CacheConfig, LoRAConfig
29 +from vllm.config import CacheConfig, VllmConfig
30 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
31 get_tensor_model_parallel_world_size)
32 from vllm.model_executor.layers.activation import SiluAndMul
@@ -332,14 +332,15 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
333 def __init__(
334 self,
335 - config: PretrainedConfig,
336 - position_embedding: str,
337 - cache_config: Optional[CacheConfig] = None,
338 - quant_config: Optional[QuantizationConfig] = None,
339 - lora_config: Optional[LoRAConfig] = None,
335 + vllm_config: VllmConfig,
336 + prefix: str = "",
337 + position_embedding: str = "ROPE",
338 ):
339 super().__init__()
342 -
340 + config = vllm_config.model_config.hf_config
341 + cache_config = vllm_config.cache_config
342 + quant_config = vllm_config.quant_config
343 + lora_config = vllm_config.lora_config
344 self.config = config
345 self.lora_config = lora_config
346
@@ -439,17 +440,14 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
440
441 def __init__(
442 self,
442 - config: PretrainedConfig,
443 - cache_config: Optional[CacheConfig] = None,
444 - quant_config: Optional[QuantizationConfig] = None,
445 - lora_config: Optional[LoRAConfig] = None,
443 + vllm_config: VllmConfig,
444 + prefix: str = "",
445 ):
446 + config = vllm_config.model_config.hf_config
447 if config.hidden_size == 4096: # baichuan2 7b
448 - super().__init__(config, "ROPE", cache_config, quant_config,
449 - lora_config)
448 + super().__init__(vllm_config, prefix, "ROPE")
449 else: # baichuan 13b, baichuan2 13b
451 - super().__init__(config, "ALIBI", cache_config, quant_config,
452 - lora_config)
450 + super().__init__(vllm_config, prefix, "ALIBI")
451
452
453 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -459,10 +457,7 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
457
458 def __init__(
459 self,
462 - config: PretrainedConfig,
463 - cache_config: Optional[CacheConfig] = None,
464 - quant_config: Optional[QuantizationConfig] = None,
465 - lora_config: Optional[LoRAConfig] = None,
460 + vllm_config: VllmConfig,
461 + prefix: str = "",
462 ):
467 - super().__init__(config, "ROPE", cache_config, quant_config,
468 - lora_config)
463 + super().__init__(vllm_config, prefix, "ROPE")
vllm/model_executor/models/bart.py
+6 -6
@@ -25,7 +25,7 @@ from transformers import BartConfig
25 from transformers.utils import logging
26
27 from vllm.attention import Attention, AttentionMetadata, AttentionType
28 -from vllm.config import CacheConfig, LoRAConfig
28 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
29 from vllm.distributed import get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -810,13 +810,13 @@ class BartModel(nn.Module):
810 class BartForConditionalGeneration(nn.Module):
811 base_model_prefix = "model"
812
813 - def __init__(self,
814 - config: BartConfig,
815 - cache_config: Optional[CacheConfig] = None,
816 - quant_config: Optional[QuantizationConfig] = None,
817 - lora_config: Optional[LoRAConfig] = None):
813 + def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
814
815 super().__init__()
816 + config = vllm_config.model_config.hf_config
817 + cache_config = vllm_config.cache_config
818 + quant_config = vllm_config.quant_config
819 + lora_config = vllm_config.lora_config
820 # currently all existing BART models have `tie_word_embeddings` enabled
821 assert config.tie_word_embeddings
822 self.config = config
vllm/model_executor/models/bert.py
+7 -5
@@ -6,7 +6,7 @@ from transformers import BertConfig
6
7 from vllm.attention import Attention, AttentionMetadata, AttentionType
8 from vllm.attention.backends.xformers import XFormersImpl
9 -from vllm.config import CacheConfig, PoolerConfig
9 +from vllm.config import CacheConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.activation import get_act_fn
12 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -384,12 +384,14 @@ class BertEmbeddingModel(nn.Module):
384
385 def __init__(
386 self,
387 - config: BertConfig,
388 - cache_config: Optional[CacheConfig] = None,
389 - quant_config: Optional[QuantizationConfig] = None,
390 - pooler_config: Optional[PoolerConfig] = None,
387 + vllm_config: VllmConfig,
388 + prefix: str = "",
389 ) -> None:
390 super().__init__()
391 + config = vllm_config.model_config.hf_config
392 + cache_config = vllm_config.cache_config
393 + quant_config = vllm_config.quant_config
394 + pooler_config = vllm_config.model_config.pooler_config
395 self.model = BertModel(config, cache_config, quant_config)
396 self._pooler = Pooler.from_config_with_defaults(
397 pooler_config,
vllm/model_executor/models/blip2.py
+11 -9
@@ -8,7 +8,7 @@ from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
8 apply_chunking_to_forward)
9
10 from vllm.attention import AttentionMetadata
11 -from vllm.config import CacheConfig, MultiModalConfig
11 +from vllm.config import CacheConfig, VllmConfig
12 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13 InputContext, token_inputs)
14 from vllm.model_executor.layers.activation import get_act_fn
@@ -483,14 +483,17 @@ def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
483 @INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
484 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
485
486 - def __init__(self,
487 - config: Blip2Config,
488 - multimodal_config: MultiModalConfig,
489 - cache_config: Optional[CacheConfig] = None,
490 - quant_config: Optional[QuantizationConfig] = None) -> None:
486 + def __init__(
487 + self,
488 + vllm_config: VllmConfig,
489 + prefix: str = "",
490 + ) -> None:
491
492 super().__init__()
493 -
493 + config = vllm_config.model_config.hf_config
494 + cache_config = vllm_config.cache_config
495 + quant_config = vllm_config.quant_config
496 + multimodal_config = vllm_config.model_config.multimodal_config
497 self.config = config
498 self.multimodal_config = multimodal_config
499
@@ -513,8 +516,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
516
517 self.language_model = init_vllm_registered_model(
518 config.text_config,
516 - cache_config,
517 - quant_config,
519 + vllm_config=vllm_config,
520 prefix="language_model")
521
522 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/bloom.py
+6 -4
@@ -24,7 +24,7 @@ from transformers import BloomConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 from vllm.compilation.decorators import support_torch_compile
27 -from vllm.config import CacheConfig
27 +from vllm.config import CacheConfig, VllmConfig
28 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
29 get_tensor_model_parallel_world_size)
30 from vllm.model_executor.layers.activation import get_act_fn
@@ -283,11 +283,13 @@ class BloomForCausalLM(nn.Module, SupportsPP):
283
284 def __init__(
285 self,
286 - config: BloomConfig,
287 - cache_config: Optional[CacheConfig] = None,
288 - quant_config: Optional[QuantizationConfig] = None,
286 + vllm_config: VllmConfig,
287 + prefix: str = "",
288 ):
289 super().__init__()
290 + config = vllm_config.model_config.hf_config
291 + cache_config = vllm_config.cache_config
292 + quant_config = vllm_config.quant_config
293 self.config = config
294 self.quant_config = quant_config
295 self.transformer = BloomModel(config, cache_config, quant_config)
vllm/model_executor/models/chameleon.py
+7 -5
@@ -9,7 +9,7 @@ from torch import nn
9 from transformers import ChameleonConfig, ChameleonVQVAEConfig
10
11 from vllm.attention import Attention, AttentionMetadata
12 -from vllm.config import CacheConfig, MultiModalConfig
12 +from vllm.config import CacheConfig, VllmConfig
13 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 InputContext, token_inputs)
@@ -926,12 +926,14 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
926
927 def __init__(
928 self,
929 - config: ChameleonConfig,
930 - multimodal_config: MultiModalConfig,
931 - cache_config: Optional[CacheConfig] = None,
932 - quant_config: Optional[QuantizationConfig] = None,
929 + vllm_config: VllmConfig,
930 + prefix: str = "",
931 ) -> None:
932 super().__init__()
933 + config = vllm_config.model_config.hf_config
934 + cache_config = vllm_config.cache_config
935 + quant_config = vllm_config.quant_config
936 + multimodal_config = vllm_config.model_config.multimodal_config
937 self.config = config
938 self.multimodal_config = multimodal_config
939 self.model = ChameleonModel(config, cache_config, quant_config)
vllm/model_executor/models/chatglm.py
+8 -7
@@ -11,7 +11,7 @@ from torch import nn
11 from torch.nn import LayerNorm
12
13 from vllm.attention import Attention, AttentionMetadata
14 -from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
14 +from vllm.config import CacheConfig, VllmConfig
15 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
17 InputContext, token_inputs)
@@ -595,14 +595,15 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
595
596 def __init__(
597 self,
598 - config: ChatGLMConfig,
599 - multimodal_config: MultiModalConfig,
600 - cache_config: Optional[CacheConfig] = None,
601 - quant_config: Optional[QuantizationConfig] = None,
602 - lora_config: Optional[LoRAConfig] = None,
598 + vllm_config: VllmConfig,
599 + prefix: str = "",
600 ):
601 super().__init__()
605 -
602 + config = vllm_config.model_config.hf_config
603 + cache_config = vllm_config.cache_config
604 + quant_config = vllm_config.quant_config
605 + lora_config = vllm_config.lora_config
606 + multimodal_config = vllm_config.model_config.multimodal_config
607 self.config = config
608 self.lora_config = lora_config
609 self.multimodal_config = multimodal_config
vllm/model_executor/models/commandr.py
+7 -5
@@ -28,7 +28,7 @@ from transformers import CohereConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -334,12 +334,14 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
334
335 def __init__(
336 self,
337 - config: CohereConfig,
338 - cache_config: Optional[CacheConfig] = None,
339 - quant_config: Optional[QuantizationConfig] = None,
340 - lora_config: Optional[LoRAConfig] = None,
337 + vllm_config: VllmConfig,
338 + prefix: str = "",
339 ) -> None:
340 super().__init__()
341 + config = vllm_config.model_config.hf_config
342 + cache_config = vllm_config.cache_config
343 + quant_config = vllm_config.quant_config
344 + lora_config = vllm_config.lora_config
345 self.config = config
346 # currently all existing command R models have `tie_word_embeddings`
347 # enabled
vllm/model_executor/models/dbrx.py
+6 -4
@@ -4,7 +4,7 @@ import torch
4 import torch.nn as nn
5
6 from vllm.attention import Attention, AttentionMetadata
7 -from vllm.config import CacheConfig
7 +from vllm.config import CacheConfig, VllmConfig
8 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
9 get_tensor_model_parallel_world_size)
10 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -352,11 +352,13 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
352
353 def __init__(
354 self,
355 - config: DbrxConfig,
356 - cache_config: Optional[CacheConfig] = None,
357 - quant_config: Optional[QuantizationConfig] = None,
355 + vllm_config: VllmConfig,
356 + prefix: str = "",
357 ):
358 super().__init__()
359 + config = vllm_config.model_config.hf_config
360 + cache_config = vllm_config.cache_config
361 + quant_config = vllm_config.quant_config
362 self.config = config
363 if config.tie_word_embeddings:
364 raise ValueError(
vllm/model_executor/models/decilm.py
+6 -12
@@ -22,13 +22,11 @@
22 # limitations under the License.
23 """Inference-only DeciLM model compatible with HuggingFace weights."""
24
25 -from typing import Iterable, Optional, Tuple
25 +from typing import Iterable, Tuple
26
27 import torch
28 -from transformers import LlamaConfig
28
30 -from vllm.config import CacheConfig, LoRAConfig
31 -from vllm.model_executor.layers.quantization import QuantizationConfig
29 +from vllm.config import VllmConfig
30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31 from vllm.model_executor.models.llama import LlamaForCausalLM
32
@@ -55,17 +53,13 @@ class DeciLMForCausalLM(LlamaForCausalLM):
53
54 def __init__(
55 self,
58 - config: LlamaConfig,
59 - cache_config: Optional[CacheConfig] = None,
60 - quant_config: Optional[QuantizationConfig] = None,
61 - lora_config: Optional[LoRAConfig] = None,
56 + vllm_config: VllmConfig,
57 + prefix: str = "",
58 ) -> None:
59 + config = vllm_config.model_config.hf_config
60 config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
61 delattr(config, "num_key_value_heads_per_layer")
65 - super().__init__(config=config,
66 - cache_config=cache_config,
67 - quant_config=quant_config,
68 - lora_config=lora_config)
62 + super().__init__(vllm_config=vllm_config)
63
64 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
65 stacked_params_mapping = [
vllm/model_executor/models/deepseek.py
+6 -4
@@ -27,7 +27,7 @@ from torch import nn
27 from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 -from vllm.config import CacheConfig
30 +from vllm.config import CacheConfig, VllmConfig
31 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32 get_tensor_model_parallel_world_size,
33 tensor_model_parallel_all_reduce)
@@ -385,11 +385,13 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
385
386 def __init__(
387 self,
388 - config: PretrainedConfig,
389 - cache_config: Optional[CacheConfig] = None,
390 - quant_config: Optional[QuantizationConfig] = None,
388 + vllm_config: VllmConfig,
389 + prefix: str = "",
390 ) -> None:
391 super().__init__()
392 + config = vllm_config.model_config.hf_config
393 + cache_config = vllm_config.cache_config
394 + quant_config = vllm_config.quant_config
395 self.config = config
396 self.quant_config = quant_config
397 self.model = DeepseekModel(config, cache_config, quant_config)
vllm/model_executor/models/deepseek_v2.py
+6 -4
@@ -28,7 +28,7 @@ from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import (get_pp_group,
33 get_tensor_model_parallel_world_size,
34 tensor_model_parallel_all_reduce)
@@ -481,11 +481,13 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
481
482 def __init__(
483 self,
484 - config: PretrainedConfig,
485 - cache_config: Optional[CacheConfig] = None,
486 - quant_config: Optional[QuantizationConfig] = None,
484 + vllm_config: VllmConfig,
485 + prefix: str = "",
486 ) -> None:
487 super().__init__()
488 + config = vllm_config.model_config.hf_config
489 + cache_config = vllm_config.cache_config
490 + quant_config = vllm_config.quant_config
491 self.config = config
492 self.quant_config = quant_config
493 self.model = DeepseekV2Model(config,
vllm/model_executor/models/eagle.py
+4 -3
@@ -4,6 +4,7 @@ import torch
4 import torch.nn as nn
5
6 from vllm.attention.backends.abstract import AttentionMetadata
7 +from vllm.config import VllmConfig
8 from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 from vllm.model_executor.layers.sampler import SamplerOutput
10 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -12,7 +13,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 from vllm.model_executor.models import ModelRegistry
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 from vllm.sequence import IntermediateTensors
15 -from vllm.transformers_utils.configs.eagle import EAGLEConfig
16
17
18 class EAGLE(nn.Module):
@@ -34,14 +34,15 @@ class EAGLE(nn.Module):
34 in the draft checkpoint (using key token_map). Also, the draft config
35 needs to have truncated_vocab_size (=k) as an attribute."""
36
37 - def __init__(self, config: EAGLEConfig, *args, **kwargs) -> None:
37 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
38 super().__init__()
39 + config = vllm_config.model_config.hf_config
40 self.config = config
41
42 architectures = getattr(self.config.model, "architectures", [])
43 model_cls, _ = ModelRegistry.resolve_model_cls(architectures)
44
44 - self.model = model_cls(self.config.model, *args, **kwargs)
45 + self.model = model_cls(vllm_config, prefix)
46 self.fc = nn.Linear(config.model.hidden_size * 2,
47 config.model.hidden_size,
48 bias=getattr(self.config, "eagle_fc_bias", False))
vllm/model_executor/models/exaone.py
+7 -5
@@ -29,7 +29,7 @@ from torch import nn
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -440,12 +440,14 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
440
441 def __init__(
442 self,
443 - config: ExaoneConfig,
444 - cache_config: Optional[CacheConfig] = None,
445 - quant_config: Optional[QuantizationConfig] = None,
446 - lora_config: Optional[LoRAConfig] = None,
443 + vllm_config: VllmConfig,
444 + prefix: str = "",
445 ) -> None:
446 super().__init__()
447 + config = vllm_config.model_config.hf_config
448 + cache_config = vllm_config.cache_config
449 + quant_config = vllm_config.quant_config
450 + lora_config = vllm_config.lora_config
451
452 self.config = config
453 self.lora_config = lora_config
vllm/model_executor/models/falcon.py
+6 -4
@@ -27,7 +27,7 @@ from transformers import FalconConfig as HF_FalconConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.compilation.decorators import support_torch_compile
30 -from vllm.config import CacheConfig
30 +from vllm.config import CacheConfig, VllmConfig
31 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32 get_tensor_model_parallel_world_size,
33 tensor_model_parallel_all_reduce)
@@ -403,11 +403,13 @@ class FalconForCausalLM(nn.Module, SupportsPP):
403
404 def __init__(
405 self,
406 - config: FalconConfig,
407 - cache_config: Optional[CacheConfig] = None,
408 - quant_config: Optional[QuantizationConfig] = None,
406 + vllm_config: VllmConfig,
407 + prefix: str = "",
408 ):
409 super().__init__()
410 + config = vllm_config.model_config.hf_config
411 + cache_config = vllm_config.cache_config
412 + quant_config = vllm_config.quant_config
413 self.config = config
414 self.quant_config = quant_config
415 self.transformer = FalconModel(config, cache_config, quant_config)
vllm/model_executor/models/florence2.py
+5 -5
@@ -6,7 +6,7 @@ import torch.nn as nn
6 from transformers import PretrainedConfig
7
8 from vllm.attention import AttentionMetadata
9 -from vllm.config import CacheConfig
9 +from vllm.config import CacheConfig, VllmConfig
10 from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
@@ -189,11 +189,11 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
189
190 class Florence2ForConditionalGeneration(nn.Module):
191
192 - def __init__(self,
193 - config: PretrainedConfig,
194 - cache_config: Optional[CacheConfig] = None,
195 - quant_config: Optional[QuantizationConfig] = None):
192 + def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
193 super().__init__()
194 + config = vllm_config.model_config.hf_config
195 + cache_config = vllm_config.cache_config
196 + quant_config = vllm_config.quant_config
197
198 # TODO(Isotr0py): Add vision backbone
199 self.language_model = Florence2LanguageForConditionalGeneration(
vllm/model_executor/models/fuyu.py
+7 -8
@@ -22,14 +22,13 @@ import torch
22 import torch.nn as nn
23 import torch.utils.checkpoint
24 from PIL import Image
25 -from transformers import FuyuConfig, FuyuImageProcessor
25 +from transformers import FuyuImageProcessor
26
27 from vllm.attention import AttentionMetadata
28 -from vllm.config import CacheConfig, MultiModalConfig
28 +from vllm.config import VllmConfig
29 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30 InputContext, token_inputs)
31 from vllm.model_executor.layers.linear import ColumnParallelLinear
32 -from vllm.model_executor.layers.quantization import QuantizationConfig
32 from vllm.model_executor.layers.sampler import SamplerOutput
33 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -227,12 +226,12 @@ def input_mapper_for_fuyu(ctx: InputContext, data: object):
226 @INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
227 class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
228
230 - def __init__(self,
231 - config: FuyuConfig,
232 - multimodal_config: MultiModalConfig,
233 - cache_config: Optional[CacheConfig] = None,
234 - quant_config: Optional[QuantizationConfig] = None) -> None:
229 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
230 super().__init__()
231 + config = vllm_config.model_config.hf_config
232 + cache_config = vllm_config.cache_config
233 + quant_config = vllm_config.quant_config
234 + multimodal_config = vllm_config.model_config.multimodal_config
235 self.config = config
236 self.multimodal_config = multimodal_config
237
vllm/model_executor/models/gemma.py
+6 -5
@@ -22,7 +22,7 @@ from transformers import GemmaConfig
22
23 from vllm.attention import Attention, AttentionMetadata
24 from vllm.compilation.decorators import support_torch_compile
25 -from vllm.config import CacheConfig, LoRAConfig
25 +from vllm.config import CacheConfig, VllmConfig
26 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import GeluAndMul
@@ -374,13 +374,14 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
374
375 def __init__(
376 self,
377 - config: GemmaConfig,
378 - cache_config: Optional[CacheConfig] = None,
379 - quant_config: Optional[QuantizationConfig] = None,
380 - lora_config: Optional[LoRAConfig] = None,
377 + vllm_config: VllmConfig,
378 prefix: str = "",
379 ) -> None:
380 super().__init__()
381 + config = vllm_config.model_config.hf_config
382 + cache_config = vllm_config.cache_config
383 + quant_config = vllm_config.quant_config
384 + lora_config = vllm_config.lora_config
385
386 self.config = config
387 # currently all existing Gemma models have `tie_word_embeddings` enabled
vllm/model_executor/models/gemma2.py
+15 -12
@@ -21,7 +21,7 @@ from transformers import Gemma2Config
21
22 from vllm.attention import Attention, AttentionMetadata
23 from vllm.compilation.decorators import support_torch_compile
24 -from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
24 +from vllm.config import CacheConfig, VllmConfig
25 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
26 from vllm.logger import init_logger
27 from vllm.model_executor.layers.activation import GeluAndMul
@@ -245,12 +245,13 @@ class Gemma2Model(nn.Module):
245
246 def __init__(
247 self,
248 - config: Gemma2Config,
249 - cache_config: Optional[CacheConfig] = None,
250 - quant_config: Optional[QuantizationConfig] = None,
248 + vllm_config: VllmConfig,
249 prefix: str = "",
250 ) -> None:
251 super().__init__()
252 + config = vllm_config.model_config.hf_config
253 + cache_config = vllm_config.cache_config
254 + quant_config = vllm_config.quant_config
255 self.config = config
256
257 self.embed_tokens = VocabParallelEmbedding(
@@ -400,11 +401,13 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
401
402 def __init__(
403 self,
403 - config: Gemma2Config,
404 - cache_config: Optional[CacheConfig] = None,
405 - quant_config: Optional[QuantizationConfig] = None,
406 - lora_config: Optional[LoRAConfig] = None,
404 + vllm_config: VllmConfig,
405 + prefix: str = "",
406 ) -> None:
407 + config = vllm_config.model_config.hf_config
408 + cache_config = vllm_config.cache_config
409 + quant_config = vllm_config.quant_config
410 + lora_config = vllm_config.lora_config
411 del lora_config # Unused.
412 super().__init__()
413 self.config = config
@@ -470,14 +473,14 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
473
474 def __init__(
475 self,
473 - pooler_config: Optional[PoolerConfig] = None,
474 - **kwargs,
476 + vllm_config: VllmConfig,
477 + prefix: str = "",
478 ) -> None:
479 super().__init__()
480
478 - self.model = Gemma2Model(**kwargs)
481 + self.model = Gemma2Model(vllm_config, prefix)
482 self._pooler = Pooler.from_config_with_defaults(
480 - pooler_config,
483 + vllm_config.model_config.pooler_config,
484 pooling_type=PoolingType.LAST,
485 normalize=True,
486 softmax=False)
vllm/model_executor/models/gpt2.py
+6 -4
@@ -24,7 +24,7 @@ from transformers import GPT2Config
24
25 from vllm.attention import Attention, AttentionMetadata
26 from vllm.compilation.decorators import support_torch_compile
27 -from vllm.config import CacheConfig
27 +from vllm.config import CacheConfig, VllmConfig
28 from vllm.distributed.parallel_state import (
29 get_pp_group, get_tensor_model_parallel_world_size)
30 from vllm.model_executor.layers.activation import get_act_fn
@@ -242,11 +242,13 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
242
243 def __init__(
244 self,
245 - config: GPT2Config,
246 - cache_config: Optional[CacheConfig] = None,
247 - quant_config: Optional[QuantizationConfig] = None,
245 + vllm_config: VllmConfig,
246 + prefix: str = "",
247 ):
248 super().__init__()
249 + config = vllm_config.model_config.hf_config
250 + cache_config = vllm_config.cache_config
251 + quant_config = vllm_config.quant_config
252 self.config = config
253 self.quant_config = quant_config
254 self.transformer = GPT2Model(config,
vllm/model_executor/models/gpt_bigcode.py
+7 -5
@@ -25,7 +25,7 @@ from transformers import GPTBigCodeConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 from vllm.compilation.decorators import support_torch_compile
28 -from vllm.config import CacheConfig, LoRAConfig
28 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
29 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -260,12 +260,14 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
260
261 def __init__(
262 self,
263 - config: GPTBigCodeConfig,
264 - cache_config: Optional[CacheConfig] = None,
265 - quant_config: Optional[QuantizationConfig] = None,
266 - lora_config: Optional[LoRAConfig] = None,
263 + vllm_config: VllmConfig,
264 + prefix: str = "",
265 ):
266 super().__init__()
267 + config = vllm_config.model_config.hf_config
268 + cache_config = vllm_config.cache_config
269 + quant_config = vllm_config.quant_config
270 + lora_config = vllm_config.lora_config
271
272 self.config = config
273 self.lora_config = lora_config
vllm/model_executor/models/gpt_j.py
+6 -4
@@ -23,7 +23,7 @@ from transformers import GPTJConfig
23
24 from vllm.attention import Attention, AttentionMetadata
25 from vllm.compilation.decorators import support_torch_compile
26 -from vllm.config import CacheConfig
26 +from vllm.config import CacheConfig, VllmConfig
27 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -231,11 +231,13 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
231
232 def __init__(
233 self,
234 - config: GPTJConfig,
235 - cache_config: Optional[CacheConfig] = None,
236 - quant_config: Optional[QuantizationConfig] = None,
234 + vllm_config: VllmConfig,
235 + prefix: str = "",
236 ):
237 super().__init__()
238 + config = vllm_config.model_config.hf_config
239 + cache_config = vllm_config.cache_config
240 + quant_config = vllm_config.quant_config
241 self.config = config
242 self.quant_config = quant_config
243 assert not config.tie_word_embeddings
vllm/model_executor/models/gpt_neox.py
+6 -4
@@ -23,7 +23,7 @@ from transformers import GPTNeoXConfig
23
24 from vllm.attention import Attention, AttentionMetadata
25 from vllm.compilation.decorators import support_torch_compile
26 -from vllm.config import CacheConfig
26 +from vllm.config import CacheConfig, VllmConfig
27 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -244,11 +244,13 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
244
245 def __init__(
246 self,
247 - config: GPTNeoXConfig,
248 - cache_config: Optional[CacheConfig] = None,
249 - quant_config: Optional[QuantizationConfig] = None,
247 + vllm_config: VllmConfig,
248 + prefix: str = "",
249 ):
250 super().__init__()
251 + config = vllm_config.model_config.hf_config
252 + cache_config = vllm_config.cache_config
253 + quant_config = vllm_config.quant_config
254 self.config = config
255 self.quant_config = quant_config
256 self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config)
vllm/model_executor/models/granite.py
+7 -5
@@ -28,7 +28,7 @@ from transformers import GraniteConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -372,12 +372,14 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
372
373 def __init__(
374 self,
375 - config: GraniteConfig,
376 - cache_config: Optional[CacheConfig] = None,
377 - quant_config: Optional[QuantizationConfig] = None,
378 - lora_config: Optional[LoRAConfig] = None,
375 + vllm_config: VllmConfig,
376 + prefix: str = "",
377 ) -> None:
378 super().__init__()
379 + config = vllm_config.model_config.hf_config
380 + cache_config = vllm_config.cache_config
381 + quant_config = vllm_config.quant_config
382 + lora_config = vllm_config.lora_config
383
384 self.config = config
385 self.lora_config = lora_config
vllm/model_executor/models/granitemoe.py
+7 -5
@@ -28,7 +28,7 @@ from transformers.models.granitemoe import GraniteMoeConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -335,12 +335,14 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
335
336 def __init__(
337 self,
338 - config: GraniteMoeConfig,
339 - cache_config: Optional[CacheConfig] = None,
340 - quant_config: Optional[QuantizationConfig] = None,
341 - lora_config: Optional[LoRAConfig] = None,
338 + vllm_config: VllmConfig,
339 + prefix: str = "",
340 ) -> None:
341 super().__init__()
342 + config = vllm_config.model_config.hf_config
343 + cache_config = vllm_config.cache_config
344 + quant_config = vllm_config.quant_config
345 + lora_config = vllm_config.lora_config
346
347 self.config = config
348 self.lora_config = lora_config
vllm/model_executor/models/idefics3.py
+8 -5
@@ -26,7 +26,7 @@ from transformers import PretrainedConfig as Idefics3Config
26 from transformers import ProcessorMixin as Idefics3ImageProcessor
27
28 from vllm.attention import AttentionMetadata
29 -from vllm.config import CacheConfig, MultiModalConfig
29 +from vllm.config import CacheConfig, VllmConfig
30 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
31 InputContext, token_inputs)
32 from vllm.logger import init_logger
@@ -615,13 +615,16 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
615
616 def __init__(
617 self,
618 - config: Idefics3Config,
619 - multimodal_config: MultiModalConfig,
620 - cache_config: Optional[CacheConfig] = None,
621 - quant_config: Optional[QuantizationConfig] = None,
618 + vllm_config: VllmConfig,
619 + prefix: str = "",
620 ) -> None:
621 super().__init__()
622
623 + config = vllm_config.model_config.hf_config
624 + cache_config = vllm_config.cache_config
625 + quant_config = vllm_config.quant_config
626 + multimodal_config = vllm_config.model_config.multimodal_config
627 +
628 self.config = config
629 self.multimodal_config = multimodal_config
630
vllm/model_executor/models/interfaces_base.py
+4 -20
@@ -11,9 +11,8 @@ from vllm.utils import supports_kw
11
12 if TYPE_CHECKING:
13 from vllm.attention import AttentionMetadata
14 - from vllm.config import CacheConfig
14 + from vllm.config import VllmConfig
15 from vllm.model_executor.layers.pooler import PoolerOutput
16 - from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.sampler import SamplerOutput
17 from vllm.model_executor.pooling_metadata import PoolingMetadata
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -39,10 +38,8 @@ class VllmModel(Protocol[C_co, T_co]):
38
39 def __init__(
40 self,
42 - config: C_co,
43 - *,
44 - cache_config: Optional["CacheConfig"],
45 - quant_config: Optional["QuantizationConfig"],
41 + vllm_config: "VllmConfig",
42 + prefix: str = "",
43 ) -> None:
44 ...
45
@@ -58,20 +55,7 @@ class VllmModel(Protocol[C_co, T_co]):
55
56 def _check_vllm_model_init(model: Union[Type[object], object]) -> bool:
57 model_init = model.__init__
61 - vllm_kws = ("cache_config", "quant_config")
62 - missing_kws = tuple(kw for kw in vllm_kws
63 - if not supports_kw(model_init, kw))
64 -
65 - if missing_kws and (isinstance(model, type)
66 - and issubclass(model, nn.Module)):
67 - logger.warning(
68 - "The model (%s) is missing "
69 - "vLLM-specific keywords from its initializer: %s",
70 - model,
71 - missing_kws,
72 - )
73 -
74 - return len(missing_kws) == 0
58 + return supports_kw(model_init, "vllm_config")
59
60
61 def _check_vllm_model_forward(model: Union[Type[object], object]) -> bool:
vllm/model_executor/models/internlm2.py
+5 -4
@@ -7,7 +7,7 @@ from transformers import PretrainedConfig
7
8 from vllm.attention import Attention, AttentionMetadata
9 from vllm.compilation.decorators import support_torch_compile
10 -from vllm.config import CacheConfig
10 +from vllm.config import CacheConfig, VllmConfig
11 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
12 get_tensor_model_parallel_world_size,
13 split_tensor_along_last_dim,
@@ -319,12 +319,13 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
319
320 def __init__(
321 self,
322 - config: PretrainedConfig,
323 - cache_config: Optional[CacheConfig] = None,
324 - quant_config: Optional[QuantizationConfig] = None,
322 + vllm_config: VllmConfig,
323 prefix: str = "",
324 ) -> None:
325 super().__init__()
326 + config = vllm_config.model_config.hf_config
327 + cache_config = vllm_config.cache_config
328 + quant_config = vllm_config.quant_config
329 self.config = config
330 self.quant_config = quant_config
331 self.model = InternLM2Model(config,
vllm/model_executor/models/internlm2_ve.py
+5 -4
@@ -5,7 +5,7 @@ from torch import nn
5 from transformers import PretrainedConfig
6
7 from vllm.attention import AttentionMetadata
8 -from vllm.config import CacheConfig
8 +from vllm.config import CacheConfig, VllmConfig
9 from vllm.distributed import get_pp_group
10 from vllm.model_executor.layers.layernorm import RMSNorm
11 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -161,11 +161,12 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
161
162 def __init__(
163 self,
164 - config: PretrainedConfig,
165 - cache_config: Optional[CacheConfig] = None,
166 - quant_config: Optional[QuantizationConfig] = None,
164 + vllm_config: VllmConfig,
165 prefix: str = "",
166 ) -> None:
167 + config = vllm_config.model_config.hf_config
168 + cache_config = vllm_config.cache_config
169 + quant_config = vllm_config.quant_config
170 super().__init__(config, cache_config, quant_config)
171 self.model = InternLM2VEModel(config,
172 cache_config,
vllm/model_executor/models/internvl.py
+7 -8
@@ -16,7 +16,7 @@ from PIL import Image
16 from transformers import PretrainedConfig
17
18 from vllm.attention import AttentionMetadata
19 -from vllm.config import CacheConfig, MultiModalConfig
19 +from vllm.config import VllmConfig
20 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 InputContext, token_inputs)
22 from vllm.model_executor.layers.quantization import (AWQConfig,
@@ -410,13 +410,13 @@ input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
410 @INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
411 class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
412
413 - def __init__(self,
414 - config: PretrainedConfig,
415 - multimodal_config: MultiModalConfig,
416 - cache_config: Optional[CacheConfig] = None,
417 - quant_config: Optional[QuantizationConfig] = None) -> None:
413 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
414 super().__init__()
415
416 + config = vllm_config.model_config.hf_config
417 + quant_config = vllm_config.quant_config
418 + multimodal_config = vllm_config.model_config.multimodal_config
419 +
420 self.config = config
421 self.multimodal_config = multimodal_config
422 self._patch_quant_config(config, quant_config)
@@ -440,8 +440,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
440
441 self.language_model = init_vllm_registered_model(
442 config.text_config,
443 - cache_config,
444 - quant_config,
443 + vllm_config=vllm_config,
444 prefix="language_model")
445
446 self.mlp1 = self._init_mlp1(config)
vllm/model_executor/models/jais.py
+6 -4
@@ -26,7 +26,7 @@ from torch import nn
26
27 from vllm.attention import Attention, AttentionMetadata
28 from vllm.compilation.decorators import support_torch_compile
29 -from vllm.config import CacheConfig
29 +from vllm.config import CacheConfig, VllmConfig
30 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
31 get_tensor_model_parallel_world_size)
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -288,11 +288,13 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
288
289 def __init__(
290 self,
291 - config: JAISConfig,
292 - cache_config: Optional[CacheConfig] = None,
293 - quant_config: Optional[QuantizationConfig] = None,
291 + vllm_config: VllmConfig,
292 + prefix: str = "",
293 ):
294 super().__init__()
295 + config = vllm_config.model_config.hf_config
296 + cache_config = vllm_config.cache_config
297 + quant_config = vllm_config.quant_config
298 self.config = config
299 self.quant_config = quant_config
300 self.transformer = JAISModel(config, cache_config, quant_config)
vllm/model_executor/models/jamba.py
+8 -6
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 from vllm.attention.layer import Attention
10 -from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_world_size
12 from vllm.model_executor.layers.fused_moe import FusedMoE
13 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -350,12 +350,14 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
350
351 def __init__(
352 self,
353 - config: JambaConfig,
354 - cache_config: Optional[CacheConfig] = None,
355 - quant_config: Optional[QuantizationConfig] = None,
356 - lora_config: Optional[LoRAConfig] = None,
357 - scheduler_config: Optional[SchedulerConfig] = None,
353 + vllm_config: VllmConfig,
354 + prefix: str = "",
355 ) -> None:
356 + config = vllm_config.model_config.hf_config
357 + cache_config = vllm_config.cache_config
358 + quant_config = vllm_config.quant_config
359 + lora_config = vllm_config.lora_config
360 + scheduler_config = vllm_config.scheduler_config
361 assert not cache_config.enable_prefix_caching, \
362 "Jamba currently does not support prefix caching"
363
vllm/model_executor/models/llama.py
+20 -10
@@ -28,7 +28,7 @@ from transformers import LlamaConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
31 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -494,15 +494,15 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
494
495 def __init__(
496 self,
497 - config: LlamaConfig,
498 - cache_config: Optional[CacheConfig] = None,
499 - quant_config: Optional[QuantizationConfig] = None,
500 - lora_config: Optional[LoRAConfig] = None,
497 + vllm_config: VllmConfig,
498 prefix: str = "",
502 - pooler_config: Optional[PoolerConfig] = None,
499 ) -> None:
500 super().__init__()
505 -
501 + config = vllm_config.model_config.hf_config
502 + cache_config = vllm_config.cache_config
503 + quant_config = vllm_config.quant_config
504 + lora_config = vllm_config.lora_config
505 + pooler_config = vllm_config.model_config.pooler_config
506 self.config = config
507 self.lora_config = lora_config
508
@@ -654,12 +654,22 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
654
655 def __init__(
656 self,
657 - pooler_config: Optional[PoolerConfig] = None,
658 - **kwargs,
657 + vllm_config: VllmConfig,
658 + prefix: str = "",
659 ) -> None:
660 super().__init__()
661
662 - self.model = LlamaModel(**kwargs)
662 + config = vllm_config.model_config.hf_config
663 + cache_config = vllm_config.cache_config
664 + quant_config = vllm_config.quant_config
665 + lora_config = vllm_config.lora_config
666 + pooler_config = vllm_config.model_config.pooler_config
667 +
668 + self.model = LlamaModel(config,
669 + cache_config,
670 + quant_config,
671 + lora_config,
672 + prefix=maybe_prefix(prefix, "model"))
673 self._pooler = Pooler.from_config_with_defaults(
674 pooler_config,
675 pooling_type=PoolingType.LAST,
vllm/model_executor/models/llava.py
+7 -8
@@ -9,7 +9,7 @@ from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9 PretrainedConfig, SiglipVisionConfig)
10
11 from vllm.attention import AttentionMetadata
12 -from vllm.config import CacheConfig, MultiModalConfig
12 +from vllm.config import VllmConfig
13 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14 InputContext)
15 from vllm.model_executor.layers.activation import get_act_fn
@@ -258,13 +258,13 @@ def init_vision_tower_for_llava(
258 @INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
259 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
260
261 - def __init__(self,
262 - config: LlavaConfig,
263 - multimodal_config: MultiModalConfig,
264 - cache_config: Optional[CacheConfig] = None,
265 - quant_config: Optional[QuantizationConfig] = None) -> None:
261 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
262 super().__init__()
263
264 + config = vllm_config.model_config.hf_config
265 + quant_config = vllm_config.quant_config
266 + multimodal_config = vllm_config.model_config.multimodal_config
267 +
268 self.config = config
269 self.multimodal_config = multimodal_config
270
@@ -290,8 +290,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
290
291 self.language_model = init_vllm_registered_model(
292 config.text_config,
293 - cache_config,
294 - quant_config,
293 + vllm_config=vllm_config,
294 prefix="language_model")
295
296 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/llava_next.py
+7 -10
@@ -11,11 +11,10 @@ from transformers.models.llava_next.modeling_llava_next import (
11 from typing_extensions import NotRequired
12
13 from vllm.attention import AttentionMetadata
14 -from vllm.config import CacheConfig, MultiModalConfig, PoolerConfig
14 +from vllm.config import VllmConfig
15 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16 InputContext)
17 from vllm.model_executor.layers.pooler import Pooler, PoolingType
18 -from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19 from vllm.model_executor.pooling_metadata import PoolingMetadata
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -282,13 +281,12 @@ def input_processor_for_llava_next(ctx: InputContext,
281 class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
282 SupportsPP):
283
285 - def __init__(self,
286 - config: LlavaNextConfig,
287 - multimodal_config: MultiModalConfig,
288 - cache_config: Optional[CacheConfig] = None,
289 - quant_config: Optional[QuantizationConfig] = None,
290 - pooler_config: Optional[PoolerConfig] = None) -> None:
284 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
285 super().__init__()
286 + config = vllm_config.model_config.hf_config
287 + quant_config = vllm_config.quant_config
288 + pooler_config = vllm_config.model_config.pooler_config
289 + multimodal_config = vllm_config.model_config.multimodal_config
290
291 self.config = config
292 self.multimodal_config = multimodal_config
@@ -308,8 +306,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
306
307 self.language_model = init_vllm_registered_model(
308 config.text_config,
311 - cache_config,
312 - quant_config,
309 + vllm_config=vllm_config,
310 prefix="language_model")
311
312 # The same model class supports both language generation and embedding
vllm/model_executor/models/llava_next_video.py
+6 -9
@@ -10,11 +10,10 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
10 SiglipVisionConfig)
11
12 from vllm.attention import AttentionMetadata
13 -from vllm.config import CacheConfig, MultiModalConfig
13 +from vllm.config import VllmConfig
14 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 InputContext, token_inputs)
16 from vllm.model_executor.layers.activation import get_act_fn
17 -from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.models.clip import CLIPVisionModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -254,12 +253,11 @@ class LlavaNextMultiModalProjector(nn.Module):
253 class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
254 SupportsPP):
255
257 - def __init__(self,
258 - config: LlavaNextVideoConfig,
259 - multimodal_config: MultiModalConfig,
260 - cache_config: Optional[CacheConfig] = None,
261 - quant_config: Optional[QuantizationConfig] = None) -> None:
256 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
257 super().__init__()
258 + config = vllm_config.model_config.hf_config
259 + quant_config = vllm_config.quant_config
260 + multimodal_config = vllm_config.model_config.multimodal_config
261
262 self.config = config
263 self.multimodal_config = multimodal_config
@@ -277,8 +275,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
275 projector_hidden_act=config.projector_hidden_act)
276 self.language_model = init_vllm_registered_model(
277 config.text_config,
280 - cache_config,
281 - quant_config,
278 + vllm_config=vllm_config,
279 prefix="language_model")
280
281 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/llava_onevision.py
+6 -9
@@ -14,11 +14,10 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
14 from typing_extensions import NotRequired
15
16 from vllm.attention import AttentionMetadata
17 -from vllm.config import CacheConfig, MultiModalConfig
17 +from vllm.config import VllmConfig
18 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19 InputContext, token_inputs)
20 from vllm.model_executor.layers.activation import get_act_fn
21 -from vllm.model_executor.layers.quantization import QuantizationConfig
21 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -405,12 +404,11 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
404 class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
405 SupportsPP):
406
408 - def __init__(self,
409 - config: LlavaOnevisionConfig,
410 - multimodal_config: MultiModalConfig,
411 - cache_config: Optional[CacheConfig] = None,
412 - quant_config: Optional[QuantizationConfig] = None) -> None:
407 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
408 super().__init__()
409 + config = vllm_config.model_config.hf_config
410 + quant_config = vllm_config.quant_config
411 + multimodal_config = vllm_config.model_config.multimodal_config
412
413 self.config = config
414 self.multimodal_config = multimodal_config
@@ -424,8 +422,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
422 self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
423 self.language_model = init_vllm_registered_model(
424 config.text_config,
427 - cache_config,
428 - quant_config,
425 + vllm_config=vllm_config,
426 prefix="language_model")
427 self.image_newline = nn.Parameter(
428 torch.empty(config.text_config.hidden_size))
vllm/model_executor/models/mamba.py
+8 -6
@@ -6,7 +6,7 @@ from torch import nn
6 from transformers import MambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 -from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -132,12 +132,14 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
132
133 def __init__(
134 self,
135 - config: MambaConfig,
136 - cache_config: Optional[CacheConfig] = None,
137 - quant_config: Optional[QuantizationConfig] = None,
138 - lora_config: Optional[LoRAConfig] = None,
139 - scheduler_config: Optional[SchedulerConfig] = None,
135 + vllm_config: VllmConfig,
136 + prefix: str = "",
137 ) -> None:
138 + config = vllm_config.model_config.hf_config
139 + cache_config = vllm_config.cache_config
140 + quant_config = vllm_config.quant_config
141 + lora_config = vllm_config.lora_config
142 + scheduler_config = vllm_config.scheduler_config
143 assert not cache_config.enable_prefix_caching, \
144 "Mamba does not support prefix caching"
145
vllm/model_executor/models/medusa.py
+3 -2
@@ -3,13 +3,13 @@ from typing import Iterable, List, Optional, Tuple
3 import torch
4 import torch.nn as nn
5
6 +from vllm.config import VllmConfig
7 from vllm.model_executor.layers.logits_processor import LogitsProcessor
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.model_executor.layers.vocab_parallel_embedding import (
10 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
11 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
12 from vllm.model_executor.sampling_metadata import SamplingMetadata
12 -from vllm.transformers_utils.configs.medusa import MedusaConfig
13
14
15 class ResidualBlock(nn.Module):
@@ -44,7 +44,8 @@ class Medusa(nn.Module):
44 in the draft checkpoint (using key token_map). Also, the draft config
45 needs to have truncated_vocab_size (=k) as an attribute."""
46
47 - def __init__(self, config: MedusaConfig, **_) -> None:
47 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
48 + config = vllm_config.model_config.hf_config
49 super().__init__()
50 self.config = config
51 self.blocks = nn.ModuleList([
vllm/model_executor/models/minicpm.py
+7 -5
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
@@ -463,12 +463,14 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
463
464 def __init__(
465 self,
466 - config: PretrainedConfig,
467 - cache_config: Optional[CacheConfig] = None,
468 - quant_config: Optional[QuantizationConfig] = None,
469 - lora_config: Optional[LoRAConfig] = None,
466 + vllm_config: VllmConfig,
467 + prefix: str = "",
468 ) -> None:
469 super().__init__()
470 + config = vllm_config.model_config.hf_config
471 + cache_config = vllm_config.cache_config
472 + quant_config = vllm_config.quant_config
473 + lora_config = vllm_config.lora_config
474
475 self.config = config
476 self.lora_config = lora_config
vllm/model_executor/models/minicpmv.py
+19 -29
@@ -34,7 +34,7 @@ from transformers import PretrainedConfig
34 from typing_extensions import NotRequired
35
36 from vllm.attention import AttentionMetadata
37 -from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
37 +from vllm.config import CacheConfig, VllmConfig
38 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39 InputContext, token_inputs)
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -385,11 +385,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
385
386 def __init__(
387 self,
388 - config: PretrainedConfig,
389 - multimodal_config: MultiModalConfig,
390 - cache_config: Optional[CacheConfig] = None,
391 - quant_config: Optional[QuantizationConfig] = None,
388 + vllm_config: VllmConfig,
389 + prefix: str = "",
390 ):
391 + config = vllm_config.model_config.hf_config
392 + multimodal_config = vllm_config.model_config.multimodal_config
393 + cache_config = vllm_config.cache_config
394 + quant_config = vllm_config.quant_config
395 super().__init__()
396 # All MiniCPM-V models disable `tie_word_embeddings` but
397 # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
@@ -701,12 +703,10 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
703
704 def __init__(
705 self,
704 - config: PretrainedConfig,
705 - multimodal_config: MultiModalConfig,
706 - cache_config: Optional[CacheConfig] = None,
707 - quant_config: Optional[QuantizationConfig] = None,
706 + vllm_config: VllmConfig,
707 + prefix: str = "",
708 ):
709 - super().__init__(config, multimodal_config, cache_config, quant_config)
709 + super().__init__(vllm_config)
710 assert self.version == (2, 0)
711
712 def init_llm(
@@ -867,13 +867,10 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
867
868 def __init__(
869 self,
870 - config: PretrainedConfig,
871 - multimodal_config: MultiModalConfig,
872 - cache_config: Optional[CacheConfig] = None,
873 - quant_config: Optional[QuantizationConfig] = None,
874 - lora_config: Optional[LoRAConfig] = None,
870 + vllm_config: VllmConfig,
871 + prefix: str = "",
872 ):
876 - super().__init__(config, multimodal_config, cache_config, quant_config)
873 + super().__init__(vllm_config)
874 assert self.version == (2, 5)
875
876 def init_llm(
@@ -1017,12 +1014,10 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1014
1015 def __init__(
1016 self,
1020 - config: PretrainedConfig,
1021 - multimodal_config: MultiModalConfig,
1022 - cache_config: Optional[CacheConfig] = None,
1023 - quant_config: Optional[QuantizationConfig] = None,
1017 + vllm_config: VllmConfig,
1018 + prefix: str = "",
1019 ):
1025 - super().__init__(config, multimodal_config, cache_config, quant_config)
1020 + super().__init__(vllm_config)
1021 assert self.version == (2, 6)
1022
1023 def init_llm(
@@ -1141,12 +1136,8 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1136 embedding_modules = {}
1137 embedding_padding_modules = []
1138
1144 - def __new__(cls,
1145 - config: PretrainedConfig,
1146 - multimodal_config: MultiModalConfig,
1147 - cache_config: Optional[CacheConfig] = None,
1148 - quant_config: Optional[QuantizationConfig] = None,
1149 - lora_config: Optional[LoRAConfig] = None):
1139 + def __new__(cls, vllm_config: VllmConfig, prefix: str = ""):
1140 + config = vllm_config.model_config.hf_config
1141 if not hasattr(config, "version"):
1142 if config.hidden_size == 2304 and config.query_num == 64:
1143 version = (2, 0)
@@ -1160,5 +1151,4 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1151 if instance_class is None:
1152 raise ValueError(
1153 "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1163 - return instance_class(config, multimodal_config, cache_config,
1164 - quant_config)
1154 + return instance_class(vllm_config, prefix=prefix)
vllm/model_executor/models/mixtral.py
+7 -6
@@ -28,7 +28,7 @@ from transformers import MixtralConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -334,13 +334,14 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
334
335 def __init__(
336 self,
337 - config: MixtralConfig,
338 - cache_config: Optional[CacheConfig] = None,
339 - quant_config: Optional[QuantizationConfig] = None,
340 - lora_config: Optional[LoRAConfig] = None,
337 + vllm_config: VllmConfig,
338 + prefix: str = "",
339 ) -> None:
340 super().__init__()
343 -
341 + config = vllm_config.model_config.hf_config
342 + cache_config = vllm_config.cache_config
343 + quant_config = vllm_config.quant_config
344 + lora_config = vllm_config.lora_config
345 self.config = config
346 self.lora_config = lora_config
347
vllm/model_executor/models/mixtral_quant.py
+6 -4
@@ -29,7 +29,7 @@ from torch import nn
29 from transformers import MixtralConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 -from vllm.config import CacheConfig
32 +from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
@@ -352,11 +352,13 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
352
353 def __init__(
354 self,
355 - config: MixtralConfig,
356 - cache_config: Optional[CacheConfig] = None,
357 - quant_config: Optional[QuantizationConfig] = None,
355 + vllm_config: VllmConfig,
356 + prefix: str = "",
357 ) -> None:
358 super().__init__()
359 + config = vllm_config.model_config.hf_config
360 + cache_config = vllm_config.cache_config
361 + quant_config = vllm_config.quant_config
362 self.config = config
363 self.quant_config = quant_config
364 self.model = MixtralModel(config, cache_config, quant_config)
vllm/model_executor/models/mllama.py
+9 -6
@@ -33,7 +33,7 @@ from transformers.models.mllama.processing_mllama import (
33 import vllm.distributed.parallel_state as ps
34 from vllm.attention import Attention, AttentionMetadata, AttentionType
35 from vllm.attention.ops.paged_attn import PagedAttention
36 -from vllm.config import CacheConfig, MultiModalConfig
36 +from vllm.config import CacheConfig, VllmConfig
37 from vllm.distributed import get_tensor_model_parallel_world_size
38 from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
39 InputContext, TokenInputs, token_inputs)
@@ -1108,12 +1108,15 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1108 "up_proj": ("gate_up_proj", 1),
1109 }
1110
1111 - def __init__(self,
1112 - config: config_mllama.MllamaConfig,
1113 - multimodal_config: MultiModalConfig,
1114 - cache_config: Optional[CacheConfig] = None,
1115 - quant_config: Optional[QuantizationConfig] = None):
1111 + def __init__(
1112 + self,
1113 + vllm_config: VllmConfig,
1114 + prefix: str = "",
1115 + ) -> None:
1116 super().__init__()
1117 + config = vllm_config.model_config.hf_config
1118 + cache_config = vllm_config.cache_config
1119 + quant_config = vllm_config.quant_config
1120 self.vocab_size = config.text_config.vocab_size
1121 self.hidden_size = config.text_config.hidden_size
1122 self.max_num_tiles = config.vision_config.max_num_tiles
vllm/model_executor/models/molmo.py
+8 -8
@@ -3,8 +3,7 @@ import re
3 from array import array
4 from dataclasses import dataclass
5 from functools import lru_cache, partial
6 -from typing import (Any, Iterable, List, Mapping, Optional, Tuple, TypedDict,
7 - Union)
6 +from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
7
8 import torch
9 from einops import rearrange
@@ -16,7 +15,7 @@ from transformers import PretrainedConfig
15 from vllm.attention import Attention, AttentionMetadata
16 from vllm.attention.selector import _Backend
17 from vllm.compilation.decorators import support_torch_compile
19 -from vllm.config import CacheConfig, MultiModalConfig
18 +from vllm.config import CacheConfig, VllmConfig
19 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
20 get_tensor_model_parallel_world_size,
21 split_tensor_along_last_dim,
@@ -1027,13 +1026,14 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1026
1027 def __init__(
1028 self,
1030 - config: PretrainedConfig,
1031 - multimodal_config: Optional[MultiModalConfig] = None,
1032 - cache_config: Optional[CacheConfig] = None,
1033 - quant_config: Optional[Mapping[str, Any]] = None,
1029 + vllm_config: VllmConfig,
1030 + prefix: str = "",
1031 ) -> None:
1032 super().__init__()
1036 -
1033 + config = vllm_config.model_config.hf_config
1034 + cache_config = vllm_config.cache_config
1035 + quant_config = vllm_config.quant_config
1036 + multimodal_config = vllm_config.model_config.multimodal_config
1037 self.config = config
1038 self.multimodal_config = multimodal_config
1039
vllm/model_executor/models/mpt.py
+7 -5
@@ -7,7 +7,7 @@ import torch.nn as nn
7
8 from vllm.attention import Attention, AttentionMetadata
9 from vllm.compilation.decorators import support_torch_compile
10 -from vllm.config import CacheConfig
10 +from vllm.config import CacheConfig, VllmConfig
11 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
12 get_tensor_model_parallel_world_size)
13 from vllm.model_executor.layers.activation import get_act_fn
@@ -269,11 +269,13 @@ class MPTForCausalLM(nn.Module, SupportsPP):
269
270 def __init__(
271 self,
272 - config: MPTConfig,
273 - cache_config: Optional[CacheConfig] = None,
274 - quant_config: Optional[QuantizationConfig] = None,
275 - ):
272 + vllm_config: VllmConfig,
273 + prefix: str = "",
274 + ) -> None:
275 super().__init__()
276 + config = vllm_config.model_config.hf_config
277 + cache_config = vllm_config.cache_config
278 + quant_config = vllm_config.quant_config
279 self.config = config
280 assert config.tie_word_embeddings
281 self.quant_config = quant_config
vllm/model_executor/models/nemotron.py
+7 -6
@@ -27,7 +27,7 @@ from torch import nn
27
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.compilation.decorators import support_torch_compile
30 -from vllm.config import CacheConfig, LoRAConfig
30 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
31 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -403,13 +403,14 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
403
404 def __init__(
405 self,
406 - config: NemotronConfig,
407 - cache_config: Optional[CacheConfig] = None,
408 - quant_config: Optional[QuantizationConfig] = None,
409 - lora_config: Optional[LoRAConfig] = None,
406 + vllm_config: VllmConfig,
407 + prefix: str = "",
408 ) -> None:
409 super().__init__()
412 -
410 + config = vllm_config.model_config.hf_config
411 + cache_config = vllm_config.cache_config
412 + quant_config = vllm_config.quant_config
413 + lora_config = vllm_config.lora_config
414 assert isinstance(config, NemotronConfig)
415
416 self.config = config
vllm/model_executor/models/olmo.py
+9 -5
@@ -28,7 +28,7 @@ from transformers import OlmoConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig
31 +from vllm.config import CacheConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -291,11 +291,15 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
291 Extremely barebones HF model wrapper.
292 """
293
294 - def __init__(self,
295 - config: OlmoConfig,
296 - cache_config: Optional[CacheConfig] = None,
297 - quant_config: Optional[QuantizationConfig] = None):
294 + def __init__(
295 + self,
296 + vllm_config: VllmConfig,
297 + prefix: str = "",
298 + ) -> None:
299 super().__init__()
300 + config = vllm_config.model_config.hf_config
301 + cache_config = vllm_config.cache_config
302 + quant_config = vllm_config.quant_config
303 self.config = config
304 self.model = OlmoModel(config, cache_config, quant_config)
305 if config.tie_word_embeddings:
vllm/model_executor/models/olmoe.py
+6 -4
@@ -18,7 +18,7 @@ from transformers import PretrainedConfig
18
19 from vllm.attention import Attention, AttentionMetadata
20 from vllm.compilation.decorators import support_torch_compile
21 -from vllm.config import CacheConfig
21 +from vllm.config import CacheConfig, VllmConfig
22 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
23 from vllm.model_executor.layers.fused_moe import FusedMoE
24 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -311,11 +311,13 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
311
312 def __init__(
313 self,
314 - config: PretrainedConfig,
315 - cache_config: Optional[CacheConfig] = None,
316 - quant_config: Optional[QuantizationConfig] = None,
314 + vllm_config: VllmConfig,
315 + prefix: str = "",
316 ) -> None:
317 super().__init__()
318 + config = vllm_config.model_config.hf_config
319 + cache_config = vllm_config.cache_config
320 + quant_config = vllm_config.quant_config
321 self.config = config
322 self.quant_config = quant_config
323 self.model = OlmoeModel(config, cache_config, quant_config)
vllm/model_executor/models/opt.py
+7 -5
@@ -24,7 +24,7 @@ from transformers import OPTConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 from vllm.compilation.decorators import support_torch_compile
27 -from vllm.config import CacheConfig
27 +from vllm.config import CacheConfig, VllmConfig
28 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -344,11 +344,13 @@ class OPTForCausalLM(nn.Module, SupportsPP):
344
345 def __init__(
346 self,
347 - config: OPTConfig,
348 - cache_config: Optional[CacheConfig] = None,
349 - quant_config: Optional[QuantizationConfig] = None,
347 + vllm_config: VllmConfig,
348 prefix: str = "",
351 - ):
349 + ) -> None:
350 + super().__init__()
351 + config = vllm_config.model_config.hf_config
352 + cache_config = vllm_config.cache_config
353 + quant_config = vllm_config.quant_config
354 super().__init__()
355 self.config = config
356 self.quant_config = quant_config
vllm/model_executor/models/orion.py
+6 -4
@@ -11,7 +11,7 @@ from transformers import PretrainedConfig
11
12 from vllm.attention import Attention, AttentionMetadata
13 from vllm.compilation.decorators import support_torch_compile
14 -from vllm.config import CacheConfig
14 +from vllm.config import CacheConfig, VllmConfig
15 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16 from vllm.model_executor.layers.activation import SiluAndMul
17 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -270,11 +270,13 @@ class OrionForCausalLM(nn.Module, SupportsPP):
270
271 def __init__(
272 self,
273 - config: PretrainedConfig,
274 - cache_config: Optional[CacheConfig] = None,
275 - quant_config: Optional[QuantizationConfig] = None,
273 + vllm_config: VllmConfig,
274 + prefix: str = "",
275 ) -> None:
276 super().__init__()
277 + config = vllm_config.model_config.hf_config
278 + cache_config = vllm_config.cache_config
279 + quant_config = vllm_config.quant_config
280 self.config = config
281 self.quant_config = quant_config
282 self.model = OrionModel(config, cache_config, quant_config)
vllm/model_executor/models/paligemma.py
+16 -14
@@ -6,13 +6,11 @@ from torch import nn
6 from transformers import PaliGemmaConfig
7
8 from vllm.attention import AttentionMetadata
9 -from vllm.config import CacheConfig, MultiModalConfig
9 +from vllm.config import VllmConfig
10 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
11 InputContext, token_inputs)
12 from vllm.logger import init_logger
13 -from vllm.model_executor.layers.quantization import QuantizationConfig
13 from vllm.model_executor.layers.sampler import SamplerOutput
15 -from vllm.model_executor.models.gemma import GemmaForCausalLM
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 from vllm.multimodal import MULTIMODAL_REGISTRY
16 from vllm.multimodal.utils import cached_get_tokenizer
@@ -21,7 +19,8 @@ from vllm.sequence import IntermediateTensors
19 from .interfaces import SupportsMultiModal, SupportsPP
20 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
21 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
24 -from .utils import AutoWeightsLoader, merge_multimodal_embeddings
22 +from .utils import (AutoWeightsLoader, init_vllm_registered_model,
23 + merge_multimodal_embeddings)
24
25 logger = init_logger(__name__)
26
@@ -132,13 +131,15 @@ class PaliGemmaMultiModalProjector(nn.Module):
131 class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
132 SupportsPP):
133
135 - def __init__(self,
136 - config: PaliGemmaConfig,
137 - multimodal_config: MultiModalConfig,
138 - cache_config: Optional[CacheConfig] = None,
139 - quant_config: Optional[QuantizationConfig] = None) -> None:
134 + def __init__(
135 + self,
136 + vllm_config: VllmConfig,
137 + prefix: str = "",
138 + ) -> None:
139 super().__init__()
141 -
140 + config = vllm_config.model_config.hf_config
141 + quant_config = vllm_config.quant_config
142 + multimodal_config = vllm_config.model_config.multimodal_config
143 self.config = config
144 self.multimodal_config = multimodal_config
145
@@ -150,10 +151,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
151 projection_dim=config.vision_config.projection_dim)
152
153 self.quant_config = quant_config
153 - self.language_model = GemmaForCausalLM(config.text_config,
154 - cache_config,
155 - quant_config,
156 - prefix="language_model")
154 + config.text_config.architectures = ["GemmaForCausalLM"]
155 + self.language_model = init_vllm_registered_model(
156 + config.text_config,
157 + vllm_config=vllm_config,
158 + prefix="language_model")
159 logit_scale = getattr(config, "logit_scale", 1.0)
160 self.language_model.logits_processor.scale *= logit_scale
161
vllm/model_executor/models/persimmon.py
+9 -5
@@ -27,7 +27,7 @@ from transformers import PersimmonConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.compilation.decorators import support_torch_compile
30 -from vllm.config import CacheConfig
30 +from vllm.config import CacheConfig, VllmConfig
31 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -265,11 +265,15 @@ class PersimmonModel(nn.Module):
265
266 class PersimmonForCausalLM(nn.Module, SupportsPP):
267
268 - def __init__(self,
269 - config: PersimmonConfig,
270 - cache_config: Optional[CacheConfig] = None,
271 - quant_config: Optional[QuantizationConfig] = None):
268 + def __init__(
269 + self,
270 + vllm_config: VllmConfig,
271 + prefix: str = "",
272 + ) -> None:
273 super().__init__()
274 + config = vllm_config.model_config.hf_config
275 + cache_config = vllm_config.cache_config
276 + quant_config = vllm_config.quant_config
277 self.config = config
278 self.vocab_size = config.vocab_size
279 self.model = PersimmonModel(config,
vllm/model_executor/models/phi.py
+8 -7
@@ -42,7 +42,7 @@ from transformers import PhiConfig
42
43 from vllm.attention import Attention, AttentionMetadata
44 from vllm.compilation.decorators import support_torch_compile
45 -from vllm.config import CacheConfig, LoRAConfig
45 +from vllm.config import CacheConfig, VllmConfig
46 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
47 from vllm.model_executor.layers.activation import get_act_fn
48 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -279,13 +279,14 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
279
280 def __init__(
281 self,
282 - config: PhiConfig,
283 - cache_config: Optional[CacheConfig] = None,
284 - quant_config: Optional[QuantizationConfig] = None,
285 - lora_config: Optional[LoRAConfig] = None,
286 - ):
282 + vllm_config: VllmConfig,
283 + prefix: str = "",
284 + ) -> None:
285 super().__init__()
288 -
286 + config = vllm_config.model_config.hf_config
287 + cache_config = vllm_config.cache_config
288 + quant_config = vllm_config.quant_config
289 + lora_config = vllm_config.lora_config
290 self.config = config
291 # lm_head use bias, cannot share word embeddings
292 assert not config.tie_word_embeddings
vllm/model_executor/models/phi3_small.py
+7 -6
@@ -6,7 +6,7 @@ from torch import nn
6 from transformers.configuration_utils import PretrainedConfig
7
8 from vllm.attention import Attention, AttentionMetadata
9 -from vllm.config import CacheConfig, LoRAConfig
9 +from vllm.config import CacheConfig, VllmConfig
10 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
11 get_tensor_model_parallel_world_size)
12 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -365,12 +365,13 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
365
366 def __init__(
367 self,
368 - config: PretrainedConfig,
369 - cache_config: Optional[CacheConfig] = None,
370 - quant_config: Optional[QuantizationConfig] = None,
371 - lora_config: Optional[LoRAConfig] = None,
372 - ):
368 + vllm_config: VllmConfig,
369 + prefix: str = "",
370 + ) -> None:
371 super().__init__()
372 + config = vllm_config.model_config.hf_config
373 + cache_config = vllm_config.cache_config
374 + quant_config = vllm_config.quant_config
375 self.config = config
376 self.quant_config = quant_config
377 self.model = Phi3SmallModel(config, cache_config, quant_config)
vllm/model_executor/models/phi3v.py
+12 -11
@@ -25,8 +25,7 @@ from PIL import Image
25 from transformers import CLIPVisionConfig, PretrainedConfig
26
27 from vllm.attention import AttentionMetadata
28 -from vllm.config import (CacheConfig, ModelConfig, MultiModalConfig,
29 - PoolerConfig)
28 +from vllm.config import ModelConfig, VllmConfig
29 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30 InputContext, token_inputs)
31 from vllm.logger import init_logger
@@ -526,14 +525,16 @@ def input_processor_for_phi3v(ctx: InputContext,
525 @INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
526 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
527
529 - def __init__(self,
530 - config: PretrainedConfig,
531 - multimodal_config: MultiModalConfig,
532 - cache_config: Optional[CacheConfig] = None,
533 - quant_config: Optional[QuantizationConfig] = None,
534 - pooler_config: Optional[PoolerConfig] = None) -> None:
528 + def __init__(
529 + self,
530 + vllm_config: VllmConfig,
531 + prefix: str = "",
532 + ) -> None:
533 super().__init__()
536 -
534 + config = vllm_config.model_config.hf_config
535 + quant_config = vllm_config.quant_config
536 + pooler_config = vllm_config.model_config.pooler_config
537 + multimodal_config = vllm_config.model_config.multimodal_config
538 self.config = config
539 self.multimodal_config = multimodal_config
540 self.image_token_id = _IMAGE_TOKEN_ID
@@ -552,8 +553,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
553
554 # The prefix is empty intentionally because default prefix of
555 # LlamaForCausalLM is "model"
555 - self.language_model = LlamaForCausalLM(config, cache_config,
556 - quant_config)
556 + self.language_model = LlamaForCausalLM(vllm_config=vllm_config,
557 + prefix="")
558
559 # The same model class supports both language generation and embedding
560 # because the architecture name is the same
vllm/model_executor/models/phimoe.py
+7 -6
@@ -28,7 +28,7 @@ from transformers.configuration_utils import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 -from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -531,13 +531,14 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
531
532 def __init__(
533 self,
534 - config: PhiMoEConfig,
535 - cache_config: Optional[CacheConfig] = None,
536 - quant_config: Optional[QuantizationConfig] = None,
537 - lora_config: Optional[LoRAConfig] = None,
534 + vllm_config: VllmConfig,
535 + prefix: str = "",
536 ) -> None:
537 super().__init__()
540 -
538 + config = vllm_config.model_config.hf_config
539 + cache_config = vllm_config.cache_config
540 + quant_config = vllm_config.quant_config
541 + lora_config = vllm_config.lora_config
542 self.config = config
543 self.lora_config = lora_config
544
vllm/model_executor/models/pixtral.py
+10 -10
@@ -9,14 +9,14 @@ import torch.nn as nn
9 import torch.nn.functional as F
10 from mistral_common.protocol.instruct.messages import ImageChunk
11 from PIL import Image
12 -from transformers import PixtralVisionConfig, PretrainedConfig
12 +from transformers import PixtralVisionConfig
13 from transformers.models.pixtral.image_processing_pixtral import (
14 _num_image_tokens)
15 from transformers.models.pixtral.modeling_pixtral import (
16 PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
17
18 from vllm.attention import AttentionMetadata
19 -from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
19 +from vllm.config import ModelConfig, VllmConfig
20 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 InputContext, token_inputs)
22 from vllm.model_executor.layers.activation import get_act_and_mul_fn
@@ -152,13 +152,14 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
152 class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
153 SupportsPP):
154
155 - def __init__(self,
156 - config: PretrainedConfig,
157 - multimodal_config: MultiModalConfig,
158 - cache_config: Optional[CacheConfig] = None,
159 - quant_config: Optional[QuantizationConfig] = None) -> None:
155 + def __init__(
156 + self,
157 + vllm_config: VllmConfig,
158 + prefix: str = "",
159 + ) -> None:
160 super().__init__()
161 -
161 + config = vllm_config.model_config.hf_config
162 + multimodal_config = vllm_config.model_config.multimodal_config
163 self.config = config
164 self.multimodal_config = multimodal_config
165
@@ -174,8 +175,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
175 # init MistralForCausalLM
176 self.language_model = init_vllm_registered_model(
177 config.text_config,
177 - cache_config,
178 - quant_config,
178 + vllm_config=vllm_config,
179 prefix="language_model")
180
181 self.vision_encoder = VisionTransformer(self.vision_args)
vllm/model_executor/models/qwen.py
+14 -17
@@ -20,7 +20,7 @@ from transformers import PretrainedConfig
20
21 from vllm.attention import Attention, AttentionMetadata
22 from vllm.compilation.decorators import support_torch_compile
23 -from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
23 +from vllm.config import CacheConfig, VllmConfig
24 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
26 InputContext, token_inputs)
@@ -867,13 +867,14 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
867
868 def __init__(
869 self,
870 - config: PretrainedConfig,
871 - multimodal_config: MultiModalConfig,
872 - cache_config: Optional[CacheConfig] = None,
873 - quant_config: Optional[QuantizationConfig] = None,
874 - lora_config: Optional[LoRAConfig] = None,
875 - ):
870 + vllm_config: VllmConfig,
871 + prefix: str = "",
872 + ) -> None:
873 super().__init__()
874 + config = vllm_config.model_config.hf_config
875 + cache_config = vllm_config.cache_config
876 + quant_config = vllm_config.quant_config
877 + multimodal_config = vllm_config.model_config.multimodal_config
878 self.config = config
879 self.multimodal_config = multimodal_config
880 self.quant_config = quant_config
@@ -1064,17 +1065,13 @@ class QWenLMHeadModel(QWenBaseModel, SupportsLoRA):
1065
1066 def __new__(
1067 cls,
1067 - config: PretrainedConfig,
1068 - multimodal_config: MultiModalConfig,
1069 - cache_config: Optional[CacheConfig] = None,
1070 - quant_config: Optional[QuantizationConfig] = None,
1071 - lora_config: Optional[LoRAConfig] = None,
1072 - ):
1068 + vllm_config: VllmConfig,
1069 + prefix: str = "",
1070 + ) -> None:
1071 + config = vllm_config.model_config.hf_config
1072 # Initialize VL
1073 if hasattr(config, "visual"):
1075 - return QWenVL(config, multimodal_config, cache_config,
1076 - quant_config, lora_config)
1074 + return QWenVL(vllm_config)
1075 # Initialize LLM
1076 else:
1079 - return QWenLLM(config, multimodal_config, cache_config,
1080 - quant_config, lora_config)
1077 + return QWenLLM(vllm_config)
vllm/model_executor/models/qwen2.py
+7 -7
@@ -29,7 +29,7 @@ from transformers import Qwen2Config
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -405,12 +405,14 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
405
406 def __init__(
407 self,
408 - config: Qwen2Config,
409 - cache_config: Optional[CacheConfig] = None,
410 - quant_config: Optional[QuantizationConfig] = None,
411 - lora_config: Optional[LoRAConfig] = None,
408 + vllm_config: VllmConfig,
409 prefix: str = "",
410 ) -> None:
411 + super().__init__()
412 + config = vllm_config.model_config.hf_config
413 + cache_config = vllm_config.cache_config
414 + quant_config = vllm_config.quant_config
415 + lora_config = vllm_config.lora_config
416 # TODO (@robertgshaw2): see if this can be moved out
417 if (cache_config.sliding_window is not None
418 and hasattr(config, "max_window_layers")):
@@ -423,8 +425,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
425 config.num_hidden_layers,
426 ))
427
426 - super().__init__()
427 -
428 self.config = config
429 self.lora_config = lora_config
430
vllm/model_executor/models/qwen2_audio.py
+11 -10
@@ -26,16 +26,14 @@ import librosa
26 import numpy as np
27 import torch
28 import torch.nn as nn
29 -from transformers import Qwen2AudioConfig, Qwen2AudioEncoder
29 +from transformers import Qwen2AudioEncoder
30
31 from vllm.attention import AttentionMetadata
32 -from vllm.config import CacheConfig, MultiModalConfig
32 +from vllm.config import VllmConfig
33 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
34 InputContext, token_inputs)
35 from vllm.logger import init_logger
36 from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 -from vllm.model_executor.layers.quantization.base_config import (
38 - QuantizationConfig)
37 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
39 from vllm.model_executor.model_loader.weight_utils import (
@@ -266,13 +264,16 @@ def input_mapper_for_qwen2_audio(
264 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
265 SupportsPP):
266
269 - def __init__(self,
270 - config: Qwen2AudioConfig,
271 - multimodal_config: MultiModalConfig,
272 - cache_config: Optional[CacheConfig] = None,
273 - quant_config: Optional[QuantizationConfig] = None) -> None:
267 + def __init__(
268 + self,
269 + vllm_config: VllmConfig,
270 + prefix: str = "",
271 + ) -> None:
272 super().__init__()
275 -
273 + config = vllm_config.model_config.hf_config
274 + cache_config = vllm_config.cache_config
275 + quant_config = vllm_config.quant_config
276 + multimodal_config = vllm_config.model_config.multimodal_config
277 self.config = config
278 self.multimodal_config = multimodal_config
279
vllm/model_executor/models/qwen2_cls.py
+9 -11
@@ -8,14 +8,11 @@ from typing import Iterable, List, Optional, Tuple
8
9 import torch
10 from torch import nn
11 -from transformers import Qwen2Config
11
12 from vllm.attention import AttentionMetadata
14 -from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
13 +from vllm.config import VllmConfig
14 from vllm.model_executor.layers.linear import RowParallelLinear
15 from vllm.model_executor.layers.pooler import Pooler, PoolingType
17 -from vllm.model_executor.layers.quantization.base_config import (
18 - QuantizationConfig)
16 from vllm.model_executor.models.qwen2 import Qwen2Model
17 from vllm.model_executor.pooling_metadata import PoolingMetadata
18 from vllm.sequence import IntermediateTensors, PoolerOutput
@@ -48,12 +45,15 @@ class Qwen2ForSequenceClassification(nn.Module):
45
46 def __init__(
47 self,
51 - config: Qwen2Config,
52 - cache_config: Optional[CacheConfig] = None,
53 - quant_config: Optional[QuantizationConfig] = None,
54 - lora_config: Optional[LoRAConfig] = None,
55 - pooler_config: Optional[PoolerConfig] = None,
48 + vllm_config: VllmConfig,
49 + prefix: str = "",
50 ) -> None:
51 + super().__init__()
52 + config = vllm_config.model_config.hf_config
53 + cache_config = vllm_config.cache_config
54 + quant_config = vllm_config.quant_config
55 + lora_config = vllm_config.lora_config
56 + pooler_config = vllm_config.model_config.pooler_config
57 # TODO (@robertgshaw2): see if this can be moved out
58 if (cache_config.sliding_window is not None
59 and hasattr(config, "max_window_layers")):
@@ -66,8 +66,6 @@ class Qwen2ForSequenceClassification(nn.Module):
66 config.num_hidden_layers,
67 ))
68
69 - super().__init__()
70 -
69 self.config = config
70 self.lora_config = lora_config
71
vllm/model_executor/models/qwen2_moe.py
+6 -4
@@ -30,7 +30,7 @@ from transformers import PretrainedConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 from vllm.compilation.decorators import support_torch_compile
33 -from vllm.config import CacheConfig
33 +from vllm.config import CacheConfig, VllmConfig
34 from vllm.distributed import (get_pp_group,
35 get_tensor_model_parallel_world_size,
36 tensor_model_parallel_all_reduce)
@@ -379,11 +379,13 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
379
380 def __init__(
381 self,
382 - config: PretrainedConfig,
383 - cache_config: Optional[CacheConfig] = None,
384 - quant_config: Optional[QuantizationConfig] = None,
382 + vllm_config: VllmConfig,
383 + prefix: str = "",
384 ) -> None:
385 super().__init__()
386 + config = vllm_config.model_config.hf_config
387 + cache_config = vllm_config.cache_config
388 + quant_config = vllm_config.quant_config
389 self.config = config
390 self.quant_config = quant_config
391 self.model = Qwen2MoeModel(config, cache_config, quant_config)
vllm/model_executor/models/qwen2_rm.py
+9 -10
@@ -7,14 +7,12 @@ from typing import Iterable, List, Optional, Tuple, Union
7
8 import torch
9 from torch import nn
10 -from transformers import Qwen2Config
10
11 from vllm.attention import AttentionMetadata
13 -from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
12 +from vllm.config import VllmConfig
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 RowParallelLinear)
15 from vllm.model_executor.layers.pooler import Pooler, PoolingType
17 -from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.pooling_metadata import PoolingMetadata
17 from vllm.sequence import IntermediateTensors, PoolerOutput
18
@@ -59,12 +57,15 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
57
58 def __init__(
59 self,
62 - config: Qwen2Config,
63 - cache_config: Optional[CacheConfig] = None,
64 - quant_config: Optional[QuantizationConfig] = None,
65 - lora_config: Optional[LoRAConfig] = None,
66 - pooler_config: Optional[PoolerConfig] = None,
60 + vllm_config: VllmConfig,
61 + prefix: str = "",
62 ) -> None:
63 + super().__init__()
64 + config = vllm_config.model_config.hf_config
65 + cache_config = vllm_config.cache_config
66 + quant_config = vllm_config.quant_config
67 + lora_config = vllm_config.lora_config
68 + pooler_config = vllm_config.model_config.pooler_config
69 # TODO (@robertgshaw2): see if this can be moved out
70 if (cache_config.sliding_window is not None
71 and hasattr(config, "max_window_layers")):
@@ -77,8 +78,6 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
78 config.num_hidden_layers,
79 ))
80
80 - super().__init__()
81 -
81 self.config = config
82 self.lora_config = lora_config
83
vllm/model_executor/models/qwen2_vl.py
+10 -9
@@ -40,7 +40,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
41 from vllm.attention import AttentionMetadata
42 from vllm.attention.selector import _Backend
43 -from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
43 +from vllm.config import VllmConfig
44 from vllm.distributed import get_pp_group, parallel_state
45 from vllm.distributed import utils as dist_utils
46 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
@@ -966,15 +966,16 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
966 embedding_modules = {}
967 embedding_padding_modules = []
968
969 - def __init__(self,
970 - config: Qwen2VLConfig,
971 - multimodal_config: MultiModalConfig,
972 - cache_config: Optional[CacheConfig] = None,
973 - quant_config: Optional[QuantizationConfig] = None,
974 - lora_config: Optional[LoRAConfig] = None) -> None:
975 -
969 + def __init__(
970 + self,
971 + vllm_config: VllmConfig,
972 + prefix: str = "",
973 + ) -> None:
974 super().__init__()
977 -
975 + config = vllm_config.model_config.hf_config
976 + cache_config = vllm_config.cache_config
977 + quant_config = vllm_config.quant_config
978 + multimodal_config = vllm_config.model_config.multimodal_config
979 assert not cache_config.enable_prefix_caching, \
980 "Qwen2-VL currently does not support prefix caching"
981
vllm/model_executor/models/solar.py
+7 -6
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.config import CacheConfig, LoRAConfig, VllmConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -411,13 +411,14 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411
412 def __init__(
413 self,
414 - config: PretrainedConfig,
415 - cache_config: Optional[CacheConfig] = None,
416 - quant_config: Optional[QuantizationConfig] = None,
417 - lora_config: Optional[LoRAConfig] = None,
414 + vllm_config: VllmConfig,
415 + prefix: str = "",
416 ) -> None:
417 super().__init__()
420 -
418 + config = vllm_config.model_config.hf_config
419 + cache_config = vllm_config.cache_config
420 + quant_config = vllm_config.quant_config
421 + lora_config = vllm_config.lora_config
422 self.config = config
423 self.lora_config = lora_config
424
vllm/model_executor/models/stablelm.py
+6 -4
@@ -25,7 +25,7 @@ from torch import nn
25 from transformers import PretrainedConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 -from vllm.config import CacheConfig
28 +from vllm.config import CacheConfig, VllmConfig
29 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import SiluAndMul
31 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -247,11 +247,13 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
247
248 def __init__(
249 self,
250 - config: PretrainedConfig,
251 - cache_config: Optional[CacheConfig] = None,
252 - quant_config: Optional[QuantizationConfig] = None,
250 + vllm_config: VllmConfig,
251 + prefix: str = "",
252 ) -> None:
253 super().__init__()
254 + config = vllm_config.model_config.hf_config
255 + cache_config = vllm_config.cache_config
256 + quant_config = vllm_config.quant_config
257 self.config = config
258 self.quant_config = quant_config
259 self.model = StableLMEpochModel(config, cache_config, quant_config)
vllm/model_executor/models/starcoder2.py
+9 -5
@@ -25,7 +25,7 @@ from transformers import Starcoder2Config
25
26 from vllm.attention import Attention, AttentionMetadata
27 from vllm.compilation.decorators import support_torch_compile
28 -from vllm.config import CacheConfig
28 +from vllm.config import CacheConfig, VllmConfig
29 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -245,11 +245,15 @@ class Starcoder2Model(nn.Module):
245
246 class Starcoder2ForCausalLM(nn.Module, SupportsPP):
247
248 - def __init__(self,
249 - config: Starcoder2Config,
250 - cache_config: Optional[CacheConfig] = None,
251 - quant_config: Optional[QuantizationConfig] = None):
248 + def __init__(
249 + self,
250 + vllm_config: VllmConfig,
251 + prefix: str = "",
252 + ) -> None:
253 super().__init__()
254 + config = vllm_config.model_config.hf_config
255 + cache_config = vllm_config.cache_config
256 + quant_config = vllm_config.quant_config
257 self.config = config
258 self.model = Starcoder2Model(config,
259 cache_config,
vllm/model_executor/models/ultravox.py
+9 -11
@@ -15,12 +15,11 @@ from transformers.models.whisper import WhisperFeatureExtractor
15 from transformers.models.whisper.modeling_whisper import WhisperEncoder
16
17 from vllm.attention import AttentionMetadata
18 -from vllm.config import CacheConfig, MultiModalConfig
18 +from vllm.config import VllmConfig
19 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20 InputContext, token_inputs)
21 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
22 from vllm.model_executor.layers.layernorm import RMSNorm
23 -from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.model_loader.loader import DefaultModelLoader
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -340,12 +339,14 @@ class ModifiedWhisperEncoder(WhisperEncoder):
339 @INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox)
340 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
341
343 - def __init__(self,
344 - config: UltravoxConfig,
345 - multimodal_config: MultiModalConfig,
346 - cache_config: Optional[CacheConfig] = None,
347 - quant_config: Optional["QuantizationConfig"] = None):
342 + def __init__(
343 + self,
344 + vllm_config: VllmConfig,
345 + prefix: str = "",
346 + ) -> None:
347 super().__init__()
348 + config = vllm_config.model_config.hf_config
349 + multimodal_config = vllm_config.model_config.multimodal_config
350 self.config = config
351 self.multi_modal_config = multimodal_config
352 assert self.multi_modal_config
@@ -361,10 +362,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
362 ))
363 self.multi_modal_projector = UltravoxProjector(config)
364 self.language_model = init_vllm_registered_model(
364 - config.text_config,
365 - cache_config,
366 - quant_config,
367 - prefix="language_model")
365 + config.text_config, vllm_config, prefix="language_model")
366 if config.text_model_id is not None:
367 self.secondary_weights.append(
368 DefaultModelLoader.Source(model_or_path=config.text_model_id,
vllm/model_executor/models/utils.py
+6 -21
@@ -11,11 +11,8 @@ from transformers import PretrainedConfig
11 import vllm.envs as envs
12 from vllm.attention.selector import (_Backend, backend_name_to_enum,
13 get_global_forced_attn_backend)
14 -from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
15 - SchedulerConfig)
14 +from vllm.config import VllmConfig
15 from vllm.logger import init_logger
17 -from vllm.model_executor.layers.quantization import QuantizationConfig
18 -from vllm.model_executor.model_loader.loader import build_model
16 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.model_executor.models import ModelRegistry
18 from vllm.multimodal.base import MultiModalPlaceholderMap, NestedTensors
@@ -236,12 +233,7 @@ class AutoWeightsLoader:
233
234 def init_vllm_registered_model(
235 hf_config: PretrainedConfig,
239 - cache_config: Optional[CacheConfig],
240 - quant_config: Optional[QuantizationConfig],
241 - *,
242 - lora_config: Optional[LoRAConfig] = None,
243 - multimodal_config: Optional[MultiModalConfig] = None,
244 - scheduler_config: Optional[SchedulerConfig] = None,
236 + vllm_config: VllmConfig,
237 prefix: str = "",
238 ) -> nn.Module:
239 """
@@ -249,18 +241,11 @@ def init_vllm_registered_model(
241 based on the arguments passed to the outer vLLM model.
242 """
243 model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures)
244 + import copy
245 + copied_config = copy.deepcopy(vllm_config)
246 + copied_config.model_config.hf_config = hf_config
247
253 - return build_model(
254 - model_class,
255 - None,
256 - hf_config,
257 - cache_config,
258 - quant_config,
259 - lora_config=lora_config,
260 - multimodal_config=multimodal_config,
261 - scheduler_config=scheduler_config,
262 - prefix=prefix,
263 - )
248 + return model_class(vllm_config=copied_config, prefix=prefix)
249
250
251 @overload
vllm/model_executor/models/xverse.py
+13 -9
@@ -27,7 +27,7 @@ from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.compilation.decorators import support_torch_compile
30 -from vllm.config import CacheConfig, LoRAConfig
30 +from vllm.config import CacheConfig, VllmConfig
31 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -225,13 +225,14 @@ class XverseModel(nn.Module):
225
226 def __init__(
227 self,
228 - config: PretrainedConfig,
229 - cache_config: Optional[CacheConfig] = None,
230 - quant_config: Optional[QuantizationConfig] = None,
231 - lora_config: Optional[LoRAConfig] = None,
228 + vllm_config: VllmConfig,
229 prefix: str = "",
230 ) -> None:
231 super().__init__()
232 + config = vllm_config.model_config.hf_config
233 + cache_config = vllm_config.cache_config
234 + quant_config = vllm_config.quant_config
235 + lora_config = vllm_config.lora_config
236 self.config = config
237 self.padding_idx = config.pad_token_id
238 lora_vocab = (lora_config.lora_extra_vocab_size *
@@ -316,13 +317,16 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
317
318 def __init__(
319 self,
319 - config: PretrainedConfig,
320 - cache_config: Optional[CacheConfig] = None,
321 - quant_config: Optional[QuantizationConfig] = None,
322 - lora_config: Optional[LoRAConfig] = None,
320 + vllm_config: VllmConfig,
321 + prefix: str = "",
322 ) -> None:
323 super().__init__()
324
325 + config = vllm_config.model_config.hf_config
326 + cache_config = vllm_config.cache_config
327 + quant_config = vllm_config.quant_config
328 + lora_config = vllm_config.lora_config
329 +
330 self.config = config
331 self.lora_config = lora_config
332
vllm/plugins/__init__.py
-12
@@ -61,15 +61,3 @@ def set_compilation_config(config: Optional[CompilationConfig]):
61
62 def get_compilation_config() -> Optional[CompilationConfig]:
63 return _compilation_config
64 -
65 -
66 -_vllm_config: Optional[VllmConfig] = None
67 -
68 -
69 -def set_vllm_config(config: Optional[VllmConfig]):
70 - global _vllm_config
71 - _vllm_config = config
72 -
73 -
74 -def get_vllm_config() -> Optional[VllmConfig]:
75 - return _vllm_config