@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ca5f82c2a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 22:35 UTC

[Misc] Remove redundant config definitions (#21891)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jul 30, 2025 at 14:54 UTC 2ca5f82c2a8152ba67eaa033fbdb479d28f4cc3b

23 files changed +54 -1910

vllm/model_executor/models/aimv2.py

+11 -11

@@ -8,6 +8,7 @@ from typing import Optional

9 import torch

10 import torch.nn as nn

11 +from transformers import PretrainedConfig

13 from vllm.attention.layer import MultiHeadAttention

14 from vllm.distributed import get_tensor_model_parallel_world_size

@@ -20,13 +21,12 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

21 from vllm.model_executor.layers.quantization.base_config import (

22 QuantizationConfig)

23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

23 -from vllm.transformers_utils.configs.ovis import AIMv2Config

26 class AIMv2SwiGLUFFN(nn.Module):

28 - def __init__(self, config: AIMv2Config, quant_config: QuantizationConfig,

29 - prefix: str):

28 + def __init__(self, config: PretrainedConfig,

29 + quant_config: QuantizationConfig, prefix: str):

30 super().__init__()

31 hidden_features = config.intermediate_size

32 in_features = config.hidden_size

@@ -57,7 +57,7 @@ class AIMv2SwiGLUFFN(nn.Module):

58 class AIMv2PatchEmbed(nn.Module):

60 - def __init__(self, config: AIMv2Config):

60 + def __init__(self, config: PretrainedConfig):

61 super().__init__()

62 self.proj = nn.Conv2d(

63 config.num_channels,

@@ -75,7 +75,7 @@ class AIMv2PatchEmbed(nn.Module):

76 class AIMv2ViTPreprocessor(nn.Module):

78 - def __init__(self, config: AIMv2Config):

78 + def __init__(self, config: PretrainedConfig):

79 super().__init__()

80 num_patches = (config.image_size // config.patch_size)**2

@@ -93,8 +93,8 @@ class AIMv2ViTPreprocessor(nn.Module):

94 class AIMv2Attention(nn.Module):

96 - def __init__(self, config: AIMv2Config, quant_config: QuantizationConfig,

97 - prefix: str):

96 + def __init__(self, config: PretrainedConfig,

97 + quant_config: QuantizationConfig, prefix: str):

98 super().__init__()

99 self.config = config

100 self.embed_dim = config.hidden_size

@@ -141,8 +141,8 @@ class AIMv2Attention(nn.Module):

141

142 class AIMv2Block(nn.Module):

143

144 - def __init__(self, config: AIMv2Config, quant_config: QuantizationConfig,

145 - prefix: str):

144 + def __init__(self, config: PretrainedConfig,

145 + quant_config: QuantizationConfig, prefix: str):

146 super().__init__()

147 self.attn = AIMv2Attention(config,

148 quant_config=quant_config,

@@ -163,7 +163,7 @@ class AIMv2Transformer(nn.Module):

163

164 def __init__(

165 self,

166 - config: AIMv2Config,

166 + config: PretrainedConfig,

167 quant_config: QuantizationConfig,

168 *,

169 require_post_norm: Optional[bool] = None,

@@ -193,7 +193,7 @@ class AIMv2Transformer(nn.Module):

193 class AIMv2Model(torch.nn.Module):

194

195 def __init__(self,

196 - config: AIMv2Config,

196 + config: PretrainedConfig,

197 quant_config: QuantizationConfig,

198 *,

199 require_post_norm: Optional[bool] = None,

vllm/model_executor/models/dbrx.py

+7 -7

@@ -6,6 +6,7 @@ from typing import Optional, Union

7 import torch

8 import torch.nn as nn

9 +from transformers import PretrainedConfig

11 from vllm.attention import Attention

12 from vllm.config import CacheConfig, VllmConfig

@@ -24,7 +25,6 @@ from vllm.model_executor.model_loader.weight_utils import (

25 default_weight_loader, maybe_remap_kv_scale_name)

26 from vllm.model_executor.sampling_metadata import SamplingMetadata

27 from vllm.sequence import IntermediateTensors

27 -from vllm.transformers_utils.configs.dbrx import DbrxConfig

29 from .interfaces import SupportsPP

30 from .utils import (AutoWeightsLoader, is_pp_missing_parameter,

@@ -39,7 +39,7 @@ class DbrxRouter(nn.Module):

40 def __init__(

41 self,

42 - config: DbrxConfig,

42 + config: PretrainedConfig,

43 params_dtype: Optional[torch.dtype] = None,

44 ):

45 super().__init__()

@@ -63,7 +63,7 @@ class DbrxExperts(FusedMoE):

64 def __init__(

65 self,

66 - config: DbrxConfig,

66 + config: PretrainedConfig,

67 quant_config: Optional[QuantizationConfig] = None,

68 params_dtype: Optional[torch.dtype] = None,

69 prefix: str = "",

@@ -138,7 +138,7 @@ class DbrxMoE(nn.Module):

138

139 def __init__(

140 self,

141 - config: DbrxConfig,

141 + config: PretrainedConfig,

142 quant_config: Optional[QuantizationConfig] = None,

143 params_dtype: Optional[torch.dtype] = None,

144 prefix: str = "",

@@ -169,7 +169,7 @@ class DbrxAttention(nn.Module):

169

170 def __init__(

171 self,

172 - config: DbrxConfig,

172 + config: PretrainedConfig,

173 cache_config: Optional[CacheConfig] = None,

174 quant_config: Optional[QuantizationConfig] = None,

175 prefix: str = "",

@@ -249,7 +249,7 @@ class DbrxFusedNormAttention(nn.Module):

249

250 def __init__(

251 self,

252 - config: DbrxConfig,

252 + config: PretrainedConfig,

253 cache_config: Optional[CacheConfig] = None,

254 quant_config: Optional[QuantizationConfig] = None,

255 prefix: str = "",

@@ -284,7 +284,7 @@ class DbrxBlock(nn.Module):

284

285 def __init__(

286 self,

287 - config: DbrxConfig,

287 + config: PretrainedConfig,

288 cache_config: Optional[CacheConfig] = None,

289 quant_config: Optional[QuantizationConfig] = None,

290 prefix: str = "",

vllm/model_executor/models/exaone.py

+4 -4

@@ -30,6 +30,7 @@ from typing import Any, Optional, Union

31 import torch

32 from torch import nn

33 +from transformers import PretrainedConfig

35 from vllm.attention import Attention

36 from vllm.compilation.decorators import support_torch_compile

@@ -49,7 +50,6 @@ from vllm.model_executor.model_loader.weight_utils import (

50 default_weight_loader, maybe_remap_kv_scale_name)

51 from vllm.model_executor.sampling_metadata import SamplingMetadata

52 from vllm.sequence import IntermediateTensors

52 -from vllm.transformers_utils.configs.exaone import ExaoneConfig

54 from .interfaces import SupportsLoRA, SupportsPP

55 from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,

@@ -99,7 +99,7 @@ class ExaoneAttention(nn.Module):

100 def __init__(

101 self,

102 - config: ExaoneConfig,

102 + config: PretrainedConfig,

103 hidden_size: int,

104 num_heads: int,

105 num_kv_heads: int,

@@ -194,7 +194,7 @@ class ExaoneBlockAttention(nn.Module):

194

195 def __init__(

196 self,

197 - config: ExaoneConfig,

197 + config: PretrainedConfig,

198 hidden_size: int,

199 num_heads: int,

200 num_kv_heads: int,

@@ -236,7 +236,7 @@ class ExaoneDecoderLayer(nn.Module):

236

237 def __init__(

238 self,

239 - config: ExaoneConfig,

239 + config: PretrainedConfig,

240 cache_config: Optional[CacheConfig] = None,

241 quant_config: Optional[QuantizationConfig] = None,

242 prefix: str = "",

vllm/model_executor/models/exaone4.py

+3 -3

@@ -26,6 +26,7 @@ from typing import Any, Optional, Union

27 import torch

28 from torch import nn

29 +from transformers import PretrainedConfig

31 from vllm.attention import Attention

32 from vllm.compilation.decorators import support_torch_compile

@@ -45,7 +46,6 @@ from vllm.model_executor.model_loader.weight_utils import (

46 default_weight_loader, maybe_remap_kv_scale_name)

47 from vllm.model_executor.sampling_metadata import SamplingMetadata

48 from vllm.sequence import IntermediateTensors

48 -from vllm.transformers_utils.configs.exaone4 import Exaone4Config

50 from .interfaces import SupportsLoRA, SupportsPP

51 from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,

@@ -96,7 +96,7 @@ class Exaone4Attention(nn.Module):

97 def __init__(

98 self,

99 - config: Exaone4Config,

99 + config: PretrainedConfig,

100 hidden_size: int,

101 num_heads: int,

102 num_kv_heads: int,

@@ -224,7 +224,7 @@ class Exaone4DecoderLayer(nn.Module):

224

225 def __init__(

226 self,

227 - config: Exaone4Config,

227 + config: PretrainedConfig,

228 cache_config: Optional[CacheConfig] = None,

229 quant_config: Optional[QuantizationConfig] = None,

230 prefix: str = "",

vllm/model_executor/models/keye.py

-3

@@ -980,9 +980,6 @@ class KeyeMultiModalDataParser(MultiModalDataParser):

980

981 class KeyeProcessingInfo(BaseProcessingInfo):

982

983 - def get_hf_config(self):

984 - return self.ctx.get_hf_config(PretrainedConfig)

985 -

983 def get_hf_processor(

984 self,

985 *,

vllm/model_executor/models/minimax_vl_01.py

+3 -4

@@ -5,7 +5,7 @@ from typing import Literal, Optional, TypedDict, Union, cast

6 import torch

7 import torch.nn as nn

8 -from transformers import BatchFeature

8 +from transformers import BatchFeature, PretrainedConfig

10 from vllm.config import VllmConfig

11 from vllm.jsontree import json_map_leaves

@@ -17,7 +17,6 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata

17 from vllm.multimodal import MULTIMODAL_REGISTRY

18 from vllm.multimodal.inputs import MultiModalFieldConfig

19 from vllm.sequence import IntermediateTensors

20 -from vllm.transformers_utils.configs.minimax_vl_01 import MiniMaxVL01Config

21 from .clip import CLIPVisionModel

22 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

@@ -90,8 +89,8 @@ class MiniMaxVL01DummyInputsBuilder(LlavaDummyInputsBuilder):

90 class MiniMaxVL01ProcessingInfo(LlavaNextProcessingInfo):

93 - def get_hf_config(self):

94 - return self.ctx.get_hf_config(MiniMaxVL01Config)

92 + def get_hf_config(self): # Need to override the config type

93 + return self.ctx.get_hf_config(PretrainedConfig)

95 def get_hf_processor(self, **kwargs: object):

96 hf_processor = self.ctx.get_hf_processor(**kwargs)

vllm/model_executor/models/mpt.py

+4 -4

@@ -8,6 +8,7 @@ from typing import Optional, Union

9 import torch

10 import torch.nn as nn

11 +from transformers import PretrainedConfig

13 from vllm.attention import Attention

14 from vllm.compilation.decorators import support_torch_compile

@@ -25,7 +26,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (

26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

27 from vllm.model_executor.sampling_metadata import SamplingMetadata

28 from vllm.sequence import IntermediateTensors

28 -from vllm.transformers_utils.configs.mpt import MPTConfig

30 from .interfaces import SupportsPP

31 from .utils import (AutoWeightsLoader, is_pp_missing_parameter,

@@ -50,7 +50,7 @@ class MPTAttention(nn.Module):

51 def __init__(

52 self,

53 - config: MPTConfig,

53 + config: PretrainedConfig,

54 cache_config: Optional[CacheConfig] = None,

55 quant_config: Optional[QuantizationConfig] = None,

56 prefix: str = "",

@@ -144,7 +144,7 @@ class MPTMLP(nn.Module):

144

145 def __init__(

146 self,

147 - config: MPTConfig,

147 + config: PretrainedConfig,

148 quant_config: Optional[QuantizationConfig] = None,

149 ):

150 super().__init__()

@@ -176,7 +176,7 @@ class MPTBlock(nn.Module):

176

177 def __init__(

178 self,

179 - config: MPTConfig,

179 + config: PretrainedConfig,

180 cache_config: Optional[CacheConfig] = None,

181 quant_config: Optional[QuantizationConfig] = None,

182 prefix: str = "",

vllm/model_executor/models/ovis.py

+4 -9

@@ -25,7 +25,7 @@ import torch

25 import torch.nn as nn

26 from torch import Tensor

27 from torch.nn.functional import gumbel_softmax, pad, softmax

28 -from transformers import BaseImageProcessor, BatchFeature

28 +from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

30 from vllm.config import VllmConfig

31 from vllm.model_executor.layers.linear import ReplicatedLinear

@@ -48,8 +48,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

48 BaseProcessingInfo, PromptReplacement)

49 from vllm.multimodal.profiling import BaseDummyInputsBuilder

50 from vllm.sequence import IntermediateTensors

51 -from vllm.transformers_utils.configs.ovis import (BaseVisualTokenizerConfig,

52 - OvisConfig)

51 from vllm.transformers_utils.processors.ovis import OvisProcessor

53 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

@@ -83,7 +81,7 @@ class VisualTokenizer(torch.nn.Module):

82 def __init__(

83 self,

86 - config: BaseVisualTokenizerConfig,

84 + config: PretrainedConfig,

85 quant_config: Optional[QuantizationConfig] = None,

86 prefix: str = "",

87 ):

@@ -107,7 +105,7 @@ class VisualTokenizer(torch.nn.Module):

105

106 def _init_backbone(

107 self,

110 - config: BaseVisualTokenizerConfig,

108 + config: PretrainedConfig,

109 quant_config: Optional[QuantizationConfig] = None,

110 prefix: str = "",

111 ) -> nn.Module:

@@ -247,9 +245,6 @@ class VisualEmbedding(torch.nn.Embedding):

245

246 class OvisProcessingInfo(BaseProcessingInfo):

247

250 - def get_hf_config(self):

251 - return self.ctx.get_hf_config(OvisConfig)

252 -

248 def get_hf_processor(self, **kwargs):

249 return self.ctx.get_hf_processor(

250 OvisProcessor,

@@ -417,7 +412,7 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):

412 config = vllm_config.model_config.hf_config

413 quant_config = vllm_config.quant_config

414

420 - self.config: OvisConfig = config

415 + self.config: PretrainedConfig = config

416 self.llm = init_vllm_registered_model(

417 vllm_config=vllm_config.with_hf_config(config.get_text_config()),

418 prefix=maybe_prefix(prefix, "llm"),

vllm/transformers_utils/config.py

+5 -23

@@ -29,19 +29,13 @@ from vllm import envs

29 from vllm.logger import init_logger

30 # yapf conflicts with isort for this block

31 # yapf: disable

32 -from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,

33 - DbrxConfig, DeepseekVLV2Config,

34 - EAGLEConfig, Exaone4Config,

35 - ExaoneConfig, JAISConfig,

32 +from vllm.transformers_utils.configs import (ChatGLMConfig, DeepseekVLV2Config,

33 + EAGLEConfig, JAISConfig,

34 KimiVLConfig, MedusaConfig,

37 - MiniMaxText01Config,

38 - MiniMaxVL01Config, MllamaConfig,

39 - MLPSpeculatorConfig, MPTConfig,

35 + MllamaConfig, MLPSpeculatorConfig,

36 Nemotron_Nano_VL_Config,

41 - NemotronConfig, NVLM_D_Config,

42 - OvisConfig, RWConfig,

43 - SkyworkR1VChatConfig, SolarConfig,

44 - Telechat2Config, UltravoxConfig)

37 + NemotronConfig, RWConfig,

38 + UltravoxConfig)

39 # yapf: enable

40 from vllm.transformers_utils.configs.mistral import adapt_config_dict

41 from vllm.transformers_utils.utils import check_gguf_file

@@ -77,28 +71,16 @@ _CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = {

72 _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {

73 "chatglm": ChatGLMConfig,

80 - "cohere2": Cohere2Config,

81 - "dbrx": DbrxConfig,

74 "deepseek_vl_v2": DeepseekVLV2Config,

75 "kimi_vl": KimiVLConfig,

76 "Llama_Nemotron_Nano_VL": Nemotron_Nano_VL_Config,

85 - "mpt": MPTConfig,

77 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)

78 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)

79 "jais": JAISConfig,

80 "mlp_speculator": MLPSpeculatorConfig,

81 "medusa": MedusaConfig,

82 "eagle": EAGLEConfig,

92 - "exaone": ExaoneConfig,

93 - "exaone4": Exaone4Config,

94 - "minimax_text_01": MiniMaxText01Config,

95 - "minimax_vl_01": MiniMaxVL01Config,

83 "nemotron": NemotronConfig,

97 - "NVLM_D": NVLM_D_Config,

98 - "ovis": OvisConfig,

99 - "solar": SolarConfig,

100 - "skywork_chat": SkyworkR1VChatConfig,

101 - "telechat": Telechat2Config,

84 "ultravox": UltravoxConfig,

85 **_CONFIG_REGISTRY_OVERRIDE_HF

86 }

vllm/transformers_utils/configs/__init__.py

+6 -24

@@ -1,13 +1,15 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +Model configs may be defined in this directory for the following reasons:

5 +

6 +- There is no configuration file defined by HF Hub or Transformers library.

7 +- There is a need to override the existing config to support vLLM.

8 +"""

10 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig

5 -from vllm.transformers_utils.configs.cohere2 import Cohere2Config

6 -from vllm.transformers_utils.configs.dbrx import DbrxConfig

11 from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config

12 from vllm.transformers_utils.configs.eagle import EAGLEConfig

9 -from vllm.transformers_utils.configs.exaone import ExaoneConfig

10 -from vllm.transformers_utils.configs.exaone4 import Exaone4Config

13 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and

14 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the

15 # `FalconConfig` class from the official HuggingFace transformers library.

@@ -15,36 +17,21 @@ from vllm.transformers_utils.configs.falcon import RWConfig

17 from vllm.transformers_utils.configs.jais import JAISConfig

18 from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig

19 from vllm.transformers_utils.configs.medusa import MedusaConfig

18 -from vllm.transformers_utils.configs.minimax_text_01 import MiniMaxText01Config

19 -from vllm.transformers_utils.configs.minimax_vl_01 import MiniMaxVL01Config

20 from vllm.transformers_utils.configs.mllama import MllamaConfig

21 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig

22 from vllm.transformers_utils.configs.moonvit import MoonViTConfig

23 -from vllm.transformers_utils.configs.mpt import MPTConfig

23 from vllm.transformers_utils.configs.nemotron import NemotronConfig

24 from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig

25 from vllm.transformers_utils.configs.nemotron_vl import Nemotron_Nano_VL_Config

27 -from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config

28 -from vllm.transformers_utils.configs.ovis import OvisConfig

29 -from vllm.transformers_utils.configs.skyworkr1v import SkyworkR1VChatConfig

30 -from vllm.transformers_utils.configs.solar import SolarConfig

31 -from vllm.transformers_utils.configs.telechat2 import Telechat2Config

26 from vllm.transformers_utils.configs.ultravox import UltravoxConfig

28 __all__ = [

29 "ChatGLMConfig",

36 - "Cohere2Config",

37 - "DbrxConfig",

30 "DeepseekVLV2Config",

39 - "MPTConfig",

31 "RWConfig",

32 "JAISConfig",

33 "MedusaConfig",

34 "EAGLEConfig",

44 - "ExaoneConfig",

45 - "Exaone4Config",

46 - "MiniMaxText01Config",

47 - "MiniMaxVL01Config",

35 "MllamaConfig",

36 "MLPSpeculatorConfig",

37 "MoonViTConfig",

@@ -52,10 +39,5 @@ __all__ = [

39 "NemotronConfig",

40 "NemotronHConfig",

41 "Nemotron_Nano_VL_Config",

55 - "NVLM_D_Config",

56 - "OvisConfig",

57 - "SkyworkR1VChatConfig",

58 - "SolarConfig",

59 - "Telechat2Config",

42 "UltravoxConfig",

43 ]

vllm/transformers_utils/configs/cohere2.py deleted

-195

@@ -1,195 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# ruff: noqa

5 -

6 -# Adapted from

7 -# https://github.com/huggingface/transformers/blob/main/src/transformers/models/cohere2/configuration_cohere2.py

8 -from transformers import PretrainedConfig

9 -from transformers.modeling_rope_utils import rope_config_validation

10 -

11 -

12 -class Cohere2Config(PretrainedConfig):

13 - r"""

14 - This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere

15 - model according to the specified arguments, defining the model architecture.

16 -

17 - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the

18 - documentation from [`PretrainedConfig`] for more information. Instantiating a configuration

19 - with the defaults will yield a similar configuration to that of the [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) model.

20 -

21 -

22 - Args:

23 - vocab_size (`int`, *optional*, defaults to 256000):

24 - Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the

25 - `inputs_ids` passed when calling [`CohereModel`]

26 - hidden_size (`int`, *optional*, defaults to 8192):

27 - Dimension of the hidden representations.

28 - intermediate_size (`int`, *optional*, defaults to 22528):

29 - Dimension of the MLP representations.

30 - logit_scale (`float`, *optional*, defaults to 0.0625):

31 - The scaling factor for the output logits.

32 - num_hidden_layers (`int`, *optional*, defaults to 40):

33 - Number of hidden layers in the Transformer decoder.

34 - num_attention_heads (`int`, *optional*, defaults to 64):

35 - Number of attention heads for each attention layer in the Transformer decoder.

36 - num_key_value_heads (`int`, *optional*):

37 - This is the number of key_value heads that should be used to implement Grouped Query Attention. If

38 - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if

39 - `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When

40 - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed

41 - by meanpooling all the original heads within that group. For more details checkout [this

42 - paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to

43 - `num_attention_heads`.

44 - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):

45 - The non-linear activation function (function or string) in the decoder.

46 - max_position_embeddings (`int`, *optional*, defaults to 8192):

47 - The maximum sequence length that this model might ever be used with.

48 - initializer_range (`float`, *optional*, defaults to 0.02):

49 - The standard deviation of the truncated_normal_initializer for initializing all weight matrices.

50 - layer_norm_eps (`float`, *optional*, defaults to 1e-05):

51 - The epsilon used by the layer normalization.

52 - use_cache (`bool`, *optional*, defaults to `True`):

53 - Whether or not the model should return the last key/values attentions (not used by all models). Only

54 - relevant if `config.is_decoder=True`.

55 - pad_token_id (`int`, *optional*, defaults to 0):

56 - Padding token id.

57 - bos_token_id (`int`, *optional*, defaults to 5):

58 - Beginning of stream token id.

59 - eos_token_id (`int`, *optional*, defaults to 255001):

60 - End of stream token id.

61 - tie_word_embeddings (`bool`, *optional*, defaults to `True`):

62 - Whether to tie weight embeddings

63 - rope_theta (`float`, *optional*, defaults to 10000.0):

64 - The base period of the RoPE embeddings.

65 - rope_scaling (`dict`, *optional*):

66 - Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type

67 - and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value

68 - accordingly.

69 - Expected contents:

70 - `rope_type` (`str`):

71 - The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',

72 - 'llama3'], with 'default' being the original RoPE implementation.

73 - `factor` (`float`, *optional*):

74 - Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In

75 - most scaling types, a `factor` of x will enable the model to handle sequences of length x *

76 - original maximum pre-trained length.

77 - `original_max_position_embeddings` (`int`, *optional*):

78 - Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during

79 - pretraining.

80 - `attention_factor` (`float`, *optional*):

81 - Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention

82 - computation. If unspecified, it defaults to value recommended by the implementation, using the

83 - `factor` field to infer the suggested value.

84 - `beta_fast` (`float`, *optional*):

85 - Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear

86 - ramp function. If unspecified, it defaults to 32.

87 - `beta_slow` (`float`, *optional*):

88 - Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear

89 - ramp function. If unspecified, it defaults to 1.

90 - `short_factor` (`list[float]`, *optional*):

91 - Only used with 'longrope'. The scaling factor to be applied to short contexts (<

92 - `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden

93 - size divided by the number of attention heads divided by 2

94 - `long_factor` (`list[float]`, *optional*):

95 - Only used with 'longrope'. The scaling factor to be applied to long contexts (<

96 - `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden

97 - size divided by the number of attention heads divided by 2

98 - `low_freq_factor` (`float`, *optional*):

99 - Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE

100 - `high_freq_factor` (`float`, *optional*):

101 - Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE

102 - attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):

103 - Whether to use a bias in the query, key, value and output projection layers during self-attention.

104 - attention_dropout (`float`, *optional*, defaults to 0.0):

105 - The dropout ratio for the attention probabilities.

106 - sliding_window (`int`, *optional*, defaults to 4096):

107 - Size of the sliding window attention context.

108 - sliding_window_pattern (`int`, *optional*, defaults to 4):

109 - Pattern for the sliding window attention.

110 - cache_implementation (`str`, *optional*, defaults to `"hybrid"`): the cache type to be used with `generate`.

111 -

112 - ```python

113 - >>> from transformers import Cohere2Model, Cohere2Config

114 -

115 - >>> # Initializing a Cohere Nextmodel configuration

116 - >>> configuration = Cohere2Config()

117 -

118 - >>> # Initializing a model from the Cohere2 configuration

119 - >>> model = Cohere2Model(configuration) # doctest: +SKIP

120 -

121 - >>> # Accessing the model configuration

122 - >>> configuration = model.config # doctest: +SKIP

123 - ```

124 - """

125 -

126 - model_type = "cohere2"

127 - keys_to_ignore_at_inference = ["past_key_values"]

128 -

129 - def __init__(

130 - self,

131 - vocab_size=256000,

132 - hidden_size=8192,

133 - intermediate_size=22528,

134 - logit_scale=0.0625,

135 - num_hidden_layers=40,

136 - num_attention_heads=64,

137 - num_key_value_heads=None,

138 - hidden_act="silu",

139 - max_position_embeddings=8192,

140 - initializer_range=0.02,

141 - layer_norm_eps=1e-5,

142 - use_cache=True,

143 - pad_token_id=0,

144 - bos_token_id=5,

145 - eos_token_id=255001,

146 - tie_word_embeddings=True,

147 - rope_theta=10000.0,

148 - rope_scaling=None,

149 - attention_bias=False,

150 - attention_dropout=0.0,

151 - sliding_window=4096,

152 - sliding_window_pattern=4,

153 - cache_implementation="hybrid",

154 - **kwargs,

155 - ):

156 - self.vocab_size = vocab_size

157 - self.max_position_embeddings = max_position_embeddings

158 - self.hidden_size = hidden_size

159 - self.logit_scale = logit_scale

160 - self.intermediate_size = intermediate_size

161 - self.num_hidden_layers = num_hidden_layers

162 - self.num_attention_heads = num_attention_heads

163 -

164 - # for backward compatibility

165 - if num_key_value_heads is None:

166 - num_key_value_heads = num_attention_heads

167 -

168 - self.num_key_value_heads = num_key_value_heads

169 - self.hidden_act = hidden_act

170 - self.initializer_range = initializer_range

171 - self.layer_norm_eps = layer_norm_eps

172 - self.use_cache = use_cache

173 - self.rope_theta = rope_theta

174 - self.rope_scaling = rope_scaling

175 - self.attention_bias = attention_bias

176 - self.attention_dropout = attention_dropout

177 - self.sliding_window = sliding_window

178 - self.sliding_window_pattern = sliding_window_pattern

179 - # Need to specify head_dim in the config so it can be used in the attention forward functions

180 - self.head_dim = hidden_size // num_attention_heads

181 - self.cache_implementation = cache_implementation

182 -

183 - # Validate the correctness of rotary position embeddings parameters

184 - rope_config_validation(self)

185 -

186 - super().__init__(

187 - pad_token_id=pad_token_id,

188 - bos_token_id=bos_token_id,

189 - eos_token_id=eos_token_id,

190 - tie_word_embeddings=tie_word_embeddings,

191 - **kwargs,

192 - )

193 -

194 -

195 -__all__ = ["Cohere2Config"]

vllm/transformers_utils/configs/dbrx.py deleted

-280

@@ -1,280 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# yapf: disable

5 -# ruff: noqa: E501

6 -# coding=utf-8

7 -# Copied from

8 -# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py

9 -"""Dbrx configuration."""

10 -

11 -from typing import Any, Optional

12 -

13 -from transformers.configuration_utils import PretrainedConfig

14 -from transformers.utils import logging

15 -

16 -logger = logging.get_logger(__name__)

17 -

18 -DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore

19 -

20 -

21 -class DbrxAttentionConfig(PretrainedConfig):

22 - """Configuration class for Dbrx Attention.

23 -

24 - [`DbrxAttention`] class. It is used to instantiate attention layers

25 - according to the specified arguments, defining the layers architecture.

26 -

27 - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the

28 - documentation from [`PretrainedConfig`] for more information.

29 -

30 - Args:

31 - attn_pdrop (`float`, *optional*, defaults to 0.0):

32 - The dropout probability for the attention layers.

33 - clip_qkv (`float`, *optional*, defaults to None):

34 - If not `None`, clip the queries, keys, and values in the attention layer to this value.

35 - kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.

36 - rope_theta (float): The base frequency for rope.

37 - """

38 -

39 - def __init__(

40 - self,

41 - attn_pdrop: float = 0,

42 - clip_qkv: Optional[float] = None,

43 - kv_n_heads: int = 1,

44 - rope_theta: float = 10000.0,

45 - **kwargs: Any,

46 - ):

47 - super().__init__(**kwargs)

48 - self.attn_pdrop = attn_pdrop

49 - self.clip_qkv = clip_qkv

50 - self.kv_n_heads = kv_n_heads

51 - self.rope_theta = rope_theta

52 -

53 - for k in ["model_type"]:

54 - if k in kwargs:

55 - kwargs.pop(k)

56 - if len(kwargs) != 0:

57 - raise ValueError(f"Found unknown {kwargs=}")

58 -

59 - @classmethod

60 - def from_pretrained(

61 - cls, pretrained_model_name_or_path: str, **kwargs: Any

62 - ) -> "PretrainedConfig":

63 - cls._set_token_in_kwargs(kwargs)

64 -

65 - config_dict, kwargs = cls.get_config_dict(

66 - pretrained_model_name_or_path, **kwargs

67 - )

68 -

69 - if config_dict.get("model_type") == "dbrx":

70 - config_dict = config_dict["attn_config"]

71 -

72 - if (

73 - "model_type" in config_dict

74 - and hasattr(cls, "model_type")

75 - and config_dict["model_type"] != cls.model_type

76 - ):

77 - logger.warning(

78 - "You are using a model of type %s to instantiate a model of "

79 - "type %s. This is not supported for all configurations of "

80 - "models and can yield errors.",

81 - config_dict["model_type"], cls.model_type)

82 -

83 - return cls.from_dict(config_dict, **kwargs)

84 -

85 -

86 -class DbrxFFNConfig(PretrainedConfig):

87 - """Configuration class for Dbrx FFN.

88 -

89 - [`DbrxFFN`] class. It is used to instantiate feedforward layers according to

90 - the specified arguments, defining the layers architecture.

91 -

92 - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the

93 - documentation from [`PretrainedConfig`] for more information.

94 -

95 - Args:

96 - ffn_act_fn (dict, optional): A dict specifying activation function for the FFN.

97 - The dict should have a key 'name' with the value being the name of

98 - the activation function along with any additional keyword arguments.

99 - ffn_hidden_size (int, optional): The hidden size of the feedforward network.

100 - moe_num_experts (int, optional): The number of experts in the mixture of experts layer.

101 - moe_top_k (int, optional): The number of experts to use in the mixture of experts layer.

102 - moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer.

103 - moe_loss_weight (float, optional): The loss weight for the mixture of experts layer.

104 - moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights.

105 - uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment.

106 - This should only be used for benchmarking purposes.

107 - """

108 -

109 - def __init__(

110 - self,

111 - ffn_act_fn: Optional[dict] = None,

112 - ffn_hidden_size: int = 3584,

113 - moe_num_experts: int = 4,

114 - moe_top_k: int = 1,

115 - moe_jitter_eps: Optional[float] = None,

116 - moe_loss_weight: float = 0.01,

117 - moe_normalize_expert_weights: Optional[float] = 1,

118 - uniform_expert_assignment: bool = False,

119 - **kwargs: Any,

120 - ):

121 - super().__init__()

122 - if ffn_act_fn is None:

123 - ffn_act_fn = {"name": "silu"}

124 - self.ffn_act_fn = ffn_act_fn

125 - self.ffn_hidden_size = ffn_hidden_size

126 - self.moe_num_experts = moe_num_experts

127 - self.moe_top_k = moe_top_k

128 - self.moe_jitter_eps = moe_jitter_eps

129 - self.moe_loss_weight = moe_loss_weight

130 - self.moe_normalize_expert_weights = moe_normalize_expert_weights

131 - self.uniform_expert_assignment = uniform_expert_assignment

132 -

133 - for k in ["model_type"]:

134 - if k in kwargs:

135 - kwargs.pop(k)

136 - if len(kwargs) != 0:

137 - raise ValueError(f"Found unknown {kwargs=}")

138 -

139 - @classmethod

140 - def from_pretrained(

141 - cls, pretrained_model_name_or_path: str, **kwargs: Any

142 - ) -> "PretrainedConfig":

143 - cls._set_token_in_kwargs(kwargs)

144 -

145 - config_dict, kwargs = cls.get_config_dict(

146 - pretrained_model_name_or_path, **kwargs

147 - )

148 -

149 - if config_dict.get("model_type") == "dbrx":

150 - config_dict = config_dict["ffn_config"]

151 -

152 - if (

153 - "model_type" in config_dict

154 - and hasattr(cls, "model_type")

155 - and config_dict["model_type"] != cls.model_type

156 - ):

157 - logger.warning(

158 - "You are using a model of type %s to instantiate a model of "

159 - "type %s. This is not supported for all "

160 - "configurations of models and can yield errors.", config_dict["model_type"], cls.model_type)

161 -

162 - return cls.from_dict(config_dict, **kwargs)

163 -

164 -

165 -class DbrxConfig(PretrainedConfig):

166 - """Configuration class for Dbrx.

167 -

168 - [`DbrxModel`]. It is used to instantiate a Dbrx model according to the

169 - specified arguments, defining the model architecture.

170 -

171 - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the

172 - documentation from [`PretrainedConfig`] for more information.

173 -

174 -

175 - Args:

176 - d_model (`int`, *optional*, defaults to 6144):

177 - Dimensionality of the embeddings and hidden states.

178 - n_heads (`int`, *optional*, defaults to 48):

179 - Number of attention heads for each attention layer in the Transformer encoder.

180 - n_layers (`int`, *optional*, defaults to 40):

181 - Number of hidden layers in the Transformer encoder.

182 - max_seq_len (`int`, *optional*, defaults to 32768):

183 - The maximum sequence length of the model.

184 - vocab_size (`int`, *optional*, defaults to 100352):

185 - Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by

186 - the `inputs_ids` passed when calling [`DbrxModel`].

187 - resid_pdrop (`float`, *optional*, defaults to 0.0):

188 - The dropout probability applied to the attention output before combining with residual.

189 - emb_pdrop (`float`, *optional*, defaults to 0.0):

190 - The dropout probability for the embedding layer.

191 - attn_config (`dict`, *optional*):

192 - A dictionary used to configure the model's attention module.

193 - ffn_config (`dict`, *optional*):

194 - A dictionary used to configure the model's FFN module.

195 - use_cache (`bool`, *optional*, defaults to `False`):

196 - Whether or not the model should return the last key/values attentions (not used by all models).

197 - initializer_range (`float`, *optional*, defaults to 0.02):

198 - The standard deviation of the truncated_normal_initializer for initializing all weight matrices.

199 - output_router_logits (`bool`, *optional*, defaults to `False`):

200 - Whether or not the router logits should be returned by the model. Enabling this will also allow the model to output the auxiliary loss.

201 - router_aux_loss_coef (`float`, *optional*, defaults to 0.001):

202 - The aux loss factor for the total loss.

203 -

204 -

205 - Example:

206 - ```python

207 - >>> from transformers import DbrxConfig, DbrxModel

208 -

209 - >>> # Initializing a Dbrx configuration

210 - >>> configuration = DbrxConfig()

211 -

212 - >>> # Initializing a model (with random weights) from the configuration

213 - >>> model = DbrxModel(configuration)

214 -

215 - >>> # Accessing the model configuration

216 - >>> configuration = model.config

217 - ```

218 - """

219 -

220 - model_type = "dbrx"

221 - attribute_map = {

222 - "num_attention_heads": "n_heads",

223 - "hidden_size": "d_model",

224 - "num_hidden_layers": "n_layers",

225 - "max_position_embeddings": "max_seq_len",

226 - }

227 -

228 - def __init__(

229 - self,

230 - d_model: int = 2048,

231 - n_heads: int = 16,

232 - n_layers: int = 24,

233 - max_seq_len: int = 2048,

234 - vocab_size: int = 32000,

235 - resid_pdrop: float = 0.0,

236 - emb_pdrop: float = 0.0,

237 - attn_config: Optional[DbrxAttentionConfig] = None,

238 - ffn_config: Optional[DbrxFFNConfig] = None,

239 - use_cache: bool = True,

240 - initializer_range: float = 0.02,

241 - output_router_logits: bool = False,

242 - router_aux_loss_coef: float = 0.05,

243 - **kwargs: Any,

244 - ):

245 - if attn_config is None:

246 - self.attn_config = DbrxAttentionConfig()

247 - elif isinstance(attn_config, dict):

248 - self.attn_config = DbrxAttentionConfig(**attn_config)

249 - else:

250 - self.attn_config = attn_config

251 -

252 - if ffn_config is None:

253 - self.ffn_config = DbrxFFNConfig()

254 - elif isinstance(ffn_config, dict):

255 - self.ffn_config = DbrxFFNConfig(**ffn_config)

256 - else:

257 - self.ffn_config = ffn_config

258 -

259 - self.d_model = d_model

260 - self.n_heads = n_heads

261 - self.n_layers = n_layers

262 - self.max_seq_len = max_seq_len

263 - self.vocab_size = vocab_size

264 - self.resid_pdrop = resid_pdrop

265 - self.emb_pdrop = emb_pdrop

266 - self.use_cache = use_cache

267 - self.initializer_range = initializer_range

268 - self.output_router_logits = output_router_logits

269 - self.router_aux_loss_coef = router_aux_loss_coef

270 -

271 - tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)

272 - if tie_word_embeddings:

273 - raise ValueError(

274 - "tie_word_embeddings is not supported for Dbrx models."

275 - )

276 -

277 - super().__init__(

278 - tie_word_embeddings=tie_word_embeddings,

279 - **kwargs,

280 - )

vllm/transformers_utils/configs/exaone.py deleted

-190

@@ -1,190 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# Copied from

5 -# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/configuration_exaone.py

7 -#

8 -# Licensed under the Apache License, Version 2.0 (the "License");

9 -# you may not use this file except in compliance with the License.

10 -# You may obtain a copy of the License at

11 -#

12 -# http://www.apache.org/licenses/LICENSE-2.0

13 -#

14 -# Unless required by applicable law or agreed to in writing, software

15 -# distributed under the License is distributed on an "AS IS" BASIS,

16 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 -# See the License for the specific language governing permissions and

18 -# limitations under the License.

19 -"""Exaone model configuration"""

20 -

21 -from transformers.configuration_utils import PretrainedConfig

22 -from transformers.utils import logging

23 -

24 -logger = logging.get_logger(__name__)

25 -

26 -EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: dict[str, str] = {}

27 -

28 -

29 -class ExaoneConfig(PretrainedConfig):

30 - r"""

31 - This is the configuration class to store the configuration of a :class:

32 - `~transformers.ExaoneModel`. It is used to instantiate a GPT Lingvo model

33 - according to the specified arguments, defining the model architecture.

34 - Instantiating a configuration with the defaults will yield a similar

35 - configuration to that of the Exaone

36 -

37 - Configuration objects inherit from {class}`~transformers.PretrainedConfig`

38 - and can be used to control the model outputs. Read the documentation from :

39 - class:`~transformers.PretrainedConfig` for more information.

40 -

41 - Args:

42 - vocab_size ({obj}`int`, `optional`, defaults to 50257):

43 - Vocabulary size of the GPT Lingvo model. Defines the number of

44 - different tokens that can be represented by the {obj}`inputs_ids`

45 - passed when calling {class}`~transformers.ExaoneModel`. Vocabulary

46 - size of the model.

47 - Defines the different tokens that can be represented by the

48 - `inputs_ids` passed to the forward method of :class:

49 - `~transformers.EXAONEModel`.

50 - hidden_size ({obj}`int`, `optional`, defaults to 2048):

51 - Dimensionality of the encoder layers and the pooler layer.

52 - num_layers ({obj}`int`, `optional`, defaults to 24):

53 - Number of hidden layers in the Transformer encoder.

54 - num_attention_heads (`int`, *optional*, defaults to 32):

55 - Number of attention heads for each attention layer in the

56 - Transformer decoder.

57 - num_key_value_heads (`int`, *optional*):

58 - This is the number of key_value heads that should be used to

59 - implement Grouped Query Attention. If

60 - `num_key_value_heads=num_attention_heads`, the model will use Multi

61 - Head Attention (MHA), if `num_key_value_heads=1 the model will use

62 - Multi Query Attention (MQA) otherwise GQA is used. When

63 - converting a multi-head checkpoint to a GQA checkpoint,

64 - each group key and value head should be constructed by meanpooling

65 - all the original heads within that group. For more details checkout

66 - [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not

67 - specified, will default to `num_attention_heads`.

68 - rotary_pct (`float`, *optional*, defaults to 0.25):

69 - percentage of hidden dimensions to allocate to rotary embeddings

70 - intermediate_size ({obj}`int`, `optional`, defaults to 8192):

71 - Dimensionality of the "intermediate" (i.e., feed-forward) layer in

72 - the Transformer encoder.

73 - activation_function ({obj}`str` or {obj}`function`, `optional`,

74 - defaults to {obj}`"gelu_new"`):

75 - The non-linear activation function (function or string) in the

76 - encoder and pooler. If string, {obj}`"gelu"`, {obj}`"relu"`,

77 - {obj}`"selu"` and {obj}`"gelu_new"` are supported.

78 - embed_dropout ({obj}`float`, `optional`, defaults to 0.0):

79 - The dropout probabilitiy for all fully connected layers in the

80 - embeddings, encoder, and pooler.

81 - attention_dropout ({obj}`float`, `optional`, defaults to 0.0):

82 - The dropout ratio for the attention probabilities.

83 - max_position_embeddings ({obj}`int`, `optional`, defaults to 2048):

84 - The maximum sequence length that this model might ever be used with.

85 - Typically set this to something large just in case

86 - (e.g., 512 or 1024 or 2048).

87 - type_vocab_size ({obj}`int`, `optional`, defaults to 2):

88 - The vocabulary size of the {obj}`token_type_ids` passed when calling

89 - {class}`~transformers.EXAONEModel`.

90 - initializer_range ({obj}`float`, `optional`, defaults to 0.02):

91 - The standard deviation of the truncated_normal_initializer for

92 - initializing all weight matrices.

93 - layer_norm_epsilon ({obj}`float`, `optional`, defaults to 1e-5):

94 - The epsilon used by the layer normalization layers.

95 - use_cache ({obj}`bool`, `optional`, defaults to {obj}`True`):

96 - Whether or not the model should return the last key/values

97 - attentions (not used by all models).

98 - Only relevant if ``config.is_decoder=True``.

99 - gradient_checkpointing ({obj}`bool`, `optional`,

100 - defaults to {obj}`False`):

101 - If True, use gradient checkpointing to save memory at the expense

102 - of slower backward pass.

103 - Example::

104 -

105 - >>> from transformers import ExoneModel, ExaoneConfig

106 -

107 - >>> # Initializing a EXAONE configuration

108 - >>> configuration = ExaoneConfig()

109 -

110 - >>> # Initializing a model from configuration

111 - >>> model = ExoneModel(configuration)

112 -

113 - >>> # Accessing the model configuration

114 - >>> configuration = model.config

115 - """

116 -

117 - model_type = "exaone"

118 - keys_to_ignore_at_inference = ["past_key_values"]

119 - attribute_map = {"num_hidden_layers": "num_layers"}

120 -

121 - def __init__(

122 - self,

123 - vocab_size=102400,

124 - max_position_embeddings=2048,

125 - hidden_size=2048,

126 - num_layers=32,

127 - num_attention_heads=32,

128 - num_key_value_heads=None,

129 - intermediate_size=None,

130 - activation_function="silu",

131 - rotary_pct=0.25,

132 - resid_dropout=0.0,

133 - embed_dropout=0.0,

134 - attention_dropout=0.0,

135 - layer_norm_epsilon=1e-6,

136 - initializer_range=0.02,

137 - use_cache=True,

138 - bos_token_id=0,

139 - eos_token_id=2,

140 - tie_word_embeddings=True,

141 - **kwargs,

142 - ):

143 - super().__init__(

144 - bos_token_id=bos_token_id,

145 - eos_token_id=eos_token_id,

146 - tie_word_embeddings=tie_word_embeddings,

147 - **kwargs,

148 - )

149 -

150 - self.vocab_size = vocab_size

151 - self.max_position_embeddings = max_position_embeddings

152 - self.hidden_size = hidden_size

153 - self.num_layers = num_layers

154 - self.num_attention_heads = num_attention_heads

155 - self.num_hidden_layers = num_layers

156 - if num_key_value_heads is None:

157 - num_key_value_heads = num_attention_heads

158 - self.num_key_value_heads = num_key_value_heads

159 - if intermediate_size:

160 - self.intermediate_size = intermediate_size

161 - else:

162 - self.intermediate_size = hidden_size * 4

163 - self.activation_function = activation_function

164 - self.resid_dropout = resid_dropout

165 - self.embed_dropout = embed_dropout

166 - self.attention_dropout = attention_dropout

167 - self.layer_norm_epsilon = layer_norm_epsilon

168 - self.initializer_range = initializer_range

169 - self.use_cache = use_cache

170 - self.rotary_pct = rotary_pct

171 -

172 - self.bos_token_id = bos_token_id

173 - self.eos_token_id = eos_token_id

174 -

175 - self.use_logit_cap = kwargs.pop("use_logit_cap", False)

176 - self.ln_no_scale = kwargs.pop("ln_no_scale", False)

177 - self.use_gated = kwargs.pop("use_gated", False)

178 - self.use_emb_norm = kwargs.pop("use_emb_norm", False)

179 - self.use_rotary_pos = kwargs.pop("use_rotary_pos", False)

180 - self.rotary_type = kwargs.pop("rotary_type", None)

181 - self.scaling_factor = kwargs.pop("scaling_factor", 1)

182 - self.use_absolute_pos = kwargs.pop("use_absolute_pos", True)

183 - self.use_extra_logit = kwargs.pop("use_extra_logit", True)

184 - self.rotary_expand_length = kwargs.pop("rotary_expand_length", None)

185 - self.rotary_base = kwargs.pop("rotary_base", 10000.0)

186 - self.use_qkv_fuse = kwargs.pop("use_qkv_fuse", False)

187 - self.rescale_before_lm_head = kwargs.pop("rescale_before_lm_head",

188 - (rotary_pct == 0.25))

189 - if self.use_rotary_pos:

190 - self.use_absolute_pos = False

vllm/transformers_utils/configs/exaone4.py deleted

-252

@@ -1,252 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -# ruff: noqa: E501

4 -

5 -# Copied from

6 -# https://github.com/lgai-exaone/transformers/blob/add-exaone4/src/transformers/models/exaone4/configuration_exaone4.py

9 -#

10 -#

11 -# Licensed under the Apache License, Version 2.0 (the "License");

12 -# you may not use this file except in compliance with the License.

13 -# You may obtain a copy of the License at

14 -#

15 -# http://www.apache.org/licenses/LICENSE-2.0

16 -#

17 -# Unless required by applicable law or agreed to in writing, software

18 -# distributed under the License is distributed on an "AS IS" BASIS,

19 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

20 -# See the License for the specific language governing permissions and

21 -# limitations under the License.

22 -from transformers.configuration_utils import (PretrainedConfig,

23 - layer_type_validation)

24 -from transformers.utils import logging

25 -

26 -logger = logging.get_logger(__name__)

27 -

28 -

29 -def check_is_sliding(config, layer_idx):

30 - """

31 - Check if the current layer is a sliding window attention (local attention) layer.

32 - """

33 - if config.sliding_window is None:

34 - return False

35 - if config.layer_types is not None:

36 - return config.layer_types[layer_idx] == "sliding_attention"

37 - if isinstance(config.sliding_window_pattern, int):

38 - return ((layer_idx + 1) % config.sliding_window_pattern) != 0

39 - elif isinstance(config.sliding_window_pattern, str):

40 - assert isinstance(config.sliding_window, int), (

41 - f"Sliding window must be positive integer, but got {config.sliding_window}"

42 - )

43 - return (layer_idx != config.num_hidden_layers - 1

44 - and config.sliding_window_pattern[layer_idx % len(

45 - config.sliding_window_pattern)] == "L")

46 - else:

47 - logger.warning_once(

48 - "Sliding window is set, but none of `sliding_window_pattern` or `layer_types` is set. "

49 - "Defaulting to use 'full_attention' for all layers.")

50 - return False

51 -

52 -

53 -class Exaone4Config(PretrainedConfig):

54 - r"""

55 - This is the configuration class to store the configuration of a [`Exaone4Model`]. It is used to

56 - instantiate a EXAONE 4.0 model according to the specified arguments, defining the model architecture. Instantiating a

57 - configuration with the defaults will yield a similar configuration to that of the EXAONE-4.0-Instruct [LGAI-EXAONE/EXAONE-4.0-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-Instruct)

58 - NOTE: `EXAONE-4.0-Instruct` is a placeholder model ID. The exact model ID will be updated in the future.

59 -

60 - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model

61 - outputs. Read the documentation from [`PretrainedConfig`] for more information.

62 -

63 - Args:

64 - vocab_size (`int`, *optional*, defaults to 102400):

65 - Vocabulary size of the EXAONE 4.0 model. Defines the number of different tokens that can be represented by the

66 - `inputs_ids` passed when calling [`Exaone4Model`].

67 - hidden_size (`int`, *optional*, defaults to 4096):

68 - Dimension of the hidden representations.

69 - intermediate_size (`int`, *optional*, defaults to `hidden_size * 4`):

70 - Dimensionality of the MLP representations.

71 - num_hidden_layers (`int`, *optional*, defaults to 32):

72 - Number of hidden layers in the Transformer encoder.

73 - num_attention_heads (`int`, *optional*, defaults to 32):

74 - Number of attention heads for each attention layer in the Transformer decoder.

75 - num_key_value_heads (`int`, *optional*):

76 - This is the number of key_value heads that should be used to implement Grouped Query Attention. If

77 - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if

78 - `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When

79 - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed

80 - by meanpooling all the original heads within that group. For more details checkout [this

81 - paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to

82 - `num_attention_heads`.

83 - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):

84 - The non-linear activation function (function or string) in the decoder.

85 - max_position_embeddings (`int`, *optional*, defaults to 2048):

86 - The maximum sequence length that this model might ever be used with. Typically set this to something large

87 - just in case (e.g., 32768 for EXAONE 3.5).

88 - initializer_range (`float`, *optional*, defaults to 0.02):

89 - The standard deviation of the truncated_normal_initializer for initializing all weight matrices.

90 - rms_norm_eps (`float`, *optional*, defaults to 1e-05):

91 - The epsilon used by the layer normalization layers.

92 - use_cache (`bool`, *optional*, defaults to `True`):

93 - Whether or not the model should return the last key/values attentions (not used by all models). Only

94 - relevant if ``config.is_decoder=True``.

95 - bos_token_id (`int`, *optional*, defaults to 0):

96 - Beginning of stream token id.

97 - eos_token_id (`int`, *optional*, defaults to 2):

98 - End of stream token id.

99 - tie_word_embeddings (`bool`, *optional*, defaults to `False`):

100 - Whether to tie weight embeddings

101 - rope_theta (`float`, *optional*, defaults to 10000.0):

102 - The base period of the RoPE embeddings.

103 - rope_scaling (`Dict`, *optional*):

104 - Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type

105 - and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value

106 - accordingly.

107 - Expected contents:

108 - `rope_type` (`str`):

109 - The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',

110 - 'llama3'], with 'default' being the original RoPE implementation.

111 - `factor` (`float`, *optional*):

112 - Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In

113 - most scaling types, a `factor` of x will enable the model to handle sequences of length x *

114 - original maximum pre-trained length.

115 - `original_max_position_embeddings` (`int`, *optional*):

116 - Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during

117 - pretraining.

118 - `attention_factor` (`float`, *optional*):

119 - Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention

120 - computation. If unspecified, it defaults to value recommended by the implementation, using the

121 - `factor` field to infer the suggested value.

122 - `beta_fast` (`float`, *optional*):

123 - Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear

124 - ramp function. If unspecified, it defaults to 32.

125 - `beta_slow` (`float`, *optional*):

126 - Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear

127 - ramp function. If unspecified, it defaults to 1.

128 - `short_factor` (`List[float]`, *optional*):

129 - Only used with 'longrope'. The scaling factor to be applied to short contexts (<

130 - `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden

131 - size divided by the number of attention heads divided by 2

132 - `long_factor` (`List[float]`, *optional*):

133 - Only used with 'longrope'. The scaling factor to be applied to long contexts (<

134 - `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden

135 - size divided by the number of attention heads divided by 2

136 - `low_freq_factor` (`float`, *optional*):

137 - Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE

138 - `high_freq_factor` (`float`, *optional*):

139 - Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE

140 - attention_dropout (`float`, *optional*, defaults to 0.0):

141 - The dropout ratio for the attention probabilities.

142 - sliding_window (`int`, *optional*):

143 - The size of the sliding window for the sliding window attention.

144 - sliding_window_pattern (`str`, *optional*):

145 - The pattern to use for sliding window attention. Can be one of:

146 - - `None`: No sliding window attention is used

147 - - `int`: Every `sliding_window` layers, use global attention, else use local attention.

148 - - `str`: A sequence of "L" (local attention) and "G" (global attention) characters that defines the

149 - attention pattern. The pattern starts from layer 0 and repeats every `sliding_window` layers. The

150 - final layer always uses global attention regardless of the pattern.

151 - For instance, sliding_window_pattern="LLLG" same as sliding_window=4, which means:

152 - - Layer 0, 1, 2: local attention,

153 - - Layer 3: global attention,

154 - ...(repeated)

155 - layer_types (`list`, *optional*):

156 - Attention pattern for each layer. Prioritized over `sliding_window_pattern`.

157 -

158 - Example:

159 -

160 - ```python

161 - >>> from transformers import Exaone4Model, Exaone4Config

162 -

163 - >>> # Initializing a EXAONE configuration

164 - >>> configuration = Exaone4Config()

165 -

166 - >>> # Initializing a model from configuration

167 - >>> model = Exaone4Model(configuration)

168 -

169 - >>> # Accessing the model configuration

170 - >>> configuration = model.config

171 - ```"""

172 -

173 - model_type = "exaone4"

174 - keys_to_ignore_at_inference = ["past_key_values"]

175 - # Default tensor parallel plan for base model `LlamaModel`

176 - base_model_tp_plan = {

177 - "layers.*.self_attn.q_proj": "colwise",

178 - "layers.*.self_attn.k_proj": "colwise",

179 - "layers.*.self_attn.v_proj": "colwise",

180 - "layers.*.self_attn.o_proj": "rowwise",

181 - "layers.*.mlp.gate_proj": "colwise",

182 - "layers.*.mlp.up_proj": "colwise",

183 - "layers.*.mlp.down_proj": "rowwise",

184 - }

185 - base_model_pp_plan = {

186 - "embed_tokens": (["input_ids"], ["inputs_embeds"]),

187 - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),

188 - "norm": (["hidden_states"], ["hidden_states"]),

189 - }

190 -

191 - def __init__(

192 - self,

193 - vocab_size=102400,

194 - hidden_size=4096,

195 - intermediate_size=None,

196 - num_hidden_layers=32,

197 - num_attention_heads=32,

198 - num_key_value_heads=None,

199 - hidden_act="silu",

200 - max_position_embeddings=2048,

201 - initializer_range=0.02,

202 - rms_norm_eps=1e-5,

203 - use_cache=True,

204 - bos_token_id=0,

205 - eos_token_id=2,

206 - tie_word_embeddings=False,

207 - rope_theta=10000.0,

208 - rope_scaling=None,

209 - attention_dropout=0.0,

210 - sliding_window=None,

211 - sliding_window_pattern=None,

212 - layer_types=None,

213 - **kwargs,

214 - ):

215 - self.vocab_size = vocab_size

216 - self.hidden_size = hidden_size

217 - self.num_hidden_layers = num_hidden_layers

218 - self.num_attention_heads = num_attention_heads

219 - if num_key_value_heads is None:

220 - num_key_value_heads = num_attention_heads

221 - self.num_key_value_heads = num_key_value_heads

222 - if intermediate_size:

223 - self.intermediate_size = intermediate_size

224 - else:

225 - self.intermediate_size = hidden_size * 4

226 - self.hidden_act = hidden_act

227 - self.max_position_embeddings = max_position_embeddings

228 - self.initializer_range = initializer_range

229 - self.rms_norm_eps = rms_norm_eps

230 - self.use_cache = use_cache

231 - self.attention_dropout = attention_dropout

232 - self.rope_theta = rope_theta

233 - self.rope_scaling = rope_scaling

234 - self.sliding_window = sliding_window

235 - self.sliding_window_pattern = sliding_window_pattern

236 -

237 - self.layer_types = layer_types

238 - if self.layer_types is None:

239 - self.layer_types = [

240 - "sliding_attention"

241 - if check_is_sliding(self, i) else "full_attention"

242 - for i in range(self.num_hidden_layers)

243 - ]

244 - layer_type_validation(self.layer_types)

245 -

246 - super().__init__(bos_token_id=bos_token_id,

247 - eos_token_id=eos_token_id,

248 - tie_word_embeddings=tie_word_embeddings,

249 - **kwargs)

250 -

251 -

252 -__all__ = ["Exaone4Config"]

vllm/transformers_utils/configs/minimax_text_01.py deleted

-70

@@ -1,70 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -""" MiniMaxText01 model configuration"""

4 -

5 -from transformers.configuration_utils import PretrainedConfig

6 -

7 -

8 -class MiniMaxText01Config(PretrainedConfig):

9 - model_type = "MiniMaxText01"

10 - keys_to_ignore_at_inference = ["past_key_values"]

11 -

12 - def __init__(

13 - self,

14 - vocab_size=32000,

15 - hidden_size=4096,

16 - intermediate_size=14336,

17 - num_hidden_layers=32,

18 - num_attention_heads=32,

19 - num_key_value_heads=8,

20 - hidden_act="silu",

21 - max_position_embeddings=4096 * 32,

22 - initializer_range=0.02,

23 - rms_norm_eps=1e-5,

24 - use_cache=True,

25 - pad_token_id=None,

26 - bos_token_id=None,

27 - eos_token_id=None,

28 - tie_word_embeddings=False,

29 - rope_theta=1e6,

30 - sliding_window=None,

31 - attention_dropout=0.0,

32 - num_experts_per_tok=2,

33 - num_local_experts=8,

34 - output_router_logits=False,

35 - router_aux_loss_coef=0.001,

36 - router_jitter_noise=0.0,

37 - **kwargs,

38 - ):

39 - self.vocab_size = vocab_size

40 - self.max_position_embeddings = max_position_embeddings

41 - self.hidden_size = hidden_size

42 - self.intermediate_size = intermediate_size

43 - self.num_hidden_layers = num_hidden_layers

44 - self.num_attention_heads = num_attention_heads

45 - self.sliding_window = sliding_window

46 -

47 - # for backward compatibility

48 - if num_key_value_heads is None:

49 - num_key_value_heads = num_attention_heads

50 -

51 - self.num_key_value_heads = num_key_value_heads

52 - self.hidden_act = hidden_act

53 - self.initializer_range = initializer_range

54 - self.rms_norm_eps = rms_norm_eps

55 - self.use_cache = use_cache

56 - self.rope_theta = rope_theta

57 - self.attention_dropout = attention_dropout

58 -

59 - self.num_experts_per_tok = num_experts_per_tok

60 - self.num_local_experts = num_local_experts

61 - self.output_router_logits = output_router_logits

62 - self.router_aux_loss_coef = router_aux_loss_coef

63 - self.router_jitter_noise = router_jitter_noise

64 - super().__init__(

65 - pad_token_id=pad_token_id,

66 - bos_token_id=bos_token_id,

67 - eos_token_id=eos_token_id,

68 - tie_word_embeddings=tie_word_embeddings,

69 - **kwargs,

70 - )

vllm/transformers_utils/configs/minimax_vl_01.py deleted

-71

@@ -1,71 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""MiniMaxVL01 model configuration"""

4 -

5 -from transformers.configuration_utils import PretrainedConfig

6 -from transformers.models.auto import CONFIG_MAPPING

7 -

8 -from .minimax_text_01 import MiniMaxText01Config

9 -

10 -

11 -class MiniMaxVL01Config(PretrainedConfig):

12 - model_type = "minimax_vl_01"

13 -

14 - def __init__(

15 - self,

16 - vision_config=None,

17 - text_config=None,

18 - ignore_index=-100,

19 - image_token_index=32000,

20 - projector_hidden_act="gelu",

21 - vision_feature_select_strategy="default",

22 - vision_feature_layer=-2,

23 - image_grid_pinpoints=None,

24 - tie_word_embeddings=False,

25 - image_seq_length=576,

26 - **kwargs,

27 - ):

28 - self.ignore_index = ignore_index

29 - self.image_token_index = image_token_index

30 - self.projector_hidden_act = projector_hidden_act

31 - self.image_seq_length = image_seq_length

32 -

33 - if vision_feature_select_strategy not in ["default", "full"]:

34 - raise ValueError("vision_feature_select_strategy should " +

35 - "be one of 'default', 'full'." +

36 - f"Got: {vision_feature_select_strategy}")

37 -

38 - self.vision_feature_select_strategy = vision_feature_select_strategy

39 - self.vision_feature_layer = vision_feature_layer

40 - image_grid_pinpoints = (

41 - image_grid_pinpoints if image_grid_pinpoints is not None else

42 - [[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]])

43 - self.image_grid_pinpoints = image_grid_pinpoints

44 -

45 - if isinstance(vision_config, dict):

46 - if "model_type" not in vision_config:

47 - vision_config["model_type"] = "clip_vision_model"

48 - vision_config = CONFIG_MAPPING[vision_config["model_type"]](

49 - **vision_config)

50 - elif vision_config is None:

51 - vision_config = CONFIG_MAPPING["clip_vision_model"](

52 - intermediate_size=4096,

53 - hidden_size=1024,

54 - patch_size=14,

55 - image_size=336,

56 - num_hidden_layers=24,

57 - num_attention_heads=16,

58 - vocab_size=32000,

59 - projection_dim=768,

60 - )

61 -

62 - self.vision_config = vision_config

63 -

64 - if text_config is not None:

65 - text_config = MiniMaxText01Config(**text_config)

66 - else:

67 - text_config = MiniMaxText01Config()

68 -

69 - self.text_config = text_config

70 -

71 - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)

vllm/transformers_utils/configs/mpt.py deleted

-180

@@ -1,180 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# Copied from

5 -# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py

6 -"""A HuggingFace-style model configuration."""

7 -import warnings

8 -from typing import Any, Optional, Union

9 -

10 -from transformers import PretrainedConfig

11 -

12 -attn_config_defaults: dict = {

13 - 'attn_type': 'multihead_attention',

14 - 'attn_pdrop': 0.0,

15 - 'attn_impl': 'triton',

16 - 'qk_ln': False,

17 - 'clip_qkv': None,

18 - 'softmax_scale': None,

19 - 'prefix_lm': False,

20 - 'attn_uses_sequence_id': False,

21 - 'alibi': False,

22 - 'alibi_bias_max': 8

23 -}

24 -ffn_config_defaults: dict = {'ffn_type': 'mptmlp'}

25 -init_config_defaults: dict = {

26 - 'name': 'kaiming_normal_',

27 - 'fan_mode': 'fan_in',

28 - 'init_nonlinearity': 'relu',

29 - 'init_div_is_residual': True,

30 - 'emb_init_std': None,

31 - 'emb_init_uniform_lim': None,

32 - 'init_std': None,

33 - 'init_gain': 0.0

34 -}

35 -

36 -

37 -class MPTConfig(PretrainedConfig):

38 - model_type = 'mpt'

39 - attribute_map = {

40 - 'num_attention_heads': 'n_heads',

41 - 'hidden_size': 'd_model',

42 - 'num_hidden_layers': 'n_layers',

43 - }

44 -

45 - # pylint: disable=dangerous-default-value

46 - def __init__(self,

47 - d_model: int = 2048,

48 - n_heads: int = 16,

49 - n_layers: int = 24,

50 - expansion_ratio: int = 4,

51 - max_seq_len: int = 2048,

52 - vocab_size: int = 50368,

53 - resid_pdrop: float = 0.0,

54 - emb_pdrop: float = 0.0,

55 - learned_pos_emb: bool = True,

56 - attn_config: dict = attn_config_defaults,

57 - ffn_config: dict = ffn_config_defaults,

58 - init_device: str = 'cpu',

59 - logit_scale: Optional[Union[float, str]] = None,

60 - no_bias: bool = False,

61 - embedding_fraction: float = 1.0,

62 - norm_type: str = 'low_precision_layernorm',

63 - use_cache: bool = False,

64 - init_config: dict = init_config_defaults,

65 - fc_type: str = 'torch',

66 - verbose: Optional[int] = None,

67 - **kwargs: Any):

68 - self.d_model = d_model

69 - self.n_heads = n_heads

70 - self.n_layers = n_layers

71 - self.expansion_ratio = expansion_ratio

72 - self.max_seq_len = max_seq_len

73 - self.vocab_size = vocab_size

74 - self.resid_pdrop = resid_pdrop

75 - self.emb_pdrop = emb_pdrop

76 - self.learned_pos_emb = learned_pos_emb

77 - self.attn_config = attn_config

78 - self.ffn_config = ffn_config

79 - self.init_device = init_device

80 - self.logit_scale = logit_scale

81 - self.no_bias = no_bias

82 - self.embedding_fraction = embedding_fraction

83 - self.norm_type = norm_type

84 - self.use_cache = use_cache

85 - self.init_config = init_config

86 - self.fc_type = fc_type

87 - if verbose is not None:

88 - warnings.warn(DeprecationWarning(

89 - 'verbose argument for MPTConfig is now ignored and '

90 - 'will be removed. Use python_log_level instead.'),

91 - stacklevel=2)

92 - if 'name' in kwargs:

93 - del kwargs['name']

94 - if 'loss_fn' in kwargs:

95 - del kwargs['loss_fn']

96 - if self.attn_config.get('alibi', False):

97 - self.learned_pos_emb = False

98 - warnings.warn(

99 - f'alibi is turned on, setting `learned_pos_emb` '

100 - f'to {self.learned_pos_emb}`',

101 - stacklevel=2)

102 - super().__init__(**kwargs)

103 - self._validate_config()

104 -

105 - def _set_config_defaults(

106 - self, config: dict[str, Any],

107 - config_defaults: dict[str, Any]) -> dict[str, Any]:

108 - for (k, v) in config_defaults.items():

109 - if k not in config:

110 - config[k] = v

111 - return config

112 -

113 - def _validate_config(self) -> None:

114 - self.attn_config = self._set_config_defaults(self.attn_config,

115 - attn_config_defaults)

116 - self.ffn_config = self._set_config_defaults(self.ffn_config,

117 - ffn_config_defaults)

118 - self.init_config = self._set_config_defaults(self.init_config,

119 - init_config_defaults)

120 - if self.d_model % self.n_heads != 0:

121 - raise ValueError('d_model must be divisible by n_heads')

122 - if any(

123 - prob < 0 or prob > 1 for prob in

124 - [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop

125 - ]):

126 - raise ValueError(

127 - "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "

128 - "probabilities and must be between 0 and 1")

129 - if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:

130 - raise ValueError(

131 - f"Unknown attn_impl={self.attn_config['attn_impl']}")

132 - if self.attn_config['prefix_lm'] and self.attn_config[

133 - 'attn_impl'] not in ['torch', 'triton']:

134 - raise NotImplementedError(

135 - 'prefix_lm only implemented with torch and triton attention.')

136 - if self.attn_config['alibi'] and self.attn_config['attn_impl'] not in [

137 - 'torch', 'triton'

138 - ]:

139 - raise NotImplementedError(

140 - 'alibi only implemented with torch and triton attention.')

141 - if self.attn_config['attn_uses_sequence_id'] and self.attn_config[

142 - 'attn_impl'] not in ['torch', 'triton']:

143 - raise NotImplementedError(

144 - 'attn_uses_sequence_id only implemented with torch '

145 - 'and triton attention.')

146 - if self.embedding_fraction > 1 or self.embedding_fraction <= 0:

147 - raise ValueError(

148 - 'model.embedding_fraction must be between 0 (exclusive) '

149 - 'and 1 (inclusive)!')

150 - if isinstance(self.logit_scale,

151 - str) and self.logit_scale != 'inv_sqrt_d_model':

152 - raise ValueError(

153 - f"self.logit_scale={self.logit_scale!r} is not recognized as "

154 - "an option; use numeric value or 'inv_sqrt_d_model'.")

155 - if self.init_config.get('name', None) is None:

156 - raise ValueError(

157 - f"self.init_config={self.init_config!r} 'name' needs to be set."

158 - )

159 - if not self.learned_pos_emb and (not self.attn_config['alibi']):

160 - warnings.warn(

161 - 'Positional information not being provided to the model.',

162 - stacklevel=2)

163 - if self.fc_type == 'te' or self.ffn_config['ffn_type'] == 'te_ln_mlp':

164 - try:

165 - # pylint: disable=import-outside-toplevel

166 - import transformer_engine.pytorch as te

167 - del te

168 - except Exception as exc:

169 - raise ImportError(

170 - 'TransformerEngine import fail. `fc_type: te` requires '

171 - 'TransformerEngine be installed. '

172 - 'The required version of transformer_engine also requires '

173 - 'FlashAttention v1.0.6 is installed:\n'

174 - 'pip install flash-attn==1.0.6 --no-build-isolation \n'

175 - 'pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156'

176 - ) from exc

177 - if self.ffn_config['ffn_type'] == 'mptmlp':

178 - self.ffn_config['fc_type'] = self.fc_type

179 - elif self.ffn_config['ffn_type'] == 'te_ln_mlp':

180 - self.ffn_config['bias'] = not self.no_bias

vllm/transformers_utils/configs/nvlm_d.py deleted

-31

@@ -1,31 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# Adapted from

5 -# https://huggingface.co/nvidia/NVLM-D-72B/blob/main/configuration_nvlm_d.py

6 -# --------------------------------------------------------

7 -# NVLM-D

9 -# Licensed under Apache 2.0 License [see LICENSE for details]

10 -# --------------------------------------------------------

11 -from transformers import Qwen2Config

12 -from transformers.configuration_utils import PretrainedConfig

13 -

14 -

15 -class NVLM_D_Config(PretrainedConfig):

16 - model_type = 'NVLM_D'

17 - is_composition = True

18 -

19 - def __init__(self, vision_config=None, llm_config=None, **kwargs):

20 - super().__init__(**kwargs)

21 -

22 - # Handle vision_config initialization

23 - if vision_config is None:

24 - vision_config = {}

25 -

26 - # Handle llm_config initialization

27 - if llm_config is None:

28 - llm_config = {}

29 -

30 - self.vision_config = PretrainedConfig(**vision_config)

31 - self.text_config = Qwen2Config(**llm_config)

vllm/transformers_utils/configs/ovis.py deleted

-184

@@ -1,184 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# yapf: disable

5 -# ruff: noqa: E501

6 -# copied from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_aimv2.py

7 -# and https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py

8 -from typing import Any, Optional, Union

9 -

10 -from transformers import AutoConfig, PretrainedConfig

11 -

12 -

13 -class AIMv2Config(PretrainedConfig):

14 - """This is the configuration class to store the configuration of an [`AIMv2Model`].

15 -

16 - Instantiating a configuration with the defaults will yield a similar configuration

17 - to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224).

18 -

19 - Args:

20 - hidden_size: Dimension of the hidden representations.

21 - intermediate_size: Dimension of the SwiGLU representations.

22 - num_hidden_layers: Number of hidden layers in the Transformer.

23 - num_attention_heads: Number of attention heads for each attention layer

24 - in the Transformer.

25 - num_channels: Number of input channels.

26 - image_size: Image size.

27 - patch_size: Patch size.

28 - rms_norm_eps: Epsilon value used for the RMS normalization layer.

29 - attention_dropout: Dropout ratio for attention probabilities.

30 - projection_dropout: Dropout ratio for the projection layer after the attention.

31 - qkv_bias: Whether to add a bias to the queries, keys and values.

32 - use_bias: Whether to add a bias in the feed-forward and projection layers.

33 - kwargs: Keyword arguments for the [`PretrainedConfig`].

34 - """

35 -

36 - model_type: str = "aimv2"

37 -

38 - def __init__(

39 - self,

40 - hidden_size: int = 1024,

41 - intermediate_size: int = 2816,

42 - num_hidden_layers: int = 24,

43 - num_attention_heads: int = 8,

44 - num_channels: int = 3,

45 - image_size: int = 224,

46 - patch_size: int = 14,

47 - rms_norm_eps: float = 1e-5,

48 - attention_dropout: float = 0.0,

49 - projection_dropout: float = 0.0,

50 - qkv_bias: bool = False,

51 - use_bias: bool = False,

52 - **kwargs: Any,

53 - ):

54 - super().__init__(**kwargs)

55 - self.hidden_size = hidden_size

56 - self.intermediate_size = intermediate_size

57 - self.num_hidden_layers = num_hidden_layers

58 - self.num_attention_heads = num_attention_heads

59 - self.num_channels = num_channels

60 - self.patch_size = patch_size

61 - self.image_size = image_size

62 - self.attention_dropout = attention_dropout

63 - self.rms_norm_eps = rms_norm_eps

64 -

65 - self.projection_dropout = projection_dropout

66 - self.qkv_bias = qkv_bias

67 - self.use_bias = use_bias

68 -

69 -

70 -IGNORE_ID = -100

71 -IMAGE_TOKEN_ID = -200

72 -IMAGE_TOKEN = "<image>"

73 -IMAGE_ATOM_ID = -300

74 -IMAGE_INDICATOR_IDS = [-301, -302, -303, -304, -305]

75 -

76 -

77 -# ----------------------------------------------------------------------

78 -# Visual Tokenizer Configuration

79 -# ----------------------------------------------------------------------

80 -class BaseVisualTokenizerConfig(PretrainedConfig):

81 -

82 - def __init__(self,

83 - vocab_size=16384,

84 - tokenize_function="softmax",

85 - tau=1.0,

86 - depths=None,

87 - drop_cls_token=False,

88 - backbone_config: Optional[Union[PretrainedConfig,

89 - dict]] = None,

90 - hidden_stride: int = 1,

91 - **kwargs):

92 - super().__init__(**kwargs)

93 - self.vocab_size = vocab_size

94 - self.tokenize_function = tokenize_function

95 - self.tau = tau

96 - if isinstance(depths, str):

97 - depths = [int(x) for x in depths.split('|')]

98 - self.depths = depths

99 - self.backbone_kwargs = dict[str, Any]()

100 - self.drop_cls_token = drop_cls_token

101 - if backbone_config is not None:

102 - assert isinstance(backbone_config, (PretrainedConfig, dict)), \

103 - f"expect `backbone_config` to be instance of PretrainedConfig or dict, but got {type(backbone_config)} type"

104 - if not isinstance(backbone_config, PretrainedConfig):

105 - model_type = backbone_config['model_type']

106 - if model_type != "aimv2":

107 - backbone_config.pop('model_type')

108 - backbone_config = AutoConfig.for_model(model_type, **backbone_config)

109 - else:

110 - backbone_config = AIMv2Config(**backbone_config)

111 - self.backbone_config = backbone_config

112 - self.hidden_stride = hidden_stride

113 -

114 -

115 -class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):

116 - model_type = "aimv2_visual_tokenizer"

117 -

118 - def __init__(self, **kwargs):

119 - super().__init__(**kwargs)

120 - if self.drop_cls_token:

121 - self.drop_cls_token = False

122 - if self.depths:

123 - assert len(self.depths) == 1

124 - self.backbone_kwargs['num_hidden_layers'] = self.depths[0]

125 -

126 -

127 -class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):

128 - model_type = "siglip_visual_tokenizer"

129 -

130 - def __init__(self, **kwargs):

131 - super().__init__(**kwargs)

132 - if self.drop_cls_token:

133 - self.drop_cls_token = False

134 - if self.depths:

135 - assert len(self.depths) == 1

136 - self.backbone_kwargs['num_hidden_layers'] = self.depths[0]

137 -

138 -

139 -AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig)

140 -AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)

141 -

142 -

143 -# ----------------------------------------------------------------------

144 -# Ovis Configuration

145 -# ----------------------------------------------------------------------

146 -class OvisConfig(PretrainedConfig):

147 - model_type = "ovis"

148 -

149 - def __init__(self,

150 - llm_config: Optional[Union[PretrainedConfig, dict]] = None,

151 - visual_tokenizer_config: Optional[Union[PretrainedConfig,

152 - dict]] = None,

153 - multimodal_max_length=8192,

154 - hidden_size=None,

155 - conversation_formatter_class=None,

156 - llm_attn_implementation=None,

157 - disable_tie_weight=False,

158 - **kwargs):

159 - super().__init__(**kwargs)

160 - if llm_config is not None:

161 - assert isinstance(llm_config, (PretrainedConfig, dict)), \

162 - f"expect `llm_config` to be instance of PretrainedConfig or dict, but got {type(llm_config)} type"

163 - if not isinstance(llm_config, PretrainedConfig):

164 - model_type = llm_config['model_type']

165 - llm_config.pop('model_type')

166 - llm_config = AutoConfig.for_model(model_type, **llm_config)

167 -

168 - # map llm_config to text_config

169 - self.text_config = llm_config

170 - if visual_tokenizer_config is not None:

171 - assert isinstance(visual_tokenizer_config, (PretrainedConfig, dict)), \

172 - f"expect `visual_tokenizer_config` to be instance of PretrainedConfig or dict, but got {type(visual_tokenizer_config)} type"

173 - if not isinstance(visual_tokenizer_config, PretrainedConfig):

174 - model_type = visual_tokenizer_config['model_type']

175 - visual_tokenizer_config.pop('model_type')

176 - visual_tokenizer_config = AutoConfig.for_model(

177 - model_type, **visual_tokenizer_config)

178 -

179 - self.visual_tokenizer_config = visual_tokenizer_config

180 - self.multimodal_max_length = multimodal_max_length

181 - self.hidden_size = hidden_size

182 - self.conversation_formatter_class = conversation_formatter_class

183 - self.llm_attn_implementation = llm_attn_implementation

184 - self.disable_tie_weight = disable_tie_weight

vllm/transformers_utils/configs/skyworkr1v.py deleted

-54

@@ -1,54 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# Adapted from

5 -# https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/configuration_skywork_chat.py

6 -# --------------------------------------------------------

7 -# SkyworkR1V

9 -# Licensed under The MIT License [see LICENSE for details]

10 -# --------------------------------------------------------

11 -from transformers.configuration_utils import PretrainedConfig

12 -

13 -

14 -class SkyworkR1VChatConfig(PretrainedConfig):

15 - model_type = 'internvl_chat'

16 - is_composition = True

17 -

18 - def __init__(self,

19 - vision_config=None,

20 - llm_config=None,

21 - use_backbone_lora=0,

22 - use_llm_lora=0,

23 - select_layer=-1,

24 - force_image_size=None,

25 - downsample_ratio=0.5,

26 - template=None,

27 - dynamic_image_size=False,

28 - use_thumbnail=False,

29 - ps_version='v1',

30 - min_dynamic_patch=1,

31 - max_dynamic_patch=6,

32 - **kwargs):

33 - super().__init__(**kwargs)

34 -

35 - if vision_config is None:

36 - vision_config = {}

37 -

38 - if llm_config is None:

39 - llm_config = {}

40 -

41 - self.vision_config = PretrainedConfig(**vision_config)

42 - self.text_config = PretrainedConfig(**llm_config)

43 -

44 - self.use_backbone_lora = use_backbone_lora

45 - self.use_llm_lora = use_llm_lora

46 - self.select_layer = select_layer

47 - self.force_image_size = force_image_size

48 - self.downsample_ratio = downsample_ratio

49 - self.template = template

50 - self.dynamic_image_size = dynamic_image_size

51 - self.use_thumbnail = use_thumbnail

52 - self.ps_version = ps_version # pixel shuffle version

53 - self.min_dynamic_patch = min_dynamic_patch

54 - self.max_dynamic_patch = max_dynamic_patch

vllm/transformers_utils/configs/solar.py deleted

-247

@@ -1,247 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

5 -#

6 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX

7 -# and OPT implementations in this library. It has been modified from its

8 -# original forms to accommodate minor architectural differences compared

9 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.

10 -#

11 -# Licensed under the Apache License, Version 2.0 (the "License");

12 -# you may not use this file except in compliance with the License.

13 -# You may obtain a copy of the License at

14 -#

15 -# http://www.apache.org/licenses/LICENSE-2.0

16 -#

17 -# Unless required by applicable law or agreed to in writing, software

18 -# distributed under the License is distributed on an "AS IS" BASIS,

19 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

20 -# See the License for the specific language governing permissions and

21 -# limitations under the License.

22 -"""Solar model configuration"""

23 -

24 -from transformers import PretrainedConfig

25 -from transformers.utils import logging

26 -

27 -logger = logging.get_logger(__name__)

28 -

29 -

30 -class SolarConfig(PretrainedConfig):

31 - r"""

32 - This is the configuration class to store

33 - the configuration of a [`SolarModel`].

34 - It is used to instantiate an LLaMA model

35 - according to the specified arguments,

36 - defining the model architecture.

37 - Instantiating a configuration with the

38 - defaults will yield a similar

39 - configuration to that of the LLaMA-7B.

40 - Configuration objects inherit from [`PretrainedConfig`]

41 - and can be used to control the model outputs.

42 - Read the documentation from [`PretrainedConfig`] for more information.

43 - Args:

44 - vocab_size (`int`, *optional*, defaults to 32000):

45 - Vocabulary size of the LLaMA model.

46 - Defines the number of different tokens

47 - that can be represented by the `inputs_ids`

48 - passed when calling [`SolarModel`]

49 - hidden_size (`int`, *optional*, defaults to 4096):

50 - Dimension of the hidden representations.

51 - intermediate_size (`int`, *optional*, defaults to 11008):

52 - Dimension of the MLP representations.

53 - num_hidden_layers (`int`, *optional*, defaults to 32):

54 - Number of hidden layers in the Transformer decoder.

55 - num_attention_heads (`int`, *optional*, defaults to 32):

56 - Number of attention heads for each attention layer

57 - in the Transformer decoder.

58 - num_key_value_heads (`int`, *optional*):

59 - This is the number of key_value heads that

60 - should be used to implement Grouped Query Attention. If

61 - `num_key_value_heads=num_attention_heads`,

62 - the model will use Multi Head Attention (MHA), if

63 - `num_key_value_heads=1` the model

64 - will use Multi Query Attention (MQA)

65 - otherwise GQA is used. When

66 - converting a multi-head checkpoint to a GQA checkpoint,

67 - each group key and value head should be constructed

68 - by meanpooling all the original heads within that group.

69 - For more details checkout [this paper]

70 - (https://arxiv.org/pdf/2305.13245.pdf).

71 - If it is not specified, will default to

72 - `num_attention_heads`.

73 - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):

74 - The non-linear activation function (function or string)

75 - in the decoder.

76 - max_position_embeddings (`int`, *optional*, defaults to 2048):

77 - The maximum sequence length that this model might ever be used with.

78 - Solar 1 supports up to 2048 tokens,

79 - Solar 2 up to 4096, CodeSolar up to 16384.

80 - initializer_range (`float`, *optional*, defaults to 0.02):

81 - The standard deviation of

82 - the truncated_normal_initializer for initializing

83 - all weight matrices.

84 - rms_norm_eps (`float`, *optional*, defaults to 1e-06):

85 - The epsilon used by the rms normalization layers.

86 - use_cache (`bool`, *optional*, defaults to `True`):

87 - Whether or not the model should return

88 - the last key/values attentions (not used by all models). Only

89 - relevant if `config.is_decoder=True`.

90 - pad_token_id (`int`, *optional*):

91 - Padding token id.

92 - bos_token_id (`int`, *optional*, defaults to 1):

93 - Beginning of stream token id.

94 - eos_token_id (`int`, *optional*, defaults to 2):

95 - End of stream token id.

96 - pretraining_tp (`int`, *optional*, defaults to 1):

97 - Experimental feature. Tensor parallelism rank

98 - used during pretraining.

99 - Please refer to [this

100 - document](https://huggingface.co/docs/

101 - transformers/main/

102 - perf_train_gpu_many#tensor-parallelism)

103 - to understand more about it. This value is

104 - necessary to ensure exact reproducibility

105 - of the pretraining results.

106 - Please refer to [this

107 - issue](https://github.com/pytorch/pytorch/issues/76232).

108 - tie_word_embeddings (`bool`, *optional*, defaults to `False`):

109 - Whether to tie weight embeddings

110 - rope_theta (`float`, *optional*, defaults to 10000.0):

111 - The base period of the RoPE embeddings.

112 - rope_scaling (`dict`, *optional*):

113 - Dictionary containing the scaling configuration for

114 - the RoPE embeddings.

115 - Currently supports two scaling

116 - strategies: linear and dynamic.

117 - Their scaling factor must be a float greater than 1.

118 - The expected format is

119 - `{"type": strategy name, "factor": scaling factor}`.

120 - When using this flag, don't update

121 - `max_position_embeddings` to the expected new maximum.

122 - See the following thread for more information on how

123 - these scaling strategies behave:

124 - https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/

125 - dynamically_scaled_rope_further_increases/. This is an

126 - experimental feature, subject to breaking

127 - API changes in future versions.

128 - attention_bias (`bool`, *optional*, defaults to `False`):

129 - Whether to use a bias in the query, key, value

130 - and output projection layers during self-attention.

131 - attention_dropout (`float`, *optional*, defaults to 0.0):

132 - The dropout ratio for the attention probabilities.

133 - mlp_bias (`bool`, *optional*, defaults to `False`):

134 - Whether to use a bias in up_proj, down_proj and gate_proj

135 - layers in the MLP layers.

136 - sliding_window (`int`, *optional*, defaults to 2047):

137 - Sliding window attention window size. If not specified,

138 - will default to `2047`.

139 - ```python

140 - >>> from transformers import SolarModel, SolarConfig

141 - >>> # Initializing a Solar-pro style configuration

142 - >>> configuration = SolarConfig()

143 - >>> # Initializing a model from the Solar-pro style configuration

144 - >>> model = SolarModel(configuration)

145 - >>> # Accessing the model configuration

146 - >>> configuration = model.config

147 - ```"""

148 -

149 - model_type = "solar"

150 - keys_to_ignore_at_inference = ["past_key_values"]

151 -

152 - def __init__(

153 - self,

154 - vocab_size=32000,

155 - hidden_size=4096,

156 - intermediate_size=11008,

157 - num_hidden_layers=32,

158 - num_attention_heads=32,

159 - num_key_value_heads=None,

160 - hidden_act="silu",

161 - max_position_embeddings=2048,

162 - initializer_range=0.02,

163 - rms_norm_eps=1e-6,

164 - use_cache=True,

165 - pad_token_id=None,

166 - bos_token_id=1,

167 - eos_token_id=2,

168 - pretraining_tp=1,

169 - tie_word_embeddings=False,

170 - rope_theta=10000.0,

171 - rope_scaling=None,

172 - attention_bias=False,

173 - attention_dropout=0.0,

174 - mlp_bias=False,

175 - sliding_window=2047,

176 - bskcn_1=None,

177 - bskcn_2=None,

178 - bskcn_3=None,

179 - bskcn_4=None,

180 - bskcn_tv=None,

181 - **kwargs,

182 - ):

183 - self.vocab_size = vocab_size

184 - self.max_position_embeddings = max_position_embeddings

185 - self.hidden_size = hidden_size

186 - self.intermediate_size = intermediate_size

187 - self.num_hidden_layers = num_hidden_layers

188 - self.num_attention_heads = num_attention_heads

189 -

190 - # for backward compatibility

191 - if num_key_value_heads is None:

192 - num_key_value_heads = num_attention_heads

193 -

194 - self.num_key_value_heads = num_key_value_heads

195 - self.hidden_act = hidden_act

196 - self.initializer_range = initializer_range

197 - self.rms_norm_eps = rms_norm_eps

198 - self.pretraining_tp = pretraining_tp

199 - self.use_cache = use_cache

200 - self.rope_theta = rope_theta

201 - self.rope_scaling = rope_scaling

202 - self._rope_scaling_validation()

203 - self.attention_bias = attention_bias

204 - self.attention_dropout = attention_dropout

205 - self.mlp_bias = mlp_bias

206 - self.sliding_window = sliding_window

207 - self.bskcn_1 = bskcn_1 if bskcn_1 is not None else [12, 20, 32, 44]

208 - self.bskcn_2 = bskcn_2 if bskcn_2 is not None else [20, 32]

209 - self.bskcn_3 = bskcn_3 if bskcn_3 is not None else [16, 24, 36, 48]

210 - self.bskcn_4 = bskcn_4 if bskcn_4 is not None else [28, 40]

211 - self.bskcn_tv = bskcn_tv if bskcn_tv is not None else [0.9, 0.8]

212 -

213 - super().__init__(

214 - pad_token_id=pad_token_id,

215 - bos_token_id=bos_token_id,

216 - eos_token_id=eos_token_id,

217 - tie_word_embeddings=tie_word_embeddings,

218 - **kwargs,

219 - )

220 -

221 - def _rope_scaling_validation(self):

222 - """

223 - Validate the `rope_scaling` configuration.

224 - """

225 - if self.rope_scaling is None:

226 - return

227 -

228 - if (not isinstance(self.rope_scaling, dict)

229 - or len(self.rope_scaling) != 2):

230 - raise ValueError(

231 - "`rope_scaling` must be a dictionary with two fields,"

232 - " `type` and `factor`, "

233 - f"got {self.rope_scaling}")

234 - rope_scaling_type = self.rope_scaling.get("type", None)

235 - rope_scaling_factor = self.rope_scaling.get("factor", None)

236 - if rope_scaling_type is None or rope_scaling_type not in [

237 - "linear",

238 - "dynamic",

239 - ]:

240 - raise ValueError(f"`rope_scaling`'s type field must be one of "

241 - f"['linear', 'dynamic'], got {rope_scaling_type}")

242 - if (rope_scaling_factor is None

243 - or not isinstance(rope_scaling_factor, float)

244 - or rope_scaling_factor <= 1.0):

245 - raise ValueError(

246 - f"`rope_scaling`'s factor field must be a float > 1,"

247 - f" got {rope_scaling_factor}")

vllm/transformers_utils/configs/telechat2.py deleted

-64

@@ -1,64 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -# adapted from https://www.modelscope.cn/models/TeleAI/TeleChat2-3B/resolve/master/configuration_telechat2.py

5 -""" Telechat configuration compatible with LlamaConfig. """

6 -

7 -from transformers.configuration_utils import PretrainedConfig

8 -

9 -

10 -class Telechat2Config(PretrainedConfig):

11 -

12 - model_type = "telechat"

13 - keys_to_ignore_at_inference = ["past_key_values"]

14 - attribute_map = {

15 - "num_hidden_layers": "n_layer",

16 - "num_attention_heads": "n_head",

17 - "intermediate_size": "ffn_hidden_size",

18 - "rms_norm_eps": "layer_norm_epsilon"

19 - }

20 -

21 - def __init__(

22 - self,

23 - vocab_size=160256,

24 - hidden_size=4096,

25 - n_layer=30,

26 - n_head=32,

27 - layer_norm_epsilon=1e-5,

28 - initializer_range=0.02,

29 - use_cache=True,

30 - bos_token_id=1,

31 - eos_token_id=2,

32 - apply_residual_connection_post_layernorm=False,

33 - hidden_dropout=0.0,

34 - attention_dropout=0.0,

35 - ffn_hidden_size=12288,

36 - training_seqlen=8192,

37 - logn=True,

38 - embed_layernorm=False,

39 - hidden_act="silu",

40 - **kwargs,

41 - ):

42 - self.vocab_size = vocab_size

43 - n_embed = kwargs.pop("n_embed", None)

44 - self.hidden_size = hidden_size if n_embed is None else n_embed

45 - self.n_layer = n_layer

46 - self.n_head = n_head

47 - self.layer_norm_epsilon = layer_norm_epsilon

48 - self.initializer_range = initializer_range

49 - self.use_cache = use_cache

50 - self.apply_residual_connection_post_layernorm = (

51 - apply_residual_connection_post_layernorm)

52 - self.hidden_dropout = hidden_dropout

53 - self.attention_dropout = attention_dropout

54 - self.bos_token_id = bos_token_id

55 - self.eos_token_id = eos_token_id

56 - self.logn = logn

57 - self.training_seqlen = training_seqlen

58 - self.embed_layernorm = embed_layernorm

59 - self.num_key_value_heads = kwargs.pop("num_key_value_heads", None)

60 - self.ffn_hidden_size = ffn_hidden_size

61 - self.hidden_act = hidden_act

62 - super().__init__(bos_token_id=bos_token_id,

63 - eos_token_id=eos_token_id,

64 - **kwargs)

vllm/transformers_utils/processors/__init__.py

@@ -1,5 +1,12 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +Multi-modal processors may be defined in this directory for the following

5 +reasons:

6 +

7 +- There is no processing file defined by HF Hub or Transformers library.

8 +- There is a need to override the existing processor to support vLLM.

9 +"""

11 from vllm.transformers_utils.processors.deepseek_vl2 import (

12 DeepseekVLV2Processor)