A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Update transformers to `v4.55` (#21931)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: isotr0py <2037008807@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Harry Mellor committed
Aug 6, 2025 at 06:56 UTC
796bae07c59716b7b61d57343826bfbeabdd01bb
13 files changed
+235
-39
requirements/common.txt
+1
-1
@@ -7,7 +7,7 @@ requests >= 2.26.0
7
tqdm
8
blake3
9
py-cpuinfo
10
-transformers >= 4.53.2
10
+transformers >= 4.55.0
11
huggingface-hub[hf_xet] >= 0.33.0 # Required for Xet downloads.
12
tokenizers >= 0.21.1 # Required for fast incremental detokenization.
13
protobuf # Required by LlamaTokenizer.
requirements/test.in
+1
-1
@@ -35,7 +35,7 @@ opencv-python-headless >= 4.11.0 # required for video test
35
datamodel_code_generator # required for minicpm3 test
36
lm-eval[api]==0.4.8 # required for model evaluation test
37
mteb[bm25s]>=1.38.11, <2 # required for mteb test
38
-transformers==4.53.2
38
+transformers==4.55.0
39
tokenizers==0.21.1
40
huggingface-hub[hf_xet]>=0.33.0 # Required for Xet downloads.
41
schemathesis>=3.39.15 # Required for openai schema test.
requirements/test.txt
+3
-3
@@ -214,7 +214,7 @@ fiona==1.10.1
214
# via torchgeo
215
flask==3.1.1
216
# via mlflow
217
-fonttools==4.54.1
217
+fonttools==4.55.0
218
# via matplotlib
219
fqdn==1.5.1
220
# via jsonschema
@@ -286,7 +286,7 @@ httpx==0.27.2
286
# via
287
# -r requirements/test.in
288
# schemathesis
289
-huggingface-hub==0.33.1
289
+huggingface-hub==0.34.3
290
# via
291
# -r requirements/test.in
292
# accelerate
@@ -1148,7 +1148,7 @@ tqdm==4.66.6
1148
# transformers
1149
tqdm-multiprocess==0.0.11
1150
# via lm-eval
1151
-transformers==4.53.2
1151
+transformers==4.55.0
1152
# via
1153
# -r requirements/test.in
1154
# genai-perf
tests/models/multimodal/generation/test_common.py
+4
@@ -337,6 +337,10 @@ VLM_TEST_SETTINGS = {
337
vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
338
num_logprobs=10,
339
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
340
+ # FIXME(Isotr0py): This model is broken in Transformers v4.54.1, we
341
+ # should enable this again after the fix is released:
342
+ # https://github.com/huggingface/transformers/pull/39915
343
+ marks=[pytest.mark.skip("HF model is broken")],
344
),
345
"gemma3": VLMTestInfo(
346
models=["google/gemma-3-4b-it"],
tests/models/registry.py
+15
-9
@@ -179,8 +179,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
179
min_transformers_version="4.54"),
180
"Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
181
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
182
- "FalconH1ForCausalLM":_HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base",
183
- min_transformers_version="4.53"),
182
+ "FalconH1ForCausalLM":_HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
183
"GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
184
"Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
185
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
@@ -223,7 +222,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
222
trust_remote_code=True),
223
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
224
"JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
226
- extras={"tiny": "ai21labs/Jamba-tiny-dev"}), # noqa: E501
225
+ extras={
226
+ "tiny": "ai21labs/Jamba-tiny-dev",
227
+ "random": "ai21labs/Jamba-tiny-random", # noqa: E501
228
+ }),
229
"LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.2-1B-Instruct",
230
extras={"guard": "meta-llama/Llama-Guard-3-1B", # noqa: E501
231
"hermes": "NousResearch/Hermes-3-Llama-3.1-8B", # noqa: E501
@@ -239,8 +241,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
241
trust_remote_code=True),
242
"MiniCPM3ForCausalLM": _HfExamplesInfo("openbmb/MiniCPM3-4B",
243
trust_remote_code=True),
242
- "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf",
243
- min_transformers_version="4.53"),
244
+ "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
245
"MiniMaxText01ForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01",
246
trust_remote_code=True,
247
revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3"), # noqa: E501
@@ -272,6 +273,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
273
"PhiMoEForCausalLM": _HfExamplesInfo("microsoft/Phi-3.5-MoE-instruct",
274
trust_remote_code=True),
275
"Plamo2ForCausalLM": _HfExamplesInfo("pfnet/plamo-2-1b",
276
+ max_transformers_version="4.53",
277
+ transformers_version_reason="vLLM impl inherits PreTrainedModel and clashes with get_input_embeddings", # noqa: E501
278
trust_remote_code=True),
279
"QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-7B-Chat",
280
trust_remote_code=True),
@@ -299,8 +302,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
302
"Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
303
"MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL",
304
trust_remote_code=True),
302
- "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst",
303
- min_transformers_version="4.53"),
305
+ "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
306
# [Encoder-decoder]
307
"BartModel": _HfExamplesInfo("facebook/bart-base"),
308
"BartForConditionalGeneration": _HfExamplesInfo("facebook/bart-large-cnn"),
@@ -326,8 +328,12 @@ _EMBEDDING_EXAMPLE_MODELS = {
328
"NomicBertModel": _HfExamplesInfo("nomic-ai/nomic-embed-text-v2-moe",
329
trust_remote_code=True, v0_only=True), # noqa: E501
330
"Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
329
- "Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
330
- "Qwen2ForProcessRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-PRM-7B"),
331
+ "Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B",
332
+ max_transformers_version="4.53",
333
+ transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers"), # noqa: E501
334
+ "Qwen2ForProcessRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-PRM-7B",
335
+ max_transformers_version="4.53",
336
+ transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers"), # noqa: E501
337
"RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2", v0_only=True), # noqa: E501
338
"RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1", v0_only=True), # noqa: E501
339
"XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small", v0_only=True), # noqa: E501
tests/quantization/test_experts_int8.py
+4
@@ -9,6 +9,8 @@ import pytest
9
10
from tests.quantization.utils import is_quant_method_supported
11
12
+from ..models.registry import HF_EXAMPLE_MODELS
13
+
14
MODELS = ["ai21labs/Jamba-tiny-random", "pfnet/plamo-2-1b"]
15
16
@@ -25,6 +27,8 @@ def test_model_experts_int8_startup(
27
dtype: str,
28
max_tokens: int,
29
) -> None:
30
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
31
+ model_info.check_transformers_version(on_fail="skip")
32
33
with vllm_runner(model, dtype=dtype,
34
quantization="experts_int8") as vllm_model:
vllm/model_executor/models/interfaces_base.py
+8
-4
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from typing import (TYPE_CHECKING, ClassVar, Literal, Optional, Protocol,
3
+from typing import (TYPE_CHECKING, Any, ClassVar, Literal, Optional, Protocol,
4
Union, overload, runtime_checkable)
5
6
import torch
@@ -14,6 +14,10 @@ if TYPE_CHECKING:
14
from vllm.config import VllmConfig
15
from vllm.model_executor.layers.pooler import Pooler
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
+else:
18
+ VllmConfig = Any
19
+ Pooler = Any
20
+ SamplingMetadata = Any
21
22
logger = init_logger(__name__)
23
@@ -34,7 +38,7 @@ class VllmModel(Protocol[T_co]):
38
39
def __init__(
40
self,
37
- vllm_config: "VllmConfig",
41
+ vllm_config: VllmConfig,
42
prefix: str = "",
43
) -> None:
44
...
@@ -96,7 +100,7 @@ class VllmModelForTextGeneration(VllmModel[T], Protocol[T]):
100
def compute_logits(
101
self,
102
hidden_states: T,
99
- sampling_metadata: "SamplingMetadata",
103
+ sampling_metadata: SamplingMetadata,
104
) -> Optional[T]:
105
"""Return `None` if TP rank > 0."""
106
...
@@ -140,7 +144,7 @@ class VllmModelForPooling(VllmModel[T_co], Protocol[T_co]):
144
MRO of your model class.
145
"""
146
143
- pooler: "Pooler"
147
+ pooler: Pooler
148
"""The pooler is only called on TP rank 0."""
149
150
vllm/model_executor/models/qwen2_vl.py
+6
-5
@@ -1395,11 +1395,12 @@ class Tarsier2Processor(Qwen2VLProcessor):
1395
**kwargs,
1396
):
1397
self.image_processor = Tarsier2ImageProcessor(**vision_config)
1398
- super().__init__(image_processor=self.image_processor,
1399
- tokenizer=tokenizer,
1400
- video_processor=Qwen2VLVideoProcessor(),
1401
- chat_template=None,
1402
- **kwargs)
1398
+ super().__init__(
1399
+ image_processor=self.image_processor,
1400
+ tokenizer=tokenizer,
1401
+ video_processor=Qwen2VLVideoProcessor(**vision_config),
1402
+ chat_template=None,
1403
+ **kwargs)
1404
1405
1406
class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo):
vllm/model_executor/models/transformers.py
+10
-7
@@ -90,7 +90,7 @@ def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
90
def replace_linear_class(
91
linear: nn.Linear, style: Literal["colwise", "rowwise"],
92
quant_config: QuantizationConfig
93
-) -> Union[ColumnParallelLinear, RowParallelLinear]:
93
+) -> Union[ColumnParallelLinear, RowParallelLinear, ReplicatedLinear]:
94
"""
95
Replace nn.Linear with one of vLLM's tensor parallel linear classes.
96
@@ -445,7 +445,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
445
446
# Set correct attn and init on "meta" to delay allocating GPU tensors
447
# TODO: @raushan, use the public `model.set_attn_implementation()`
448
- # method after v4.54.0 is released
448
+ # method once its checks are fixed in Transformers.
449
self.text_config._attn_implementation = "vllm"
450
with init_on_device_without_buffers("meta"), config_override:
451
self.model: PreTrainedModel = AutoModel.from_config(
@@ -520,7 +520,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
520
for i in range(len(layers)):
521
if start_layer <= i and i < end_layer:
522
continue
523
- layers[i] = PPMissingLayer(return_tuple=True)
523
+ layers[i] = PPMissingLayer()
524
525
# Layers after module list
526
for name in pp_plan[module_list_idx + 1:]:
@@ -533,14 +533,16 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
533
Apply the model's tensor parallelization plan.
534
Currently only supports linear layers.
535
"""
536
- if not self.model.supports_tp_plan:
537
- if self.tp_size <= 1:
538
- return
536
+ tp_plan = getattr(self.model.config, "base_model_tp_plan", None) or {}
537
538
+ if not tp_plan and self.tp_size > 1:
539
raise ValueError(
540
f"{type(self.model)} does not support tensor parallel yet!")
541
543
- tp_plan = self.model._tp_plan
542
+ # Some weight loaders expect linear layers to inherit from vLLM's
543
+ # LinearBase class, so we set a default style which causes any
544
+ # unspecified linear layers to be replaced with ReplicatedLinear
545
+ tp_plan[".*"] = "replicated"
546
547
def _tensor_parallel(module: nn.Module, prefix: str = ""):
548
for child_name, child_module in module.named_children():
@@ -552,6 +554,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
554
child_module, style, self.quant_config)
555
setattr(module, child_name, new_module)
556
log_replacement(qual_name, child_module, new_module)
557
+ break
558
else:
559
_tensor_parallel(child_module, prefix=qual_name)
560
vllm/model_executor/models/utils.py
+2
-8
@@ -534,16 +534,10 @@ class PPMissingLayer(torch.nn.Identity):
534
535
def __init__(self, *args, **kwargs):
536
super().__init__()
537
- self.return_tuple = kwargs.get("return_tuple", False)
537
538
def forward(self, *args, **kwargs):
540
- """
541
- Return the first arg from args or the first value from kwargs.
542
-
543
- Wraps the input in a tuple if `self.return_tuple` is True.
544
- """
545
- input = args[0] if args else next(iter(kwargs.values()))
546
- return (input, ) if self.return_tuple else input
539
+ """Return the first arg from args or the first value from kwargs."""
540
+ return args[0] if args else next(iter(kwargs.values()))
541
542
543
_CPU_OFFLOAD_BYTES = 0
vllm/transformers_utils/config.py
+3
-1
@@ -35,7 +35,8 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DeepseekVLV2Config,
35
MllamaConfig, MLPSpeculatorConfig,
36
Nemotron_Nano_VL_Config,
37
NemotronConfig, NVLM_D_Config,
38
- RWConfig, SpeculatorsConfig,
38
+ OvisConfig, RWConfig,
39
+ SpeculatorsConfig,
40
Step3TextConfig, Step3VLConfig,
41
UltravoxConfig)
42
# yapf: enable
@@ -85,6 +86,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
86
"speculators": SpeculatorsConfig,
87
"nemotron": NemotronConfig,
88
"NVLM_D": NVLM_D_Config,
89
+ "ovis": OvisConfig,
90
"ultravox": UltravoxConfig,
91
"step3_vl": Step3VLConfig,
92
"step3_text": Step3TextConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -24,6 +24,7 @@ from vllm.transformers_utils.configs.nemotron import NemotronConfig
24
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
25
from vllm.transformers_utils.configs.nemotron_vl import Nemotron_Nano_VL_Config
26
from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
27
+from vllm.transformers_utils.configs.ovis import OvisConfig
28
from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
29
from vllm.transformers_utils.configs.step3_vl import (Step3TextConfig,
30
Step3VisionEncoderConfig,
@@ -45,6 +46,7 @@ __all__ = [
46
"NemotronHConfig",
47
"Nemotron_Nano_VL_Config",
48
"NVLM_D_Config",
49
+ "OvisConfig",
50
"SpeculatorsConfig",
51
"UltravoxConfig",
52
"Step3VLConfig",
vllm/transformers_utils/configs/ovis.py
new
+176
@@ -0,0 +1,176 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# yapf: disable
5
+# ruff: noqa: E501
6
+# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_aimv2.py
7
+# and https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
8
+# Ovis Config with AimV2 config registration removed for Transformers compatibility
9
+from typing import Any, Optional, Union
10
+
11
+from transformers import AutoConfig, PretrainedConfig
12
+
13
+
14
+class AIMv2Config(PretrainedConfig):
15
+ """This is the configuration class to store the configuration of an [`AIMv2Model`].
16
+ Instantiating a configuration with the defaults will yield a similar configuration
17
+ to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224).
18
+ Args:
19
+ hidden_size: Dimension of the hidden representations.
20
+ intermediate_size: Dimension of the SwiGLU representations.
21
+ num_hidden_layers: Number of hidden layers in the Transformer.
22
+ num_attention_heads: Number of attention heads for each attention layer
23
+ in the Transformer.
24
+ num_channels: Number of input channels.
25
+ image_size: Image size.
26
+ patch_size: Patch size.
27
+ rms_norm_eps: Epsilon value used for the RMS normalization layer.
28
+ attention_dropout: Dropout ratio for attention probabilities.
29
+ projection_dropout: Dropout ratio for the projection layer after the attention.
30
+ qkv_bias: Whether to add a bias to the queries, keys and values.
31
+ use_bias: Whether to add a bias in the feed-forward and projection layers.
32
+ kwargs: Keyword arguments for the [`PretrainedConfig`].
33
+ """
34
+
35
+ model_type: str = "aimv2"
36
+
37
+ def __init__(
38
+ self,
39
+ hidden_size: int = 1024,
40
+ intermediate_size: int = 2816,
41
+ num_hidden_layers: int = 24,
42
+ num_attention_heads: int = 8,
43
+ num_channels: int = 3,
44
+ image_size: int = 224,
45
+ patch_size: int = 14,
46
+ rms_norm_eps: float = 1e-5,
47
+ attention_dropout: float = 0.0,
48
+ projection_dropout: float = 0.0,
49
+ qkv_bias: bool = False,
50
+ use_bias: bool = False,
51
+ **kwargs: Any,
52
+ ):
53
+ super().__init__(**kwargs)
54
+ self.hidden_size = hidden_size
55
+ self.intermediate_size = intermediate_size
56
+ self.num_hidden_layers = num_hidden_layers
57
+ self.num_attention_heads = num_attention_heads
58
+ self.num_channels = num_channels
59
+ self.patch_size = patch_size
60
+ self.image_size = image_size
61
+ self.attention_dropout = attention_dropout
62
+ self.rms_norm_eps = rms_norm_eps
63
+
64
+ self.projection_dropout = projection_dropout
65
+ self.qkv_bias = qkv_bias
66
+ self.use_bias = use_bias
67
+
68
+
69
+# ----------------------------------------------------------------------
70
+# Visual Tokenizer Configuration
71
+# ----------------------------------------------------------------------
72
+class BaseVisualTokenizerConfig(PretrainedConfig):
73
+
74
+ def __init__(self,
75
+ vocab_size=16384,
76
+ tokenize_function="softmax",
77
+ tau=1.0,
78
+ depths=None,
79
+ drop_cls_token=False,
80
+ backbone_config: Optional[Union[PretrainedConfig,
81
+ dict]] = None,
82
+ hidden_stride: int = 1,
83
+ **kwargs):
84
+ super().__init__(**kwargs)
85
+ self.vocab_size = vocab_size
86
+ self.tokenize_function = tokenize_function
87
+ self.tau = tau
88
+ if isinstance(depths, str):
89
+ depths = [int(x) for x in depths.split('|')]
90
+ self.depths = depths
91
+ self.backbone_kwargs = dict[str, Any]()
92
+ self.drop_cls_token = drop_cls_token
93
+ if backbone_config is not None:
94
+ assert isinstance(backbone_config, (PretrainedConfig, dict)), \
95
+ f"expect `backbone_config` to be instance of PretrainedConfig or dict, but got {type(backbone_config)} type"
96
+ if not isinstance(backbone_config, PretrainedConfig):
97
+ model_type = backbone_config['model_type']
98
+ if model_type != "aimv2":
99
+ backbone_config.pop('model_type')
100
+ backbone_config = AutoConfig.for_model(model_type, **backbone_config)
101
+ else:
102
+ backbone_config = AIMv2Config(**backbone_config)
103
+ self.backbone_config = backbone_config
104
+ self.hidden_stride = hidden_stride
105
+
106
+
107
+class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):
108
+ model_type = "aimv2_visual_tokenizer"
109
+
110
+ def __init__(self, **kwargs):
111
+ super().__init__(**kwargs)
112
+ if self.drop_cls_token:
113
+ self.drop_cls_token = False
114
+ if self.depths:
115
+ assert len(self.depths) == 1
116
+ self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
117
+
118
+
119
+class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):
120
+ model_type = "siglip_visual_tokenizer"
121
+
122
+ def __init__(self, **kwargs):
123
+ super().__init__(**kwargs)
124
+ if self.drop_cls_token:
125
+ self.drop_cls_token = False
126
+ if self.depths:
127
+ assert len(self.depths) == 1
128
+ self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
129
+
130
+
131
+AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig)
132
+AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)
133
+
134
+
135
+# ----------------------------------------------------------------------
136
+# Ovis Configuration
137
+# ----------------------------------------------------------------------
138
+class OvisConfig(PretrainedConfig):
139
+ model_type = "ovis"
140
+
141
+ def __init__(self,
142
+ llm_config: Optional[Union[PretrainedConfig, dict]] = None,
143
+ visual_tokenizer_config: Optional[Union[PretrainedConfig,
144
+ dict]] = None,
145
+ multimodal_max_length=8192,
146
+ hidden_size=None,
147
+ conversation_formatter_class=None,
148
+ llm_attn_implementation=None,
149
+ disable_tie_weight=False,
150
+ **kwargs):
151
+ super().__init__(**kwargs)
152
+ if llm_config is not None:
153
+ assert isinstance(llm_config, (PretrainedConfig, dict)), \
154
+ f"expect `llm_config` to be instance of PretrainedConfig or dict, but got {type(llm_config)} type"
155
+ if not isinstance(llm_config, PretrainedConfig):
156
+ model_type = llm_config['model_type']
157
+ llm_config.pop('model_type')
158
+ llm_config = AutoConfig.for_model(model_type, **llm_config)
159
+
160
+ # map llm_config to text_config
161
+ self.text_config = llm_config
162
+ if visual_tokenizer_config is not None:
163
+ assert isinstance(visual_tokenizer_config, (PretrainedConfig, dict)), \
164
+ f"expect `visual_tokenizer_config` to be instance of PretrainedConfig or dict, but got {type(visual_tokenizer_config)} type"
165
+ if not isinstance(visual_tokenizer_config, PretrainedConfig):
166
+ model_type = visual_tokenizer_config['model_type']
167
+ visual_tokenizer_config.pop('model_type')
168
+ visual_tokenizer_config = AutoConfig.for_model(
169
+ model_type, **visual_tokenizer_config)
170
+
171
+ self.visual_tokenizer_config = visual_tokenizer_config
172
+ self.multimodal_max_length = multimodal_max_length
173
+ self.hidden_size = hidden_size
174
+ self.conversation_formatter_class = conversation_formatter_class
175
+ self.llm_attn_implementation = llm_attn_implementation
176
+ self.disable_tie_weight = disable_tie_weight