A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 08:45 UTC
[Transformers v5] Vendor HCXVisionConfig for compatibility (#38447)
Signed-off-by: Fang Han <fhan0520@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Fang Han committed
May 3, 2026 at 21:19 UTC
c103c02a1a97819f1f83c03c60dcc532293f520f
4 files changed
+79
-9
tests/models/registry.py
-7
@@ -946,13 +946,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
946
"HCXVisionForCausalLM": _HfExamplesInfo(
947
"naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
948
trust_remote_code=True,
949
- max_transformers_version="4.57",
950
- transformers_version_reason={
951
- "vllm": (
952
- "Custom config cannot be loaded with Transformers "
953
- "v5 because `text_config` is not always set"
954
- )
955
- },
949
),
950
"HCXVisionV2ForCausalLM": _HfExamplesInfo(
951
"naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
vllm/transformers_utils/config.py
+4
-2
@@ -101,6 +101,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
101
fireredlid="FireRedLIDConfig",
102
funaudiochat="FunAudioChatConfig",
103
granite4_vision="Granite4VisionConfig",
104
+ hyperclovax_vlm="HCXVisionConfig",
105
hunyuan_vl="HunYuanVLConfig",
106
hy_v3="HYV3Config",
107
isaac="IsaacConfig",
@@ -217,8 +218,9 @@ class HFConfigParser(ConfigParserBase):
218
)
219
else:
220
if model_type in _CONFIG_REGISTRY:
220
- # Register the config class to AutoConfig to ensure it's used in future
221
- # calls to `from_pretrained`
221
+ # Register the config class to AutoConfig to ensure it's used
222
+ # in future calls to `from_pretrained` (e.g. from
223
+ # AutoTokenizer or AutoProcessor).
224
config_class = _CONFIG_REGISTRY[model_type]
225
config_class.model_type = model_type
226
AutoConfig.register(model_type, config_class, exist_ok=True)
vllm/transformers_utils/configs/__init__.py
+2
@@ -37,6 +37,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
37
"HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
38
"HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
39
"HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
40
+ "HCXVisionConfig": "vllm.transformers_utils.configs.hyperclovax",
41
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
42
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
43
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
@@ -104,6 +105,7 @@ __all__ = [
105
"HunYuanVLConfig",
106
"HunYuanVLTextConfig",
107
"HunYuanVLVisionConfig",
108
+ "HCXVisionConfig",
109
"HYV3Config",
110
"HyperCLOVAXConfig",
111
"IsaacConfig",
vllm/transformers_utils/configs/hyperclovax.py
+73
@@ -17,6 +17,7 @@
17
# limitations under the License.
18
"""HyperCLOVA X model configuration."""
19
20
+from transformers import AutoConfig
21
from transformers.configuration_utils import PretrainedConfig
22
23
@@ -275,3 +276,75 @@ class HyperCLOVAXConfig(PretrainedConfig):
276
auto_map=auto_map,
277
**kwargs,
278
)
279
+
280
+
281
+class HCXVisionConfig(PretrainedConfig):
282
+ """Vendored HyperCLOVAX Vision config with transformers v5 fix.
283
+
284
+ The original remote code config does not handle empty initialization
285
+ (text_config=None), which breaks transformers v5's @strict validation.
286
+
287
+ TODO: Remove this class once HyperCLOVAX is upstreamed to transformers.
288
+ Tracking PR: https://github.com/huggingface/transformers/pull/44956
289
+ """
290
+
291
+ model_type = "hyperclovax_vlm"
292
+ keys_to_ignore_at_inference = ["past_key_values"]
293
+
294
+ text_config_attribute_map = {
295
+ "n_embd": "hidden_size",
296
+ "n_positions": "max_position_embeddings",
297
+ "n_head": "num_attention_heads",
298
+ "n_layer": "num_hidden_layers",
299
+ }
300
+
301
+ def __init__(
302
+ self,
303
+ text_config=None,
304
+ vision_config=None,
305
+ use_nth_layer=-2,
306
+ img_start_id=100009,
307
+ decoder_max_length=4096,
308
+ anyres=False,
309
+ unpad=False,
310
+ max_num_grids=-1,
311
+ num_queries_vis_abstractor=-1,
312
+ ignore_index=-100,
313
+ proj_pos_emb=True,
314
+ proj_prenorm=False,
315
+ use_1x1_grid=False,
316
+ **kwargs,
317
+ ):
318
+ for key, val in self.text_config_attribute_map.items():
319
+ if text_config is not None and key in text_config:
320
+ text_config[val] = text_config.pop(key)
321
+
322
+ self.text_config = None
323
+ if text_config is not None:
324
+ _text_config = AutoConfig.for_model(text_config["model_type"])
325
+ self.text_config = _text_config.from_dict(text_config)
326
+ self.hidden_size = self.text_config.hidden_size
327
+
328
+ self.vision_config = None
329
+ if vision_config is not None:
330
+ _vision_config = AutoConfig.for_model(
331
+ vision_config["model_type"])
332
+ self.vision_config = _vision_config.from_dict(vision_config)
333
+
334
+ self.use_nth_layer = use_nth_layer
335
+ self.decoder_max_length = decoder_max_length
336
+ self.anyres = anyres
337
+ self.unpad = unpad
338
+ self.max_num_grids = max_num_grids
339
+ self.num_queries_vis_abstractor = num_queries_vis_abstractor
340
+ self.img_start_id = img_start_id
341
+ self.ignore_index = ignore_index
342
+ self.proj_pos_emb = proj_pos_emb
343
+ self.proj_prenorm = proj_prenorm
344
+ self.use_1x1_grid = use_1x1_grid
345
+ super().__init__(**kwargs)
346
+
347
+ def get_text_config(self, decoder=False):
348
+ if self.text_config is not None:
349
+ return self.text_config
350
+ return self