A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Clean up duplicated hf overrides (#22311)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Aug 6, 2025 at 15:50 UTC
fa00c5d75bc63c87f5822f839db1342f19e4acc8
3 files changed
+71
-103
tests/models/multimodal/test_tensor_schema.py
+3
-48
@@ -1,11 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from functools import partial
4
-from typing import Any
4
from unittest.mock import patch
5
6
import pytest
8
-from transformers import PretrainedConfig
7
8
from vllm.config import ModelConfig
9
from vllm.engine.llm_engine import LLMEngine as V0LLMEngine
@@ -19,6 +17,7 @@ from vllm.v1.engine.core import EngineCore as V1EngineCore
17
18
from ...conftest import VllmRunner
19
from ..registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS
20
+from ..utils import dummy_hf_overrides
21
22
ARCH_TO_SKIP = {
23
"MolmoForCausalLM": "incompatible requirements",
@@ -51,51 +50,6 @@ def create_batched_mm_kwargs(
50
return mm_kwargs
51
52
54
-# Avoid OOM and reduce initialization time by only using 1 layer
55
-def hf_overrides(hf_config: PretrainedConfig,
56
- exist_overrides: dict[str, Any]) -> PretrainedConfig:
57
- hf_config.update(exist_overrides)
58
- text_config = hf_config.get_text_config()
59
- # Ensure at least 2 expert per group
60
- # Since `grouped_topk` assumes top-2
61
- n_group = getattr(text_config, 'n_group', None)
62
- num_experts = n_group * 2 if n_group is not None else 2
63
- # we use three layers for Gemma-3n to check
64
- # both normal layer and kv_shared_layer
65
- text_config.update({
66
- "num_layers": 1,
67
- "num_hidden_layers": 1,
68
- "num_experts": num_experts,
69
- "num_experts_per_tok": 2,
70
- "num_local_experts": num_experts,
71
- # Otherwise there will not be any expert layers
72
- "first_k_dense_replace": 0,
73
- # To avoid OOM on DeepSeek-V3
74
- "n_routed_experts": num_experts,
75
- # For Gemma-3n
76
- "num_kv_shared_layers": 1,
77
- })
78
- if hasattr(hf_config, "vision_config"):
79
- hf_config.vision_config.update({
80
- "num_layers": 1,
81
- "num_hidden_layers": 1,
82
- })
83
- # e.g.: ibm-granite/granite-speech-3.3-2b
84
- if hasattr(hf_config, "encoder_config"):
85
- hf_config.encoder_config.update({
86
- "num_layers": 1,
87
- "num_hidden_layers": 1,
88
- })
89
- # e.g.: Qwen/Qwen2-Audio-7B-Instruct
90
- if hasattr(hf_config, "audio_config"):
91
- hf_config.audio_config.update({
92
- "num_layers": 1,
93
- "num_hidden_layers": 1,
94
- "encoder_layers": 1,
95
- })
96
- return hf_config
97
-
98
-
53
@pytest.mark.core_model
54
@pytest.mark.parametrize("model_arch", list(_MULTIMODAL_EXAMPLE_MODELS.keys()))
55
def test_model_tensor_schema(model_arch: str, vllm_runner: type[VllmRunner],
@@ -110,7 +64,8 @@ def test_model_tensor_schema(model_arch: str, vllm_runner: type[VllmRunner],
64
65
model_id = model_info.default
66
113
- hf_overrides_fn = partial(hf_overrides,
67
+ hf_overrides_fn = partial(dummy_hf_overrides,
68
+ model_arch=model_arch,
69
exist_overrides=model_info.hf_overrides)
70
71
model_config = ModelConfig(
tests/models/test_initialization.py
+7
-55
@@ -1,10 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from functools import partial
5
from unittest.mock import patch
6
7
import pytest
7
-from transformers import PretrainedConfig
8
9
from vllm import LLM
10
from vllm.config import ModelImpl
@@ -16,6 +16,7 @@ from vllm.v1.engine.core import EngineCore as V1EngineCore
16
from ..utils import create_new_process_for_each_test
17
from .registry import (_TRANSFORMERS_BACKEND_MODELS, AUTO_EXAMPLE_MODELS,
18
HF_EXAMPLE_MODELS, HfExampleModels)
19
+from .utils import dummy_hf_overrides
20
21
22
@create_new_process_for_each_test()
@@ -33,64 +34,15 @@ def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
34
model_info.check_available_online(on_fail="skip")
35
model_info.check_transformers_version(on_fail="skip")
36
37
+ hf_overrides_fn = partial(dummy_hf_overrides,
38
+ model_arch=model_arch,
39
+ exist_overrides=model_info.hf_overrides)
40
+
41
if model_arch in ("Llama4ForCausalLM", "EagleLlama4ForCausalLM"):
42
from vllm.model_executor.models.llama4 import Llama4ForCausalLM
43
from vllm.model_executor.models.registry import ModelRegistry
44
ModelRegistry.register_model("Llama4ForCausalLM", Llama4ForCausalLM)
45
41
- # Avoid OOM and reduce initialization time by only using 1 layer
42
- def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
43
- hf_config.update(model_info.hf_overrides)
44
-
45
- text_config = hf_config.get_text_config()
46
-
47
- # Ensure at least 2 expert per group
48
- # Since `grouped_topk` assumes top-2
49
- n_group = getattr(text_config, 'n_group', None)
50
- num_experts = n_group * 2 if n_group is not None else 2
51
-
52
- # we use three layers for Gemma-3n to check
53
- # both normal layer and kv_shared_layer
54
- num_hidden_layers = (3 if model_arch
55
- == "Gemma3nForConditionalGeneration" else 1)
56
-
57
- text_config.update({
58
- "num_layers": 1,
59
- "num_hidden_layers": num_hidden_layers,
60
- "num_experts": num_experts,
61
- "num_experts_per_tok": 2,
62
- "num_local_experts": num_experts,
63
- # Otherwise there will not be any expert layers
64
- "first_k_dense_replace": 0,
65
- # To avoid OOM on DeepSeek-V3
66
- "n_routed_experts": num_experts,
67
- # For Gemma-3n
68
- "num_kv_shared_layers": 1,
69
- })
70
-
71
- if hasattr(hf_config, "vision_config"):
72
- hf_config.vision_config.update({
73
- "num_layers": 1,
74
- "num_hidden_layers": 1,
75
- })
76
-
77
- # e.g.: ibm-granite/granite-speech-3.3-2b
78
- if hasattr(hf_config, "encoder_config"):
79
- hf_config.encoder_config.update({
80
- "num_layers": 1,
81
- "num_hidden_layers": 1,
82
- })
83
-
84
- # e.g.: Qwen/Qwen2-Audio-7B-Instruct
85
- if hasattr(hf_config, "audio_config"):
86
- hf_config.audio_config.update({
87
- "num_layers": 1,
88
- "num_hidden_layers": 1,
89
- "encoder_layers": 1,
90
- })
91
-
92
- return hf_config
93
-
46
# Avoid calling model.forward()
47
def _initialize_kv_caches_v0(self) -> None:
48
self.cache_config.num_gpu_blocks = 0
@@ -132,7 +84,7 @@ def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
84
load_format="dummy",
85
model_impl=ModelImpl.TRANSFORMERS
86
if model_arch in _TRANSFORMERS_BACKEND_MODELS else ModelImpl.VLLM,
135
- hf_overrides=hf_overrides,
87
+ hf_overrides=hf_overrides_fn,
88
)
89
90
tests/models/utils.py
+61
@@ -7,6 +7,7 @@ from typing import Any, NamedTuple, Optional, Union
7
8
import torch
9
import torch.nn.functional as F
10
+from transformers import PretrainedConfig
11
12
from vllm.config import ModelConfig, RunnerOption
13
from vllm.inputs import InputContext
@@ -351,3 +352,63 @@ class RerankModelInfo(NamedTuple):
352
architecture: str = ""
353
dtype: str = "auto"
354
enable_test: bool = True
355
+
356
+
357
+def dummy_hf_overrides(
358
+ hf_config: PretrainedConfig,
359
+ model_arch: str,
360
+ exist_overrides: Optional[dict[str, Any]] = None,
361
+) -> PretrainedConfig:
362
+ """
363
+ Dummy HF overrides function used to create dummy model
364
+ with only minimum nums of layer.
365
+ """
366
+ hf_config.update(exist_overrides or {})
367
+
368
+ text_config = hf_config.get_text_config()
369
+
370
+ # Ensure at least 2 expert per group
371
+ # Since `grouped_topk` assumes top-2
372
+ n_group = getattr(text_config, 'n_group', None)
373
+ num_experts = n_group * 2 if n_group is not None else 2
374
+
375
+ # we use three layers for Gemma-3n to check
376
+ # both normal layer and kv_shared_layer
377
+ num_hidden_layers = (3 if model_arch == "Gemma3nForConditionalGeneration"
378
+ else 1)
379
+ text_config.update({
380
+ "num_layers": 1,
381
+ "num_hidden_layers": num_hidden_layers,
382
+ "num_experts": num_experts,
383
+ "num_experts_per_tok": 2,
384
+ "num_local_experts": num_experts,
385
+ # Otherwise there will not be any expert layers
386
+ "first_k_dense_replace": 0,
387
+ # To avoid OOM on DeepSeek-V3
388
+ "n_routed_experts": num_experts,
389
+ # For Gemma-3n
390
+ "num_kv_shared_layers": 1,
391
+ })
392
+
393
+ if hasattr(hf_config, "vision_config"):
394
+ hf_config.vision_config.update({
395
+ "num_layers": 1,
396
+ "num_hidden_layers": 1,
397
+ })
398
+
399
+ # e.g.: ibm-granite/granite-speech-3.3-2b
400
+ if hasattr(hf_config, "encoder_config"):
401
+ hf_config.encoder_config.update({
402
+ "num_layers": 1,
403
+ "num_hidden_layers": 1,
404
+ })
405
+
406
+ # e.g.: Qwen/Qwen2-Audio-7B-Instruct
407
+ if hasattr(hf_config, "audio_config"):
408
+ hf_config.audio_config.update({
409
+ "num_layers": 1,
410
+ "num_hidden_layers": 1,
411
+ "encoder_layers": 1,
412
+ })
413
+
414
+ return hf_config