A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 22:15 UTC
[Bug] Fix torch dynamo warning Dynamo detected a call to a `functools.lru_cache` (#29038)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Nov 20, 2025 at 03:52 UTC
2c52c7fd9a480f96ac93e63eccf9a3ee01686ad4
5 files changed
+52
-40
tests/v1/determinism/conftest.py
+3
-2
@@ -1,11 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
import pytest
4
5
+import vllm.model_executor.layers.batch_invariant as batch_invariant
6
+
7
8
@pytest.fixture(autouse=True)
9
def enable_batch_invariant_mode(monkeypatch: pytest.MonkeyPatch):
10
"""Automatically enable batch invariant kernel overrides for all tests."""
11
+ monkeypatch.setattr(batch_invariant, "VLLM_BATCH_INVARIANT", True)
12
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")
11
- yield
tests/v1/determinism/test_batch_invariance.py
+9
-26
@@ -6,29 +6,16 @@ import random
6
7
import pytest
8
import torch
9
-from utils import _extract_step_logprobs, _random_prompt, skip_unsupported
9
+from utils import (
10
+ BACKENDS,
11
+ _extract_step_logprobs,
12
+ _random_prompt,
13
+ resolve_model_name,
14
+ skip_unsupported,
15
+)
16
17
+import vllm.model_executor.layers.batch_invariant as batch_invariant
18
from vllm import LLM, SamplingParams
12
-from vllm.platforms import current_platform
13
-
14
-BACKENDS: list[str] = [
15
- "FLASH_ATTN",
16
- "FLASHINFER",
17
-]
18
-
19
-if current_platform.is_cuda() and current_platform.is_device_capability(90):
20
- BACKENDS.append("FLASH_ATTN_MLA")
21
-
22
-DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
23
-MLA_MODEL = "deepseek-ai/DeepSeek-V2-Lite-Chat"
24
-
25
-
26
-def resolve_model_name(backend: str) -> str:
27
- """Resolve the model name for the given backend, respecting env overrides."""
28
- model = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
29
- if backend.endswith("MLA") and model == DEFAULT_MODEL:
30
- return MLA_MODEL
31
- return model
19
20
21
@skip_unsupported
@@ -454,14 +441,10 @@ def test_logprobs_without_batch_invariance_should_fail(
441
The test will PASS if we detect differences (proving batch invariance matters).
442
The test will FAIL if everything matches (suggesting batch invariance isn't needed).
443
"""
457
- from vllm.model_executor.layers.batch_invariant import vllm_is_batch_invariant
458
-
459
- vllm_is_batch_invariant.cache_clear()
444
monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend)
445
446
# CRITICAL: Disable batch invariance for this test
463
- monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0")
464
-
447
+ monkeypatch.setattr(batch_invariant, "VLLM_BATCH_INVARIANT", False)
448
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
449
random.seed(seed)
450
model_name = resolve_model_name(backend)
tests/v1/determinism/test_online_batch_invariance.py
+9
-3
@@ -16,7 +16,8 @@ import sys
16
from typing import Any
17
18
import openai
19
-from utils import _random_prompt, skip_unsupported
19
+import pytest
20
+from utils import BACKENDS, _random_prompt, resolve_model_name, skip_unsupported
21
22
from tests.utils import RemoteOpenAIServer
23
@@ -133,9 +134,14 @@ def _compare_bs1_vs_bsn_single_process(
134
135
136
@skip_unsupported
136
-def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN():
137
+@pytest.mark.parametrize("backend", BACKENDS)
138
+def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
139
+ backend: str, monkeypatch: pytest.MonkeyPatch
140
+) -> None:
141
random.seed(int(os.getenv("VLLM_TEST_SEED", "12345")))
138
- model_name = os.getenv("VLLM_TEST_MODEL", "Qwen/Qwen3-1.7B")
142
+ # Override backend for this test (and the RemoteOpenAIServer child process).
143
+ monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend)
144
+ model_name = resolve_model_name(backend)
145
prompts_all = [_random_prompt(10, 50) for _ in range(32)]
146
147
sp_kwargs: dict[str, Any] = {
tests/v1/determinism/utils.py
+20
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import os
4
import random
5
6
import pytest
@@ -12,6 +13,25 @@ skip_unsupported = pytest.mark.skipif(
13
reason="Requires CUDA and >= Hopper (SM90)",
14
)
15
16
+BACKENDS: list[str] = [
17
+ "FLASH_ATTN",
18
+ "FLASHINFER",
19
+]
20
+
21
+if current_platform.is_cuda() and current_platform.is_device_capability(90):
22
+ BACKENDS.append("FLASH_ATTN_MLA")
23
+
24
+DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
25
+MLA_MODEL = "deepseek-ai/DeepSeek-V2-Lite-Chat"
26
+
27
+
28
+def resolve_model_name(backend: str) -> str:
29
+ """Resolve the model name for the given backend."""
30
+ model = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
31
+ if backend.endswith("MLA") and model == DEFAULT_MODEL:
32
+ return MLA_MODEL
33
+ return model
34
+
35
36
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
37
# Generate more realistic prompts that will actually produce varied tokens
vllm/model_executor/layers/batch_invariant.py
+11
-9
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import os
4
from collections.abc import Callable
5
-from functools import cache
5
from typing import Any
6
7
import torch
@@ -785,16 +784,19 @@ def enable_batch_invariant_mode():
784
torch.backends.cuda.preferred_blas_library(backend="cublaslt")
785
786
788
-@cache
789
-def vllm_is_batch_invariant():
790
- env_key = "VLLM_BATCH_INVARIANT"
791
- is_overridden = False
792
- val = os.getenv(env_key, "0")
787
+def _read_vllm_batch_invariant() -> bool:
788
+ val = os.getenv("VLLM_BATCH_INVARIANT", "0")
789
try:
794
- is_overridden = int(val) != 0
790
+ return int(val) != 0
791
except ValueError:
796
- is_overridden = False
797
- return is_overridden
792
+ return False
793
+
794
+
795
+VLLM_BATCH_INVARIANT: bool = _read_vllm_batch_invariant()
796
+
797
+
798
+def vllm_is_batch_invariant() -> bool:
799
+ return VLLM_BATCH_INVARIANT
800
801
802
def override_envs_for_invariance():