A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 19:00 UTC
[Feature] Support compile mode for batch invariance on SM80 (#42456)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
May 13, 2026 at 11:02 UTC
e35c0d4c63128b38a6a3cee63dc3e20626802e9d
2 files changed
+14
-8
.buildkite/test_areas/misc.yaml
+14
@@ -305,6 +305,20 @@ steps:
305
- pytest -v -s transformers_utils
306
- pytest -v -s config
307
308
+- label: Batch Invariance (A100)
309
+ key: batch-invariance-a100
310
+ timeout_in_minutes: 30
311
+ device: a100
312
+ source_file_dependencies:
313
+ - vllm/v1/attention
314
+ - vllm/model_executor/layers
315
+ - tests/v1/determinism/
316
+ commands:
317
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
318
+ - pip install pytest-timeout pytest-forked
319
+ - pytest -v -s v1/determinism/test_batch_invariance.py
320
+ - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
321
+
322
- label: Batch Invariance (H100)
323
key: batch-invariance-h100
324
timeout_in_minutes: 30
tests/v1/determinism/test_batch_invariance.py
-8
@@ -11,15 +11,12 @@ from utils import (
11
TEST_MODEL,
12
_extract_step_logprobs,
13
_random_prompt,
14
- is_device_capability_below_90,
14
skip_unsupported,
15
)
16
17
import vllm.envs as envs
18
from vllm import LLM, SamplingParams
19
21
-IS_DEVICE_CAPABILITY_BELOW_90 = is_device_capability_below_90()
22
-
20
21
@skip_unsupported
22
@pytest.mark.timeout(1000)
@@ -175,7 +172,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
172
max_model_len=8192,
173
dtype="auto", # not everything is supported
174
gpu_memory_utilization=0.9,
178
- enforce_eager=IS_DEVICE_CAPABILITY_BELOW_90,
175
attention_config={"backend": backend},
176
)
177
@@ -388,7 +384,6 @@ def test_simple_generation(backend):
384
max_model_len=2048,
385
dtype="auto",
386
enable_prefix_caching=False,
391
- enforce_eager=IS_DEVICE_CAPABILITY_BELOW_90,
387
attention_config={"backend": backend},
388
)
389
@@ -453,7 +448,6 @@ def test_logprobs_without_batch_invariance_should_fail(
448
max_num_seqs=32,
449
max_model_len=8192,
450
dtype="auto",
456
- enforce_eager=IS_DEVICE_CAPABILITY_BELOW_90,
451
attention_config={"backend": backend},
452
)
453
@@ -673,7 +667,6 @@ def test_decode_logprobs_match_prefill_logprobs(
667
max_num_seqs=32,
668
max_model_len=8192,
669
dtype="auto",
676
- enforce_eager=IS_DEVICE_CAPABILITY_BELOW_90,
670
attention_config={"backend": backend},
671
)
672
@@ -920,7 +913,6 @@ def LLM_with_max_seqs(
913
dtype="auto",
914
tensor_parallel_size=int(os.getenv("VLLM_TP_SIZE", "1")),
915
enable_prefix_caching=False,
923
- enforce_eager=IS_DEVICE_CAPABILITY_BELOW_90,
916
attention_config=attention_config,
917
# Enable for MOE models
918
# enable_expert_parallel=True,