[Test][VLM] Add batch-invariance tests for Qwen3-VL (#53531)
Signed-off-by: machero <sr1123640465@126.com> Co-authored-by: Claude <noreply@anthropic.com>
machero committed
Aug 30, 2026 at 21:49 UTC
87b9b5b8d9dadc3edb31efa6ea71ee7d49d0bdcd
3 files changed
+105
.buildkite/test_areas/misc.yaml
+1
@@ -435,6 +435,7 @@ steps:
435
- pip install pytest-timeout pytest-forked
436
- pytest -v -s v1/determinism/test_batch_invariance.py
437
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
438
+ - pytest -v -s v1/determinism/test_batch_invariance_vlm.py
439
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
440
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
441
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
docs/features/batch_invariance.md
+1
@@ -102,6 +102,7 @@ Batch invariance has been tested and verified on the following models:
102
103
- **DeepSeek series**: `deepseek-ai/DeepSeek-V3`, `deepseek-ai/DeepSeek-V3-0324`, `deepseek-ai/DeepSeek-R1`, `deepseek-ai/DeepSeek-V3.1`
104
- **Qwen3 (Dense)**: `Qwen/Qwen3-1.7B`, `Qwen/Qwen3-8B`, `Qwen/Qwen3-4B-AWQ`, `Qwen/Qwen3-8B-AWQ`
105
+- **Qwen3-VL (Vision-Language)**: `Qwen/Qwen3-VL-2B-Instruct`, `Qwen/Qwen3-VL-4B-Instruct` (single image and video inputs)
106
- **Qwen3 (MoE)**: `Qwen/Qwen3-30B-A3B`, `Qwen/Qwen3-Next-80B-A3B-Instruct`, `Qwen/Qwen3-30B-A3B-Thinking-2507-FP8`
107
- **Qwen2.5**: `Qwen/Qwen2.5-0.5B-Instruct`, `Qwen/Qwen2.5-1.5B-Instruct`, `Qwen/Qwen2.5-3B-Instruct`, `Qwen/Qwen2.5-7B-Instruct`, `Qwen/Qwen2.5-14B-Instruct`, `Qwen/Qwen2.5-32B-Instruct`
108
- **Llama 3**: Llama3.1 and 3.2 series, `meta-llama/Llama-3.2-3B-Instruct` for example
tests/v1/determinism/test_batch_invariance_vlm.py
new
+103
@@ -0,0 +1,103 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Batch-invariance regression tests for vision-language models (VLMs)."""
4
+
5
+import contextlib
6
+import os
7
+
8
+import numpy as np
9
+import pytest
10
+import torch
11
+from PIL import Image
12
+from utils import _extract_step_logprobs, skip_if_not_cuda
13
+
14
+from vllm import LLM, SamplingParams
15
+
16
+VLM_TEST_MODEL = os.getenv("VLLM_VLM_TEST_MODEL", "Qwen/Qwen3-VL-2B-Instruct")
17
+
18
+IMAGE_PROMPT = (
19
+ "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
20
+ "Describe this image.<|im_end|><|im_start|>assistant\n"
21
+)
22
+VIDEO_PROMPT = (
23
+ "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>"
24
+ "Describe this video.<|im_end|><|im_start|>assistant\n"
25
+)
26
+
27
+
28
+def _make_image(seed: int, size: int) -> Image.Image:
29
+ rng = np.random.default_rng(seed)
30
+ return Image.fromarray(rng.integers(0, 255, (size, size, 3), dtype=np.uint8))
31
+
32
+
33
+def _make_inputs(input_type: str, num_reqs: int) -> list[dict]:
34
+ inputs = []
35
+ for i in range(num_reqs):
36
+ if input_type == "image":
37
+ prompt = IMAGE_PROMPT
38
+ mm_data = {"image": _make_image(1000 + i, size=256)}
39
+ else:
40
+ frames = [_make_image(1000 + i + f, size=128) for f in range(4)]
41
+ prompt = VIDEO_PROMPT
42
+ mm_data = {
43
+ "video": (
44
+ frames,
45
+ {
46
+ "total_num_frames": len(frames),
47
+ "fps": 1.0,
48
+ "frames_indices": list(range(len(frames))),
49
+ "do_sample_frames": False,
50
+ },
51
+ )
52
+ }
53
+ inputs.append({"prompt": prompt, "multi_modal_data": mm_data})
54
+ return inputs
55
+
56
+
57
+def _assert_batch_invariant(llm: LLM, inputs: list[dict], sampling) -> None:
58
+ bs1 = [
59
+ _extract_step_logprobs(llm.generate([inp], sampling, use_tqdm=False)[0])
60
+ for inp in inputs
61
+ ]
62
+ bsN = [_extract_step_logprobs(out) for out in llm.generate(inputs, sampling)]
63
+
64
+ failures = []
65
+ for i, ((lp1, t1), (lpN, tN)) in enumerate(zip(bs1, bsN)):
66
+ if lp1 is None or lpN is None:
67
+ pytest.skip(
68
+ "Logits are not available on RequestOutput; "
69
+ "enable logprobs return to run this test."
70
+ )
71
+ if t1 != tN:
72
+ failures.append(f"req {i}: token mismatch bs1={t1} bsN={tN}")
73
+ elif not torch.equal(lp1, lpN):
74
+ d = (lp1 - lpN).abs().max().item()
75
+ failures.append(f"req {i}: logprob mismatch max_diff={d:.3e}")
76
+
77
+ if failures:
78
+ pytest.fail(
79
+ f"Batch invariance violated for {len(failures)}/{len(inputs)} "
80
+ f"requests:\n" + "\n".join(failures)
81
+ )
82
+
83
+
84
+@skip_if_not_cuda
85
+@pytest.mark.parametrize("input_type", ["image", "video"])
86
+@pytest.mark.parametrize("mm_encoder_attn_backend", ["FLASH_ATTN", "TORCH_SDPA"])
87
+def test_vlm_batch_invariance_bs1_vs_bsN(input_type: str, mm_encoder_attn_backend: str):
88
+ inputs = _make_inputs(input_type, num_reqs=4)
89
+ sampling = SamplingParams(temperature=0.0, max_tokens=8, seed=1234, logprobs=5)
90
+
91
+ llm = LLM(
92
+ model=VLM_TEST_MODEL,
93
+ dtype="bfloat16",
94
+ enforce_eager=True,
95
+ max_num_seqs=16,
96
+ gpu_memory_utilization=0.85,
97
+ mm_encoder_attn_backend=mm_encoder_attn_backend,
98
+ )
99
+ try:
100
+ _assert_batch_invariant(llm, inputs, sampling)
101
+ finally:
102
+ with contextlib.suppress(Exception):
103
+ llm.shutdown()