[CI] Prune PyTorch Fullgraph Test (#51074)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Aug 5, 2026 at 15:27 UTC
e6d67fddb4b27d4772ae714348a22af7fe7e35e5
3 files changed
+25
-124
.buildkite/test_areas/pytorch.yaml
+4
-37
@@ -107,9 +107,9 @@ steps:
107
commands:
108
- pytest -s -v compile/passes --ignore compile/passes/distributed
109
110
-- label: PyTorch Fullgraph Smoke Test
110
+- label: PyTorch Fullgraph Test
111
device: h200_35gb
112
- key: pytorch-fullgraph-smoke-test
112
+ key: pytorch-fullgraph-test
113
timeout_in_minutes: 90
114
source_file_dependencies:
115
- vllm/__init__.py
@@ -138,11 +138,11 @@ steps:
138
- vllm/v1/
139
- tests/compile
140
commands:
141
- # Run smoke tests under fullgraph directory, except test_full_graph.py
141
+ # Run smoke tests under fullgraph directory, except test_full_cudagraph.py
142
# as it is a heavy test that is covered in other steps.
143
# Use `find` to launch multiple instances of pytest so that
144
# they do not suffer from https://github.com/vllm-project/vllm/issues/28965
145
- - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
145
+ - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
146
147
# Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model
148
# instances and cannot fit a 35GB MIG slice. L4 retains the original coverage:
@@ -179,39 +179,6 @@ steps:
179
commands:
180
- pytest -s -v compile/fullgraph/test_full_cudagraph.py
181
182
-- label: PyTorch Fullgraph
183
- key: pytorch-fullgraph
184
- timeout_in_minutes: 40
185
- device: h200_18gb
186
- source_file_dependencies:
187
- - vllm/__init__.py
188
- - vllm/_aiter_ops.py
189
- - vllm/_custom_ops.py
190
- - vllm/compilation/
191
- - vllm/config/
192
- - vllm/distributed/
193
- - vllm/engine/
194
- - vllm/env_override.py
195
- - vllm/envs.py
196
- - vllm/forward_context.py
197
- - vllm/inputs/
198
- - vllm/ir/
199
- - vllm/kernels/
200
- - vllm/logger.py
201
- - vllm/model_executor/
202
- - vllm/multimodal/
203
- - vllm/platforms/
204
- - vllm/plugins/
205
- - vllm/sampling_params.py
206
- - vllm/sequence.py
207
- - vllm/transformers_utils/
208
- - vllm/triton_utils/
209
- - vllm/utils/
210
- - vllm/v1/
211
- - tests/compile
212
- commands:
213
- - pytest -v -s compile/fullgraph/test_full_graph.py
214
-
182
- label: Pytorch Nightly Dependency Override Check # 2min
183
key: pytorch-nightly-dependency-override-check
184
# if this test fails, it means the nightly torch version is not compatible with some
tests/compile/fullgraph/test_basic_correctness.py
+19
-53
@@ -36,21 +36,12 @@ class TestSetting:
36
attn_backend=ATTN_BACKEND,
37
method="generate",
38
),
39
- # llama model with quantization
40
- TestSetting(
41
- model="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
42
- model_args=["--quantization", "gptq", "--max-model-len", "2048"],
43
- pp_size=1,
44
- tp_size=1,
45
- attn_backend=ATTN_BACKEND,
46
- method="generate",
47
- ),
39
# MoE model
40
TestSetting(
50
- model="ibm/PowerMoE-3b",
41
+ model="ibm-granite/granite-3.0-1b-a400m-instruct",
42
model_args=["--max-model-len", "2048"],
43
pp_size=1,
53
- tp_size=2,
44
+ tp_size=1,
45
attn_backend=ATTN_BACKEND,
46
method="generate",
47
),
@@ -72,30 +63,6 @@ class TestSetting:
63
attn_backend=ATTN_BACKEND,
64
method="encode",
65
),
75
- pytest.param(
76
- TestSetting(
77
- model="BAAI/bge-base-en-v1.5",
78
- model_args=["--runner", "pooling"],
79
- pp_size=1,
80
- tp_size=1,
81
- attn_backend="FLASH_ATTN",
82
- method="encode",
83
- ),
84
- marks=pytest.mark.skipif(
85
- current_platform.is_rocm(),
86
- reason="Encoder self-attention is not implemented for ROCm",
87
- ),
88
- ),
89
- # vision language model
90
- # See https://github.com/vllm-project/vllm/issues/26716.
91
- # TestSetting(
92
- # model="microsoft/Phi-3.5-vision-instruct",
93
- # model_args=["--trust-remote-code", "--max-model-len", "2048"],
94
- # pp_size=2,
95
- # tp_size=1,
96
- # attn_backend="FLASH_ATTN",
97
- # method="generate_with_image",
98
- # ),
66
],
67
)
68
def test_compile_correctness(
@@ -129,29 +96,29 @@ def test_compile_correctness(
96
all_args: list[list[str]] = []
97
all_envs: list[dict[str, str] | None] = []
98
132
- for comp_mode in [
99
+ # Test all compilation modes with inductor backend
100
+ for mode in [
101
+ CompilationMode.NONE,
102
CompilationMode.STOCK_TORCH_COMPILE,
103
CompilationMode.DYNAMO_TRACE_ONCE,
104
CompilationMode.VLLM_COMPILE,
105
]:
137
- for mode in [CompilationMode.NONE, comp_mode]:
138
- all_args.append(
139
- final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"]
140
- )
141
- all_envs.append({})
106
+ all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"])
107
+ all_envs.append({})
108
+ # inductor will change the output, so we only compare if the output
109
+ # is close, not exactly the same.
110
+ compare_all_settings(
111
+ model,
112
+ all_args,
113
+ all_envs,
114
+ method=method if method != "generate" else "generate_close",
115
+ force_v1_runner=True,
116
+ )
117
143
- # inductor will change the output, so we only compare if the output
144
- # is close, not exactly the same.
145
- compare_all_settings(
146
- model,
147
- all_args,
148
- all_envs,
149
- method=method if method != "generate" else "generate_close",
150
- force_v1_runner=True,
151
- )
152
- all_envs.clear()
153
- all_args.clear()
118
+ all_envs.clear()
119
+ all_args.clear()
120
121
+ # Test all compilation modes with eager backend
122
for mode in [
123
CompilationMode.NONE,
124
CompilationMode.STOCK_TORCH_COMPILE,
@@ -160,5 +127,4 @@ def test_compile_correctness(
127
]:
128
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"])
129
all_envs.append({})
163
-
130
compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True)
tests/compile/fullgraph/test_full_graph.py
+2
-34
@@ -21,28 +21,12 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
21
TEST_MODELS: list[tuple[str, dict[str, Any]]] = [
22
("facebook/opt-125m", {}),
23
(
24
- "neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic",
24
+ "RedHatAI/Llama-3.2-1B-Instruct-FP8-dynamic",
25
{"dtype": torch.float16},
26
),
27
- ("meta-llama/Llama-3.2-1B-Instruct", {}),
27
]
28
29
if all:
31
- TEST_MODELS.extend(
32
- [
33
- ("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {}),
34
- (
35
- "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
36
- {"dtype": torch.float16},
37
- ),
38
- ]
39
- )
40
-
41
- if is_quant_method_supported("gptq"):
42
- TEST_MODELS.append(
43
- ("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {"quantization": "gptq"})
44
- )
45
-
30
if is_quant_method_supported("gptq_marlin"):
31
TEST_MODELS.append(
32
(
@@ -76,14 +60,6 @@ def test_full_graph(
60
model_kwargs: dict[str, Any],
61
compilation_mode: CompilationMode,
62
):
79
- if (
80
- "w8a8" in model
81
- or "w8w8" in model
82
- and current_platform.has_device_capability((10, 0))
83
- ):
84
- # int8 removed on Blackwell:
85
- pytest.skip("int8 support removed on Blackwell")
86
-
63
with monkeypatch.context():
64
print(f"MODEL={model}")
65
@@ -113,7 +89,7 @@ def test_full_graph(
89
),
90
*model_info,
91
)
116
- for model_info in models_list(keywords=["FP8-dynamic", "quantized.w8a8"])
92
+ for model_info in models_list(keywords=["FP8-dynamic"])
93
]
94
+ [
95
# Test depyf integration works
@@ -164,14 +140,6 @@ def test_custom_compile_config(
140
model: str,
141
model_kwargs: dict[str, Any],
142
):
167
- if (
168
- "w8a8" in model
169
- or "w8w8" in model
170
- and current_platform.has_device_capability((10, 0))
171
- ):
172
- # int8 removed on Blackwell:
173
- pytest.skip("int8 support removed on Blackwell")
174
-
143
if compilation_config.use_inductor_graph_partition and not is_torch_equal_or_newer(
144
"2.9.0.dev"
145
):