A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 08:15 UTC
[Refactor] Relocate tests from tests/v1/entrypoints/ to tests/entrypoints/ (#37500)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Mar 20, 2026 at 05:50 UTC
b4c1aef21c1a4cb252e7a440b3f9b0baebefbbef
16 files changed
+171
-235
.buildkite/scripts/hardware_ci/run-tpu-v1-test-part2.sh
+1
-1
@@ -127,7 +127,7 @@ run_and_track_test() {
127
128
# --- Actual Test Execution ---
129
run_and_track_test 1 "test_struct_output_generate.py" \
130
- "python3 -m pytest -s -v /workspace/vllm/tests/v1/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
130
+ "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
131
run_and_track_test 2 "test_moe_pallas.py" \
132
"python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_moe_pallas.py"
133
run_and_track_test 3 "test_lora.py" \
.buildkite/test-amd.yaml
+4
-30
@@ -483,19 +483,6 @@ steps:
483
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
484
485
486
-- label: Entrypoints V1 # TBD
487
- timeout_in_minutes: 180
488
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
489
- agent_pool: mi250_1
490
- optional: true
491
- working_dir: "/vllm-workspace/tests"
492
- source_file_dependencies:
493
- - vllm/
494
- - tests/v1
495
- commands:
496
- - pytest -v -s v1/entrypoints
497
-
498
-
486
- label: V1 Sample + Logits # TBD
487
timeout_in_minutes: 60
488
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
@@ -1173,14 +1160,14 @@ steps:
1160
- vllm/v1/engine/
1161
- vllm/v1/worker/
1162
- tests/v1/distributed
1176
- - tests/v1/entrypoints/openai/test_multi_api_servers.py
1163
+ - tests/entrypoints/openai/test_multi_api_servers.py
1164
- vllm/platforms/rocm.py
1165
commands:
1166
- export TORCH_NCCL_BLOCKING_WAIT=1
1167
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
1168
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
1169
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
1183
- - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
1170
+ - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
1171
1172
1173
- label: Distributed Compile + RPC Tests (2 GPUs) # TBD
@@ -1766,19 +1753,6 @@ steps:
1753
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
1754
1755
1769
-- label: Entrypoints V1 # 25.7m
1770
- timeout_in_minutes: 45
1771
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
1772
- agent_pool: mi325_1
1773
- optional: true
1774
- working_dir: "/vllm-workspace/tests"
1775
- source_file_dependencies:
1776
- - vllm/
1777
- - tests/v1
1778
- commands:
1779
- - pytest -v -s v1/entrypoints
1780
-
1781
-
1756
- label: V1 Spec Decode # TBD
1757
timeout_in_minutes: 40
1758
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
@@ -2391,14 +2365,14 @@ steps:
2365
- vllm/v1/engine/
2366
- vllm/v1/worker/
2367
- tests/v1/distributed
2394
- - tests/v1/entrypoints/openai/test_multi_api_servers.py
2368
+ - tests/entrypoints/openai/test_multi_api_servers.py
2369
- vllm/platforms/rocm.py
2370
commands:
2371
- export TORCH_NCCL_BLOCKING_WAIT=1
2372
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
2373
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
2374
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
2401
- - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
2375
+ - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
2376
2377
2378
- label: Distributed Compile + RPC Tests (2 GPUs) # 56.1m
.buildkite/test_areas/distributed.yaml
+2
-2
@@ -27,14 +27,14 @@ steps:
27
- vllm/v1/engine/
28
- vllm/v1/worker/
29
- tests/v1/distributed
30
- - tests/v1/entrypoints/openai/test_multi_api_servers.py
30
+ - tests/entrypoints/openai/test_multi_api_servers.py
31
commands:
32
# https://github.com/NVIDIA/nccl/issues/1838
33
- export NCCL_CUMEM_HOST_ENABLE=0
34
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
35
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
36
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
37
- - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
37
+ - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
38
39
- label: Distributed Compile + RPC Tests (2 GPUs)
40
timeout_in_minutes: 20
.buildkite/test_areas/entrypoints.yaml
+1
-14
@@ -34,7 +34,7 @@ steps:
34
- tests/entrypoints/test_chat_utils
35
commands:
36
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
37
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
37
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
38
- pytest -v -s entrypoints/test_chat_utils.py
39
mirror:
40
amd:
@@ -75,19 +75,6 @@ steps:
75
commands:
76
- pytest -v -s entrypoints/openai/responses
77
78
-- label: Entrypoints V1
79
- timeout_in_minutes: 50
80
- source_file_dependencies:
81
- - vllm/
82
- - tests/v1
83
- commands:
84
- - pytest -v -s v1/entrypoints
85
- mirror:
86
- amd:
87
- device: mi325_1
88
- depends_on:
89
- - image-build-amd
90
-
78
- label: OpenAI API Correctness
79
timeout_in_minutes: 30
80
source_file_dependencies:
.buildkite/test_areas/model_runner_v2.yaml
+2
-2
@@ -11,7 +11,7 @@ steps:
11
- vllm/v1/attention/
12
- tests/v1/engine/test_llm_engine.py
13
- tests/v1/e2e/
14
- - tests/v1/entrypoints/llm/test_struct_output_generate.py
14
+ - tests/entrypoints/llm/test_struct_output_generate.py
15
commands:
16
- set -x
17
- export VLLM_USE_V2_MODEL_RUNNER=1
@@ -22,7 +22,7 @@ steps:
22
- pytest -v -s v1/e2e/general/test_context_length.py
23
- pytest -v -s v1/e2e/general/test_min_tokens.py
24
# Temporary hack filter to exclude ngram spec decoding based tests.
25
- - pytest -v -s v1/entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
25
+ - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
26
27
- label: Model Runner V2 Examples
28
timeout_in_minutes: 45
.github/CODEOWNERS
+1
-1
@@ -75,7 +75,7 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
75
/tests/multimodal @DarkLight1337 @ywang96 @NickLucche
76
/tests/quantization @mgoin @robertgshaw2-redhat @yewentao256 @pavanimajety
77
/tests/test_inputs.py @DarkLight1337 @ywang96
78
-/tests/v1/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
78
+/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
79
/tests/v1/structured_output @mgoin @russellb @aarnphm
80
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
81
/tests/weight_loading @mgoin @youkaichao @yewentao256
.github/mergify.yml
+1
-1
@@ -260,7 +260,7 @@ pull_request_rules:
260
- files=examples/offline_inference/structured_outputs.py
261
- files=examples/online_serving/structured_outputs/structured_outputs.py
262
- files~=^tests/v1/structured_output/
263
- - files=tests/v1/entrypoints/llm/test_struct_output_generate.py
263
+ - files=tests/entrypoints/llm/test_struct_output_generate.py
264
- files~=^vllm/v1/structured_output/
265
actions:
266
label:
tests/conftest.py
+49
-3
@@ -6,9 +6,6 @@ from copy import deepcopy
6
7
from tblib import pickling_support
8
9
-# Import fixture
10
-from tests.v1.entrypoints.conftest import sample_json_schema # noqa
11
-
9
# ruff: noqa
10
11
# Install support for pickling exceptions so that we can nicely propagate
@@ -81,6 +78,55 @@ if TYPE_CHECKING:
78
79
logger = init_logger(__name__)
80
81
+
82
+@pytest.fixture
83
+def sample_json_schema():
84
+ return {
85
+ "type": "object",
86
+ "properties": {
87
+ "name": {"type": "string"},
88
+ "age": {"type": "integer"},
89
+ "skills": {
90
+ "type": "array",
91
+ "items": {
92
+ "type": "string",
93
+ },
94
+ },
95
+ "grade": {
96
+ "type": "string",
97
+ "pattern": "^[A-D]$",
98
+ },
99
+ "email": {
100
+ "type": "string",
101
+ "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
102
+ },
103
+ "work_history": {
104
+ "type": "array",
105
+ "items": {
106
+ "type": "object",
107
+ "properties": {
108
+ "company": {"type": "string"},
109
+ "duration": {
110
+ "type": "number",
111
+ "minimum": 0.0,
112
+ "maximum": 100.0,
113
+ },
114
+ "position": {"type": "string"},
115
+ },
116
+ "required": ["company", "duration", "position"],
117
+ "additionalProperties": False,
118
+ },
119
+ "minItems": 0,
120
+ "maxItems": 3,
121
+ },
122
+ },
123
+ "required": ["name", "age", "skills", "grade", "email", "work_history"],
124
+ "additionalProperties": False,
125
+ "minProperties": 1,
126
+ "maxProperties": 10,
127
+ }
128
+
129
+
130
_TEST_DIR = os.path.dirname(__file__)
131
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
132
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
tests/entrypoints/llm/test_struct_output_generate.py
renamed
+110
-8
@@ -24,6 +24,108 @@ from vllm.sampling_params import (
24
StructuredOutputsParams,
25
)
26
27
+SAMPLE_REGEX = (
28
+ r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
29
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
30
+)
31
+
32
+# Note: Ensure this only uses attributes compatible with xgrammar
33
+SAMPLE_JSON_SCHEMA = {
34
+ "type": "object",
35
+ "properties": {
36
+ "name": {"type": "string"},
37
+ "age": {"type": "integer"},
38
+ "skills": {
39
+ "type": "array",
40
+ "items": {
41
+ "type": "string",
42
+ },
43
+ },
44
+ "grade": {
45
+ "type": "string",
46
+ "pattern": "^[A-D]$", # Regex pattern
47
+ },
48
+ "email": {
49
+ "type": "string",
50
+ "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
51
+ },
52
+ "work_history": {
53
+ "type": "array",
54
+ "items": {
55
+ "type": "object",
56
+ "properties": {
57
+ "company": {"type": "string"},
58
+ "duration": {
59
+ "type": "number",
60
+ "minimum": 0.0,
61
+ "maximum": 100.0, # Numeric range
62
+ },
63
+ "position": {"type": "string"},
64
+ },
65
+ "required": ["company", "duration", "position"],
66
+ "additionalProperties": False,
67
+ },
68
+ "minItems": 0,
69
+ "maxItems": 3,
70
+ },
71
+ },
72
+ "required": ["name", "age", "skills", "grade", "email", "work_history"],
73
+ "additionalProperties": False,
74
+ "minProperties": 1,
75
+ "maxProperties": 10,
76
+}
77
+
78
+# A schema unsupported by xgrammar
79
+UNSUPPORTED_JSON_SCHEMA = {
80
+ "type": "object",
81
+ "properties": {
82
+ "score": {
83
+ "type": "integer",
84
+ "multipleOf": 5, # Numeric multiple
85
+ },
86
+ "tags": {
87
+ "type": "array",
88
+ "items": {"type": "string", "minLength": 10, "maxLength": 20},
89
+ },
90
+ },
91
+ "required": ["score", "tags"],
92
+ "additionalProperties": False,
93
+ "patternProperties": {
94
+ "^score$": {"type": "integer"},
95
+ },
96
+}
97
+
98
+SAMPLE_STRUCTURED_OUTPUTS_CHOICES = [
99
+ "Python",
100
+ "Java",
101
+ "JavaScript",
102
+ "C++",
103
+ "C#",
104
+ "PHP",
105
+ "TypeScript",
106
+ "Ruby",
107
+ "Swift",
108
+ "Kotlin",
109
+]
110
+
111
+SAMPLE_SQL_EBNF = """
112
+root ::= select_statement
113
+select_statement ::= "SELECT" column "from" table "where" condition
114
+column ::= "col_1" | "col_2"
115
+table ::= "table_1" | "table_2"
116
+condition ::= column "=" number
117
+number ::= "1" | "2"
118
+"""
119
+
120
+SAMPLE_SQL_LARK = """
121
+start: select_statement
122
+select_statement: "SELECT" column "from" table "where" condition
123
+column: "col_1" | "col_2"
124
+table: "table_1" | "table_2"
125
+condition: column "=" number
126
+number: "1" | "2"
127
+"""
128
+
129
NGRAM_SPEC_CONFIG = {
130
"model": "[ngram]",
131
"num_speculative_tokens": 5,
@@ -110,17 +212,17 @@ class CarDescription(BaseModel):
212
PARAMS_MODELS_BACKENDS_TOKENIZER_MODE,
213
)
214
def test_structured_output(
113
- sample_json_schema: dict[str, Any],
114
- unsupported_json_schema: dict[str, Any],
115
- sample_sql_ebnf: str,
116
- sample_sql_lark: str,
117
- sample_regex: str,
118
- sample_structured_outputs_choices: str,
215
backend: str,
216
tokenizer_mode: str,
217
model_name: str,
218
speculative_config: dict[str, Any],
219
):
220
+ sample_json_schema = SAMPLE_JSON_SCHEMA
221
+ unsupported_json_schema = UNSUPPORTED_JSON_SCHEMA
222
+ sample_sql_ebnf = SAMPLE_SQL_EBNF
223
+ sample_sql_lark = SAMPLE_SQL_LARK
224
+ sample_regex = SAMPLE_REGEX
225
+ sample_structured_outputs_choices = SAMPLE_STRUCTURED_OUTPUTS_CHOICES
226
if current_platform.is_tpu() and speculative_config:
227
pytest.skip("TPU does not support speculative decoding")
228
@@ -702,10 +804,10 @@ def test_structured_output_with_reasoning_matrices(
804
805
@pytest.mark.parametrize("model_name, tokenizer_mode", PARAMS_MODELS_TOKENIZER_MODE)
806
def test_structured_output_auto_mode(
705
- unsupported_json_schema: dict[str, Any],
807
model_name: str,
808
tokenizer_mode: str,
809
):
810
+ unsupported_json_schema = UNSUPPORTED_JSON_SCHEMA
811
llm = LLM(
812
model=model_name,
813
max_model_len=1024,
@@ -808,9 +910,9 @@ def test_guidance_no_additional_properties():
910
911
@pytest.mark.parametrize("backend", ["guidance", "xgrammar", "outlines"])
912
def test_structured_output_batched_with_non_structured_outputs_requests(
811
- sample_json_schema: dict[str, Any],
913
backend: str,
914
):
915
+ sample_json_schema = SAMPLE_JSON_SCHEMA
916
# Don't use eager execution on TPUs because we want to test for no
917
# recompilation at runtime
918
enforce_eager = bool(not current_platform.is_tpu())
tests/entrypoints/openai/chat_completion/test_chat_completion.py
renamed
tests/entrypoints/openai/chat_completion/test_completion_with_image_embeds.py
renamed
tests/entrypoints/openai/completion/test_completion.py
renamed
tests/entrypoints/openai/test_multi_api_servers.py
renamed
tests/v1/entrypoints/__init__.py
tests/v1/entrypoints/conftest.py
deleted
-173
@@ -1,173 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest
5
-
6
-
7
-@pytest.fixture
8
-def sample_prompts():
9
- return [
10
- "Hello, my name is",
11
- "The president of the United States is",
12
- "The capital of France is",
13
- "The future of AI is",
14
- ]
15
-
16
-
17
-@pytest.fixture
18
-def sample_token_ids():
19
- return [
20
- [0],
21
- [0, 1],
22
- [0, 2, 1],
23
- [0, 3, 1, 2],
24
- ]
25
-
26
-
27
-@pytest.fixture
28
-def sample_regex():
29
- return (
30
- r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
31
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
32
- )
33
-
34
-
35
-# Note: Ensure this only uses attributes compatible with xgrammar
36
-@pytest.fixture
37
-def sample_json_schema():
38
- return {
39
- "type": "object",
40
- "properties": {
41
- "name": {"type": "string"},
42
- "age": {"type": "integer"},
43
- "skills": {
44
- "type": "array",
45
- "items": {
46
- "type": "string",
47
- },
48
- },
49
- "grade": {
50
- "type": "string",
51
- "pattern": "^[A-D]$", # Regex pattern
52
- },
53
- "email": {
54
- "type": "string",
55
- "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
56
- },
57
- "work_history": {
58
- "type": "array",
59
- "items": {
60
- "type": "object",
61
- "properties": {
62
- "company": {"type": "string"},
63
- "duration": {
64
- "type": "number",
65
- "minimum": 0.0,
66
- "maximum": 100.0, # Numeric range
67
- },
68
- "position": {"type": "string"},
69
- },
70
- "required": ["company", "duration", "position"],
71
- "additionalProperties": False,
72
- },
73
- "minItems": 0,
74
- "maxItems": 3,
75
- },
76
- },
77
- "required": ["name", "age", "skills", "grade", "email", "work_history"],
78
- "additionalProperties": False,
79
- "minProperties": 1,
80
- "maxProperties": 10,
81
- }
82
-
83
-
84
-# A schema unsupported by xgrammar
85
-@pytest.fixture
86
-def unsupported_json_schema():
87
- return {
88
- "type": "object",
89
- "properties": {
90
- "score": {
91
- "type": "integer",
92
- "multipleOf": 5, # Numeric multiple
93
- },
94
- "tags": {
95
- "type": "array",
96
- "items": {"type": "string", "minLength": 10, "maxLength": 20},
97
- },
98
- },
99
- "required": ["score", "tags"],
100
- "additionalProperties": False,
101
- "patternProperties": {
102
- "^score$": {"type": "integer"},
103
- },
104
- }
105
-
106
-
107
-@pytest.fixture
108
-def sample_definition_json_schema():
109
- return {
110
- "$defs": {
111
- "Step": {
112
- "properties": {
113
- "explanation": {"title": "Explanation", "type": "string"},
114
- "output": {"title": "Output", "type": "string"},
115
- },
116
- "required": ["explanation", "output"],
117
- "title": "Step",
118
- "type": "object",
119
- }
120
- },
121
- "properties": {
122
- "steps": {
123
- "items": {"$ref": "#/$defs/Step"},
124
- "title": "Steps",
125
- "type": "array",
126
- },
127
- "final_answer": {"title": "Final Answer", "type": "string"},
128
- },
129
- "required": ["steps", "final_answer"],
130
- "title": "MathReasoning",
131
- "type": "object",
132
- "additionalProperties": False,
133
- }
134
-
135
-
136
-@pytest.fixture
137
-def sample_structured_outputs_choices():
138
- return [
139
- "Python",
140
- "Java",
141
- "JavaScript",
142
- "C++",
143
- "C#",
144
- "PHP",
145
- "TypeScript",
146
- "Ruby",
147
- "Swift",
148
- "Kotlin",
149
- ]
150
-
151
-
152
-@pytest.fixture
153
-def sample_sql_ebnf():
154
- return """
155
-root ::= select_statement
156
-select_statement ::= "SELECT" column "from" table "where" condition
157
-column ::= "col_1" | "col_2"
158
-table ::= "table_1" | "table_2"
159
-condition ::= column "=" number
160
-number ::= "1" | "2"
161
-"""
162
-
163
-
164
-@pytest.fixture
165
-def sample_sql_lark():
166
- return """
167
-start: select_statement
168
-select_statement: "SELECT" column "from" table "where" condition
169
-column: "col_1" | "col_2"
170
-table: "table_1" | "table_2"
171
-condition: column "=" number
172
-number: "1" | "2"
173
-"""