[Test] Migrate test_openai_schema.py to schemathesis 4.x (#46173)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
Jun 19, 2026 at 14:05 UTC
4a8abf37c75b4a2587bfdad48bc6b442dc71332a
4 files changed
+61
-48
requirements/test/cuda.in
+1
-1
@@ -40,7 +40,7 @@ lm-eval[api]>=0.4.12 # required for model evaluation test
40
mteb[bm25s]>=2, <3 # required for mteb test
41
transformers==5.5.3
42
tokenizers==0.22.2
43
-schemathesis>=3.39.15 # Required for openai schema test.
43
+schemathesis>=4.0.0 # Required for openai schema test.
44
# quantization
45
bitsandbytes==0.49.2
46
buildkite-test-collector==0.1.9
requirements/test/nightly-torch.txt
+1
-1
@@ -31,7 +31,7 @@ lm-eval[api]>=0.4.12 # required for model evaluation test
31
mteb[bm25s]>=2, <3 # required for mteb test
32
transformers==5.5.3
33
tokenizers==0.22.2
34
-schemathesis>=3.39.15 # Required for openai schema test.
34
+schemathesis>=4.0.0 # Required for openai schema test.
35
# quantization
36
bitsandbytes>=0.49.2
37
buildkite-test-collector==0.1.9
requirements/test/rocm.in
+1
-1
@@ -39,7 +39,7 @@ lm-eval[api]>=0.4.12 # required for model evaluation test
39
mteb[bm25s]>=2, <3 # required for mteb test
40
transformers==5.5.3
41
tokenizers==0.22.2
42
-schemathesis>=3.39.15 # Required for openai schema test
42
+schemathesis>=4.0.0 # Required for openai schema test
43
# quantization
44
bitsandbytes==0.49.2
45
buildkite-test-collector==0.1.9
tests/entrypoints/openai/test_openai_schema.py
+58
-45
@@ -6,15 +6,22 @@ from typing import Final
6
import pytest
7
import schemathesis
8
from hypothesis import HealthCheck, settings
9
-from schemathesis import GenerationConfig
10
-from schemathesis.models import Case
9
+from schemathesis import GenerationMode
10
+from schemathesis.config import (
11
+ ChecksConfig,
12
+ CoveragePhaseConfig,
13
+ GenerationConfig,
14
+ PhasesConfig,
15
+ PositiveDataAcceptanceConfig,
16
+ ProjectConfig,
17
+ ProjectsConfig,
18
+ SchemathesisConfig,
19
+)
20
21
from vllm.platforms import current_platform
22
23
from ...utils import RemoteOpenAIServer
24
16
-schemathesis.experimental.OPEN_API_3_1.enable()
17
-
25
MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
26
MAXIMUM_IMAGES = 2
27
_ROCM_TIMEOUT_MULTIPLIER = 3 if current_platform.is_rocm() else 1
@@ -44,21 +51,38 @@ def server():
51
@pytest.fixture(scope="module")
52
def get_schema(server):
53
# avoid generating null (\x00) bytes in strings during test case generation
47
- return schemathesis.openapi.from_uri(
54
+ return schemathesis.openapi.from_url(
55
f"{server.url_root}/openapi.json",
49
- generation_config=GenerationConfig(allow_x00=False),
56
+ config=SchemathesisConfig(
57
+ projects=ProjectsConfig(
58
+ default=ProjectConfig(
59
+ generation=GenerationConfig(
60
+ allow_x00=False,
61
+ modes=[GenerationMode.POSITIVE],
62
+ ),
63
+ checks=ChecksConfig(
64
+ positive_data_acceptance=PositiveDataAcceptanceConfig(
65
+ enabled=False,
66
+ ),
67
+ ),
68
+ phases=PhasesConfig(
69
+ coverage=CoveragePhaseConfig(enabled=False),
70
+ ),
71
+ ),
72
+ ),
73
+ ),
74
)
75
76
53
-schema = schemathesis.from_pytest_fixture("get_schema")
77
+schema = schemathesis.pytest.from_fixture("get_schema")
78
79
80
@schemathesis.hook
57
-def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
81
+def before_generate_case(context: schemathesis.HookContext, strategy):
82
op = context.operation
83
assert op is not None
84
61
- def no_invalid_types(case: schemathesis.models.Case):
85
+ def no_invalid_types(case: schemathesis.Case):
86
"""
87
Skips tool_calls with `"type": "custom"` which schemathesis incorrectly
88
generates instead of the valid `"type": "function"`.
@@ -68,39 +92,25 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
92
-d '{"messages": [{"role": "assistant", "tool_calls": [{"custom": {"input": "", "name": ""}, "id": "", "type": "custom"}]}]}' \
93
http://localhost:8000/v1/chat/completions
94
""" # noqa: E501
71
- if hasattr(case, "body") and isinstance(case.body, dict):
72
- if (
73
- "messages" in case.body
74
- and isinstance(case.body["messages"], list)
75
- and len(case.body["messages"]) > 0
76
- ):
77
- for message in case.body["messages"]:
78
- if not isinstance(message, dict):
79
- continue
80
-
81
- tool_calls = message.get("tool_calls", [])
82
- if isinstance(tool_calls, list):
83
- for tool_call in tool_calls:
84
- if isinstance(tool_call, dict):
85
- if tool_call.get("type") != "function":
86
- return False
87
- if "custom" in tool_call:
88
- return False
89
-
90
- # Sometimes structured_outputs.grammar is generated to be empty
91
- # Causing a server error in EBNF grammar parsing
92
- # https://github.com/vllm-project/vllm/pull/22587#issuecomment-3195253421
93
- structured_outputs = case.body.get("structured_outputs", {})
94
- grammar = (
95
- structured_outputs.get("grammar")
96
- if isinstance(structured_outputs, dict)
97
- else None
98
- )
99
-
100
- if grammar == "":
101
- # Allow None (will be handled as no grammar)
102
- # But skip empty strings
103
- return False
95
+ if (
96
+ hasattr(case, "body")
97
+ and isinstance(case.body, dict)
98
+ and "messages" in case.body
99
+ and isinstance(case.body["messages"], list)
100
+ and len(case.body["messages"]) > 0
101
+ ):
102
+ for message in case.body["messages"]:
103
+ if not isinstance(message, dict):
104
+ continue
105
+
106
+ tool_calls = message.get("tool_calls", [])
107
+ if isinstance(tool_calls, list):
108
+ for tool_call in tool_calls:
109
+ if isinstance(tool_call, dict):
110
+ if tool_call.get("type") != "function":
111
+ return False
112
+ if "custom" in tool_call:
113
+ return False
114
115
return True
116
@@ -108,7 +118,6 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
118
119
120
@schema.parametrize()
111
-@schema.override(headers={"Content-Type": "application/json"})
121
@settings(
122
deadline=LONG_TIMEOUT_SECONDS * 1000,
123
max_examples=50,
@@ -122,7 +131,7 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
131
# generating large-but-valid request bodies before vLLM is called.
132
suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.data_too_large],
133
)
125
-def test_openapi_stateless(case: Case):
134
+def test_openapi_stateless(case: schemathesis.Case):
135
key = (
136
case.operation.method.upper(),
137
case.operation.path,
@@ -151,4 +160,8 @@ def test_openapi_stateless(case: Case):
160
}.get(key, DEFAULT_TIMEOUT_SECONDS)
161
162
# No need to verify SSL certificate for localhost
154
- case.call_and_validate(verify=False, timeout=timeout)
163
+ case.call_and_validate(
164
+ verify=False,
165
+ timeout=timeout,
166
+ headers={"Content-Type": "application/json"},
167
+ )