A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Add property-based testing for vLLM endpoints using an API defined by an OpenAPI 3.1 schema (#16721)
Signed-off-by: Tarun Kumar <takumar@redhat.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Tarun Kumar committed
Apr 18, 2025 at 09:38 UTC
e37073efd7dea225ff8a0614f04575547d7588ff
4 files changed
+134
-6
.buildkite/test-pipeline.yaml
+1
-1
@@ -118,7 +118,7 @@ steps:
118
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
119
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
120
- VLLM_USE_V1=0 pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
121
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/
121
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_openai_schema.py
122
- pytest -v -s entrypoints/test_chat_utils.py
123
- VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
124
requirements/test.in
+1
@@ -37,6 +37,7 @@ lm-eval[api]==0.4.8 # required for model evaluation test
37
transformers==4.51.1
38
tokenizers==0.21.1
39
huggingface-hub[hf_xet]>=0.30.0 # Required for Xet downloads.
40
+schemathesis>=3.39.15 # Required for openai schema test.
41
# quantization
42
bitsandbytes>=0.45.3
43
buildkite-test-collector==0.1.9
requirements/test.txt
+83
-5
@@ -20,21 +20,29 @@ aiosignal==1.3.1
20
annotated-types==0.7.0
21
# via pydantic
22
anyio==4.6.2.post1
23
- # via httpx
23
+ # via
24
+ # httpx
25
+ # starlette
26
argcomplete==3.5.1
27
# via datamodel-code-generator
28
+arrow==1.3.0
29
+ # via isoduration
30
attrs==24.2.0
31
# via
32
# aiohttp
33
+ # hypothesis
34
# jsonlines
35
# jsonschema
36
+ # pytest-subtests
37
# referencing
38
audioread==3.0.1
39
# via librosa
40
awscli==1.35.23
41
# via -r requirements/test.in
42
backoff==2.2.1
37
- # via -r requirements/test.in
43
+ # via
44
+ # -r requirements/test.in
45
+ # schemathesis
46
bitsandbytes==0.45.3
47
# via -r requirements/test.in
48
black==24.10.0
@@ -69,11 +77,13 @@ click==8.1.7
77
# jiwer
78
# nltk
79
# ray
80
+ # schemathesis
81
# typer
82
colorama==0.4.6
83
# via
84
# awscli
85
# sacrebleu
86
+ # schemathesis
87
# tqdm-multiprocess
88
contourpy==1.3.0
89
# via matplotlib
@@ -138,6 +148,8 @@ filelock==3.16.1
148
# transformers
149
fonttools==4.54.1
150
# via matplotlib
151
+fqdn==1.5.1
152
+ # via jsonschema
153
frozendict==2.4.6
154
# via einx
155
frozenlist==1.5.0
@@ -156,8 +168,12 @@ genai-perf==0.0.8
168
# via -r requirements/test.in
169
genson==1.3.0
170
# via datamodel-code-generator
171
+graphql-core==3.2.6
172
+ # via hypothesis-graphql
173
h11==0.14.0
174
# via httpcore
175
+harfile==0.3.0
176
+ # via schemathesis
177
hf-xet==0.1.4
178
# via huggingface-hub
179
hiredis==3.0.0
@@ -165,7 +181,9 @@ hiredis==3.0.0
181
httpcore==1.0.6
182
# via httpx
183
httpx==0.27.2
168
- # via -r requirements/test.in
184
+ # via
185
+ # -r requirements/test.in
186
+ # schemathesis
187
huggingface-hub==0.30.1
188
# via
189
# -r requirements/test.in
@@ -180,17 +198,29 @@ huggingface-hub==0.30.1
198
# vocos
199
humanize==4.11.0
200
# via runai-model-streamer
201
+hypothesis==6.131.0
202
+ # via
203
+ # hypothesis-graphql
204
+ # hypothesis-jsonschema
205
+ # schemathesis
206
+hypothesis-graphql==0.11.1
207
+ # via schemathesis
208
+hypothesis-jsonschema==0.23.1
209
+ # via schemathesis
210
idna==3.10
211
# via
212
# anyio
213
# email-validator
214
# httpx
215
+ # jsonschema
216
# requests
217
# yarl
218
inflect==5.6.2
219
# via datamodel-code-generator
220
iniconfig==2.0.0
221
# via pytest
222
+isoduration==20.11.0
223
+ # via jsonschema
224
isort==5.13.2
225
# via datamodel-code-generator
226
jinja2==3.1.6
@@ -210,12 +240,18 @@ joblib==1.4.2
240
# scikit-learn
241
jsonlines==4.0.0
242
# via lm-eval
243
+jsonpointer==3.0.0
244
+ # via jsonschema
245
jsonschema==4.23.0
246
# via
247
+ # hypothesis-jsonschema
248
# mistral-common
249
# ray
250
+ # schemathesis
251
jsonschema-specifications==2024.10.1
252
# via jsonschema
253
+junit-xml==1.9
254
+ # via schemathesis
255
kaleido==0.2.1
256
# via genai-perf
257
kiwisolver==1.4.7
@@ -239,7 +275,9 @@ mamba-ssm==2.2.4
275
markdown-it-py==3.0.0
276
# via rich
277
markupsafe==3.0.2
242
- # via jinja2
278
+ # via
279
+ # jinja2
280
+ # werkzeug
281
matplotlib==3.9.2
282
# via -r requirements/test.in
283
mbstrdecoder==1.1.3
@@ -449,6 +487,8 @@ pygments==2.18.0
487
# via rich
488
pyparsing==3.2.0
489
# via matplotlib
490
+pyrate-limiter==3.7.0
491
+ # via schemathesis
492
pytablewriter==1.2.0
493
# via lm-eval
494
pytest==8.3.3
@@ -461,7 +501,9 @@ pytest==8.3.3
501
# pytest-mock
502
# pytest-rerunfailures
503
# pytest-shard
504
+ # pytest-subtests
505
# pytest-timeout
506
+ # schemathesis
507
pytest-asyncio==0.24.0
508
# via -r requirements/test.in
509
pytest-forked==1.6.0
@@ -472,10 +514,13 @@ pytest-rerunfailures==14.0
514
# via -r requirements/test.in
515
pytest-shard==0.1.2
516
# via -r requirements/test.in
517
+pytest-subtests==0.14.1
518
+ # via schemathesis
519
pytest-timeout==2.3.1
520
# via -r requirements/test.in
521
python-dateutil==2.9.0.post0
522
# via
523
+ # arrow
524
# botocore
525
# matplotlib
526
# pandas
@@ -497,6 +542,7 @@ pyyaml==6.0.2
542
# peft
543
# ray
544
# responses
545
+ # schemathesis
546
# timm
547
# transformers
548
# vocos
@@ -527,10 +573,16 @@ requests==2.32.3
573
# pooch
574
# ray
575
# responses
576
+ # schemathesis
577
+ # starlette-testclient
578
# tiktoken
579
# transformers
580
responses==0.25.3
581
# via genai-perf
582
+rfc3339-validator==0.1.4
583
+ # via jsonschema
584
+rfc3987==1.3.8
585
+ # via jsonschema
586
rich==13.9.4
587
# via
588
# genai-perf
@@ -559,6 +611,8 @@ safetensors==0.4.5
611
# peft
612
# timm
613
# transformers
614
+schemathesis==3.39.15
615
+ # via -r requirements/test.in
616
scikit-learn==1.5.2
617
# via
618
# librosa
@@ -584,12 +638,16 @@ shellingham==1.5.4
638
# via typer
639
six==1.16.0
640
# via
641
+ # junit-xml
642
# python-dateutil
643
+ # rfc3339-validator
644
# rouge-score
645
sniffio==1.3.1
646
# via
647
# anyio
648
# httpx
649
+sortedcontainers==2.4.0
650
+ # via hypothesis
651
soundfile==0.12.1
652
# via
653
# -r requirements/test.in
@@ -598,6 +656,12 @@ soxr==0.5.0.post1
656
# via librosa
657
sqlitedict==2.1.0
658
# via lm-eval
659
+starlette==0.46.2
660
+ # via
661
+ # schemathesis
662
+ # starlette-testclient
663
+starlette-testclient==0.4.1
664
+ # via schemathesis
665
statsmodels==0.14.4
666
# via genai-perf
667
sympy==1.13.1
@@ -628,6 +692,10 @@ tokenizers==0.21.1
692
# via
693
# -r requirements/test.in
694
# transformers
695
+tomli==2.2.1
696
+ # via schemathesis
697
+tomli-w==1.2.0
698
+ # via schemathesis
699
torch==2.6.0
700
# via
701
# -r requirements/test.in
@@ -693,6 +761,8 @@ typepy==1.3.2
761
# tabledata
762
typer==0.15.2
763
# via fastsafetensors
764
+types-python-dateutil==2.9.0.20241206
765
+ # via arrow
766
typing-extensions==4.12.2
767
# via
768
# huggingface-hub
@@ -705,6 +775,8 @@ typing-extensions==4.12.2
775
# typer
776
tzdata==2024.2
777
# via pandas
778
+uri-template==1.3.0
779
+ # via jsonschema
780
urllib3==2.2.3
781
# via
782
# blobfile
@@ -716,6 +788,10 @@ vector-quantize-pytorch==1.21.2
788
# via -r requirements/test.in
789
vocos==0.1.0
790
# via -r requirements/test.in
791
+webcolors==24.11.1
792
+ # via jsonschema
793
+werkzeug==3.1.3
794
+ # via schemathesis
795
word2number==1.1
796
# via lm-eval
797
xxhash==3.5.0
@@ -723,6 +799,8 @@ xxhash==3.5.0
799
# datasets
800
# evaluate
801
yarl==1.17.1
726
- # via aiohttp
802
+ # via
803
+ # aiohttp
804
+ # schemathesis
805
zstandard==0.23.0
806
# via lm-eval
tests/entrypoints/openai/test_openai_schema.py
new
+49
@@ -0,0 +1,49 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import pytest
3
+import schemathesis
4
+from schemathesis import GenerationConfig
5
+
6
+from ...utils import RemoteOpenAIServer
7
+
8
+schemathesis.experimental.OPEN_API_3_1.enable()
9
+
10
+MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
11
+MAXIMUM_IMAGES = 2
12
+
13
+
14
+@pytest.fixture(scope="module")
15
+def server():
16
+ args = [
17
+ "--task",
18
+ "generate",
19
+ "--max-model-len",
20
+ "2048",
21
+ "--max-num-seqs",
22
+ "5",
23
+ "--enforce-eager",
24
+ "--trust-remote-code",
25
+ "--limit-mm-per-prompt",
26
+ f"image={MAXIMUM_IMAGES}",
27
+ ]
28
+
29
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
30
+ yield remote_server
31
+
32
+
33
+@pytest.fixture(scope="module")
34
+def get_schema(server):
35
+ # avoid generating null (\x00) bytes in strings during test case generation
36
+ return schemathesis.openapi.from_uri(
37
+ f"{server.url_root}/openapi.json",
38
+ generation_config=GenerationConfig(allow_x00=False),
39
+ )
40
+
41
+
42
+schema = schemathesis.from_pytest_fixture("get_schema")
43
+
44
+
45
+@schema.parametrize()
46
+@schema.override(headers={"Content-Type": "application/json"})
47
+async def test_openapi_stateless(case):
48
+ #No need to verify SSL certificate for localhost
49
+ await case.call_and_validate(verify=False)