A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Bugfix] make `test_openai_schema.py` pass (#18224)
Signed-off-by: David Xia <david@davidxia.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
David Xia committed
May 22, 2025 at 14:34 UTC
1f3a1200e425faaa93b333111ee1d2480cc5d397
2 files changed
+57
-2
.buildkite/test-pipeline.yaml
+1
-1
@@ -126,7 +126,7 @@ steps:
126
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
127
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
128
- VLLM_USE_V1=0 pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
129
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_openai_schema.py
129
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/
130
- pytest -v -s entrypoints/test_chat_utils.py
131
- VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
132
tests/entrypoints/openai/test_openai_schema.py
+56
-1
@@ -1,6 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+from typing import Final
3
+
4
import pytest
5
import schemathesis
6
+from hypothesis import settings
7
from schemathesis import GenerationConfig
8
9
from ...utils import RemoteOpenAIServer
@@ -9,6 +12,8 @@ schemathesis.experimental.OPEN_API_3_1.enable()
12
13
MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
14
MAXIMUM_IMAGES = 2
15
+DEFAULT_TIMEOUT_SECONDS: Final[int] = 10
16
+LONG_TIMEOUT_SECONDS: Final[int] = 60
17
18
19
@pytest.fixture(scope="module")
@@ -42,8 +47,58 @@ def get_schema(server):
47
schema = schemathesis.from_pytest_fixture("get_schema")
48
49
50
+@schemathesis.hook
51
+def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
52
+ op = context.operation
53
+ assert op is not None
54
+
55
+ def no_file_type(case: schemathesis.models.Case):
56
+ """
57
+ This filter skips test cases for the `POST /tokenize` endpoint where the
58
+ HTTP request body uses `"type": "file"` in any message's content.
59
+ We expect these cases to fail because that type isn't implemented here
60
+ https://github.com/vllm-project/vllm/blob/0b34593017953051b3225b1483ce0f4670e3eb0e/vllm/entrypoints/chat_utils.py#L1038-L1095
61
+
62
+ Example test cases that are skipped:
63
+ curl -X POST -H 'Content-Type: application/json' \
64
+ -d '{"messages": [{"role": "assistant"}, {"content": [{"file": {}, "type": "file"}], "role": "user"}]}' \
65
+ http://localhost:8000/tokenize
66
+
67
+ curl -X POST -H 'Content-Type: application/json' \
68
+ -d '{"messages": [{"content": [{"file": {}, "type": "file"}], "role": "user"}]}' \
69
+ http://localhost:8000/tokenize
70
+ """ # noqa: E501
71
+ if (op.method.lower() == "post" and op.path == "/tokenize"
72
+ and hasattr(case, "body") and isinstance(case.body, dict)
73
+ and "messages" in case.body
74
+ and isinstance(case.body["messages"], list)
75
+ and len(case.body["messages"]) > 0):
76
+ for message in case.body["messages"]:
77
+ if not isinstance(message, dict):
78
+ continue
79
+ content = message.get("content", [])
80
+ if not isinstance(content, list) or len(content) == 0:
81
+ continue
82
+ if any(item.get("type") == "file" for item in content):
83
+ return False
84
+ return True
85
+
86
+ return strategy.filter(no_file_type)
87
+
88
+
89
@schema.parametrize()
90
@schema.override(headers={"Content-Type": "application/json"})
91
+@settings(deadline=LONG_TIMEOUT_SECONDS * 1000)
92
def test_openapi_stateless(case: schemathesis.Case):
93
+ key = (
94
+ case.operation.method.upper(),
95
+ case.operation.path,
96
+ )
97
+ timeout = {
98
+ # requires a longer timeout
99
+ ("POST", "/v1/chat/completions"):
100
+ LONG_TIMEOUT_SECONDS,
101
+ }.get(key, DEFAULT_TIMEOUT_SECONDS)
102
+
103
#No need to verify SSL certificate for localhost
49
- case.call_and_validate(verify=False)
104
+ case.call_and_validate(verify=False, timeout=timeout)