| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import json |
| 4 | from typing import Final |
| 5 | |
| 6 | import pytest |
| 7 | import schemathesis |
| 8 | from hypothesis import HealthCheck, settings |
| 9 | from schemathesis import GenerationMode |
| 10 | from schemathesis.config import ( |
| 11 | ChecksConfig, |
| 12 | CoveragePhaseConfig, |
| 13 | GenerationConfig, |
| 14 | PhasesConfig, |
| 15 | PositiveDataAcceptanceConfig, |
| 16 | ProjectConfig, |
| 17 | ProjectsConfig, |
| 18 | SchemathesisConfig, |
| 19 | ) |
| 20 | |
| 21 | from vllm.platforms import current_platform |
| 22 | |
| 23 | from ...utils import RemoteOpenAIServer |
| 24 | |
| 25 | MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct" |
| 26 | MAXIMUM_IMAGES = 2 |
| 27 | _ROCM_TIMEOUT_MULTIPLIER = 3 if current_platform.is_rocm() else 1 |
| 28 | DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 * _ROCM_TIMEOUT_MULTIPLIER |
| 29 | LONG_TIMEOUT_SECONDS: Final[int] = 60 * _ROCM_TIMEOUT_MULTIPLIER |
| 30 | |
| 31 | |
| 32 | @pytest.fixture(scope="module") |
| 33 | def server(): |
| 34 | args = [ |
| 35 | "--runner", |
| 36 | "generate", |
| 37 | "--max-model-len", |
| 38 | "2048", |
| 39 | "--max-num-seqs", |
| 40 | "5", |
| 41 | "--enforce-eager", |
| 42 | "--trust-remote-code", |
| 43 | "--limit-mm-per-prompt", |
| 44 | json.dumps({"image": MAXIMUM_IMAGES}), |
| 45 | ] |
| 46 | |
| 47 | with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: |
| 48 | yield remote_server |
| 49 | |
| 50 | |
| 51 | @pytest.fixture(scope="module") |
| 52 | def get_schema(server): |
| 53 | # avoid generating null (\x00) bytes in strings during test case generation |
| 54 | return schemathesis.openapi.from_url( |
| 55 | f"{server.url_root}/openapi.json", |
| 56 | config=SchemathesisConfig( |
| 57 | projects=ProjectsConfig( |
| 58 | default=ProjectConfig( |
| 59 | generation=GenerationConfig( |
| 60 | allow_x00=False, |
| 61 | modes=[GenerationMode.POSITIVE], |
| 62 | ), |
| 63 | checks=ChecksConfig( |
| 64 | positive_data_acceptance=PositiveDataAcceptanceConfig( |
| 65 | enabled=False, |
| 66 | ), |
| 67 | ), |
| 68 | phases=PhasesConfig( |
| 69 | coverage=CoveragePhaseConfig(enabled=False), |
| 70 | ), |
| 71 | ), |
| 72 | ), |
| 73 | ), |
| 74 | ) |
| 75 | |
| 76 | |
| 77 | schema = schemathesis.pytest.from_fixture("get_schema") |
| 78 | |
| 79 | |
| 80 | @schemathesis.hook |
| 81 | def before_generate_case(context: schemathesis.HookContext, strategy): |
| 82 | op = context.operation |
| 83 | assert op is not None |
| 84 | |
| 85 | def no_invalid_types(case: schemathesis.Case): |
| 86 | """Skips tool_calls with `"type": "custom"` which schemathesis incorrectly |
| 87 | generates instead of the valid `"type": "function"`. |
| 88 | |
| 89 | Example test case that is skipped: |
| 90 | curl -X POST -H 'Content-Type: application/json' \ |
| 91 | -d '{"messages": [{"role": "assistant", "tool_calls": [{"custom": {"input": "", "name": ""}, "id": "", "type": "custom"}]}]}' \ |
| 92 | http://localhost:8000/v1/chat/completions |
| 93 | """ # noqa: E501 |
| 94 | if ( |
| 95 | hasattr(case, "body") |
| 96 | and isinstance(case.body, dict) |
| 97 | and "messages" in case.body |
| 98 | and isinstance(case.body["messages"], list) |
| 99 | and len(case.body["messages"]) > 0 |
| 100 | ): |
| 101 | for message in case.body["messages"]: |
| 102 | if not isinstance(message, dict): |
| 103 | continue |
| 104 | |
| 105 | tool_calls = message.get("tool_calls", []) |
| 106 | if isinstance(tool_calls, list): |
| 107 | for tool_call in tool_calls: |
| 108 | if isinstance(tool_call, dict): |
| 109 | if tool_call.get("type") != "function": |
| 110 | return False |
| 111 | if "custom" in tool_call: |
| 112 | return False |
| 113 | |
| 114 | return True |
| 115 | |
| 116 | return strategy.filter(no_invalid_types) |
| 117 | |
| 118 | |
| 119 | @schema.parametrize() |
| 120 | @settings( |
| 121 | deadline=LONG_TIMEOUT_SECONDS * 1000, |
| 122 | max_examples=50, |
| 123 | # Under CI's derandomized hypothesis seed, the schemathesis strategy |
| 124 | # for /v1/chat/completions/batch's nested-message body, combined with |
| 125 | # the no_invalid_types filter (notably the grammar=="" rule), exceeds |
| 126 | # the default filtered-vs-good ratio. The filter is intentional, so |
| 127 | # suppress the health check rather than drop the filter — dropping it |
| 128 | # exposes pre-existing server bugs out of scope here. |
| 129 | # The same nested schema can also trip Hypothesis' entropy budget while |
| 130 | # generating large-but-valid request bodies before vLLM is called. |
| 131 | suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.data_too_large], |
| 132 | ) |
| 133 | def test_openapi_stateless(case: schemathesis.Case): |
| 134 | key = ( |
| 135 | case.operation.method.upper(), |
| 136 | case.operation.path, |
| 137 | ) |
| 138 | if case.operation.path.startswith("/v1/responses"): |
| 139 | # Skip responses API as it is meant to be stateful. |
| 140 | return |
| 141 | |
| 142 | # Skip weight transfer endpoints as they require special setup |
| 143 | # (weight_transfer_config) and are meant to be stateful. |
| 144 | if case.operation.path in ( |
| 145 | "/init_weight_transfer_engine", |
| 146 | "/start_weight_update", |
| 147 | "/start_draft_weight_update", |
| 148 | "/update_weights", |
| 149 | "/finish_weight_update", |
| 150 | "/update_weight_version", |
| 151 | ): |
| 152 | return |
| 153 | |
| 154 | timeout = { |
| 155 | # requires a longer timeout |
| 156 | ("POST", "/v1/chat/completions"): LONG_TIMEOUT_SECONDS, |
| 157 | ("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS, |
| 158 | ("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS, |
| 159 | ("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS, |
| 160 | ("POST", "/inference/v1/generate"): LONG_TIMEOUT_SECONDS, |
| 161 | }.get(key, DEFAULT_TIMEOUT_SECONDS) |
| 162 | |
| 163 | # No need to verify SSL certificate for localhost |
| 164 | case.call_and_validate( |
| 165 | verify=False, |
| 166 | timeout=timeout, |
| 167 | headers={"Content-Type": "application/json"}, |
| 168 | ) |