A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 07:15 UTC
[CI] Stabilize OpenAI schema fuzzing for malformed structural tags (#44131)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jun 1, 2026 at 21:56 UTC
54d0c36fff6a33469e10e8276ff9ea05c156331b
6 files changed
+201
-3
tests/entrypoints/openai/chat_completion/test_chat_error.py
+47
-1
@@ -6,9 +6,13 @@ from typing import Any
6
from unittest.mock import AsyncMock, MagicMock, patch
7
8
import pytest
9
+from pydantic import ValidationError
10
11
from vllm.config.multimodal import MultiModalConfig
11
-from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
+ BatchChatCompletionRequest,
14
+ ChatCompletionRequest,
15
+)
16
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
17
from vllm.entrypoints.openai.engine.protocol import GenerationError
18
from vllm.entrypoints.openai.models.protocol import BaseModelPath
@@ -444,3 +448,45 @@ def test_json_schema_response_format_missing_schema():
448
messages=[{"role": "user", "content": "hello"}],
449
response_format={"type": "json_schema"},
450
)
451
+
452
+
453
+@pytest.mark.parametrize("format_value", [None, {}])
454
+def test_structural_tag_response_format_invalid(format_value):
455
+ """Malformed structural tags should be rejected during request validation."""
456
+ with pytest.raises(
457
+ ValidationError,
458
+ match="Invalid response_format structural_tag",
459
+ ):
460
+ ChatCompletionRequest(
461
+ model=MODEL_NAME,
462
+ messages=[{"role": "user", "content": "hello"}],
463
+ response_format={"type": "structural_tag", "format": format_value},
464
+ )
465
+
466
+
467
+@pytest.mark.parametrize("format_value", [None, {}])
468
+def test_batch_structural_tag_response_format_invalid(format_value):
469
+ """Batch chat should reject malformed structural tags at request parsing."""
470
+ with pytest.raises(
471
+ ValidationError,
472
+ match="Invalid response_format structural_tag",
473
+ ):
474
+ BatchChatCompletionRequest(
475
+ model=MODEL_NAME,
476
+ messages=[[{"role": "user", "content": "hello"}]],
477
+ response_format={"type": "structural_tag", "format": format_value},
478
+ )
479
+
480
+
481
+@pytest.mark.parametrize("structural_tag", ["not json", ""])
482
+def test_structured_outputs_structural_tag_invalid(structural_tag):
483
+ """Malformed direct structured_outputs structural tags should be rejected."""
484
+ with pytest.raises(
485
+ ValidationError,
486
+ match="Invalid structured_outputs structural_tag",
487
+ ):
488
+ ChatCompletionRequest(
489
+ model=MODEL_NAME,
490
+ messages=[{"role": "user", "content": "hello"}],
491
+ structured_outputs={"structural_tag": structural_tag},
492
+ )
tests/entrypoints/openai/completion/test_completion_error.py
+31
@@ -6,6 +6,7 @@ from typing import Any
6
from unittest.mock import AsyncMock, MagicMock
7
8
import pytest
9
+from pydantic import ValidationError
10
11
from vllm.config.multimodal import MultiModalConfig
12
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
@@ -302,6 +303,36 @@ def test_json_schema_response_format_missing_schema():
303
)
304
305
306
+@pytest.mark.parametrize("format_value", [None, {}])
307
+def test_structural_tag_response_format_invalid(format_value):
308
+ """Malformed structural tags should be rejected during request validation."""
309
+ with pytest.raises(
310
+ ValidationError,
311
+ match="Invalid response_format structural_tag",
312
+ ):
313
+ CompletionRequest(
314
+ model=MODEL_NAME,
315
+ prompt="Test prompt",
316
+ max_tokens=10,
317
+ response_format={"type": "structural_tag", "format": format_value},
318
+ )
319
+
320
+
321
+@pytest.mark.parametrize("structural_tag", ["not json", ""])
322
+def test_structured_outputs_structural_tag_invalid(structural_tag):
323
+ """Malformed direct structured_outputs structural tags should be rejected."""
324
+ with pytest.raises(
325
+ ValidationError,
326
+ match="Invalid structured_outputs structural_tag",
327
+ ):
328
+ CompletionRequest(
329
+ model=MODEL_NAME,
330
+ prompt="Test prompt",
331
+ max_tokens=10,
332
+ structured_outputs={"structural_tag": structural_tag},
333
+ )
334
+
335
+
336
def test_negative_prompt_token_ids_nested():
337
"""Negative token IDs in prompt (nested list) should raise validation error."""
338
with pytest.raises(Exception, match="greater than or equal to 0"):
tests/tool_parsers/test_mistral_tool_parser.py
+26
-2
@@ -1382,7 +1382,20 @@ def test_adjust_request_non_mistral_tokenizer(
1382
[
1383
{"regex": r"\d+"},
1384
{"choice": ["a", "b"]},
1385
- {"structural_tag": '{"key": "value"}'},
1385
+ {
1386
+ "structural_tag": json.dumps(
1387
+ {
1388
+ "structures": [
1389
+ {
1390
+ "begin": "<tool>",
1391
+ "schema": {"type": "object"},
1392
+ "end": "</tool>",
1393
+ }
1394
+ ],
1395
+ "triggers": ["<tool>"],
1396
+ }
1397
+ )
1398
+ },
1399
{"grammar": "start: 'hello'"},
1400
],
1401
ids=["regex", "choice", "structural_tag", "grammar"],
@@ -1404,7 +1417,18 @@ def test_adjust_request_unsupported_response_format(
1417
) -> None:
1418
request = _make_request(
1419
response_format=StructuralTagResponseFormat(
1407
- type="structural_tag", format={"some": "config"}
1420
+ type="structural_tag",
1421
+ format={
1422
+ "type": "triggered_tags",
1423
+ "tags": [
1424
+ {
1425
+ "begin": "<tool>",
1426
+ "content": {"type": "any_text"},
1427
+ "end": "</tool>",
1428
+ }
1429
+ ],
1430
+ "triggers": ["<tool>"],
1431
+ },
1432
),
1433
)
1434
result = mistral_tool_parser.adjust_request(request)
vllm/entrypoints/openai/chat_completion/protocol.py
+16
@@ -30,6 +30,8 @@ from vllm.entrypoints.openai.engine.protocol import (
30
StructuralTagResponseFormat,
31
ToolCall,
32
UsageInfo,
33
+ validate_structural_tag_response_format,
34
+ validate_structured_outputs_structural_tag,
35
)
36
from vllm.exceptions import VLLMValidationError
37
from vllm.logger import init_logger
@@ -671,6 +673,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
673
parameter="response_format",
674
)
675
676
+ if rf_type == "structural_tag":
677
+ validate_structural_tag_response_format(response_format)
678
+
679
return data
680
681
@model_validator(mode="before")
@@ -754,6 +759,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
759
"You can only either use constraints for structured outputs "
760
"or tools, not both.",
761
)
762
+ validate_structured_outputs_structural_tag(structured_outputs_kwargs)
763
return data
764
765
@model_validator(mode="before")
@@ -979,6 +985,16 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
985
"Batch chat completions do not support beam search. "
986
"Please set `use_beam_search` to False."
987
)
988
+ response_format = data.get("response_format")
989
+ rf_type = (
990
+ response_format.get("type")
991
+ if isinstance(response_format, dict)
992
+ else getattr(response_format, "type", None)
993
+ )
994
+ if rf_type == "structural_tag":
995
+ validate_structural_tag_response_format(response_format)
996
+ if (structured_outputs := data.get("structured_outputs")) is not None:
997
+ validate_structured_outputs_structural_tag(structured_outputs)
998
n = data.get("n", 1)
999
if n is not None and n != 1:
1000
raise ValueError(
vllm/entrypoints/openai/completion/protocol.py
+6
@@ -18,6 +18,8 @@ from vllm.entrypoints.openai.engine.protocol import (
18
StreamOptions,
19
StructuralTagResponseFormat,
20
UsageInfo,
21
+ validate_structural_tag_response_format,
22
+ validate_structured_outputs_structural_tag,
23
)
24
from vllm.exceptions import VLLMValidationError
25
from vllm.logger import init_logger
@@ -370,6 +372,9 @@ class CompletionRequest(OpenAIBaseModel):
372
parameter="response_format",
373
)
374
375
+ if rf_type == "structural_tag":
376
+ validate_structural_tag_response_format(response_format)
377
+
378
return data
379
380
@model_validator(mode="before")
@@ -397,6 +402,7 @@ class CompletionRequest(OpenAIBaseModel):
402
"outputs ('json', 'regex' or 'choice').",
403
parameter="structured_outputs",
404
)
405
+ validate_structured_outputs_structural_tag(structured_outputs_kwargs)
406
return data
407
408
@model_validator(mode="before")
vllm/entrypoints/openai/engine/protocol.py
+75
@@ -17,6 +17,7 @@ from pydantic import (
17
)
18
19
from vllm.entrypoints.chat_utils import make_tool_call_id
20
+from vllm.exceptions import VLLMValidationError
21
from vllm.logger import init_logger
22
from vllm.utils import random_uuid
23
from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -158,6 +159,80 @@ AnyResponseFormat: TypeAlias = (
159
)
160
161
162
+def validate_structural_tag_response_format(
163
+ response_format: AnyStructuralTagResponseFormat | dict[str, Any],
164
+) -> None:
165
+ """Validate structural tags before they are sent to the engine.
166
+
167
+ Engine-side validation reports malformed structural tags as generation
168
+ failures. OpenAI request parsing should classify them as bad requests.
169
+ """
170
+ import json
171
+
172
+ from pydantic import TypeAdapter, ValidationError
173
+
174
+ if isinstance(response_format, dict):
175
+ try:
176
+ response_format = TypeAdapter(
177
+ AnyStructuralTagResponseFormat
178
+ ).validate_python(response_format)
179
+ except ValidationError as exc:
180
+ raise VLLMValidationError(
181
+ "Invalid response_format structural_tag specification.",
182
+ parameter="response_format",
183
+ ) from exc
184
+
185
+ try:
186
+ payload = json.dumps(response_format.model_dump(by_alias=True))
187
+ validate_structural_tag_payload(payload, parameter="response_format")
188
+ except (TypeError, ValueError) as exc:
189
+ raise VLLMValidationError(
190
+ "Invalid response_format structural_tag specification.",
191
+ parameter="response_format",
192
+ ) from exc
193
+
194
+
195
+def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
196
+ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
197
+ from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
198
+
199
+ if isinstance(payload, str) and not payload:
200
+ raise VLLMValidationError(
201
+ f"Invalid {parameter} structural_tag specification.",
202
+ parameter=parameter,
203
+ )
204
+
205
+ try:
206
+ validate_xgrammar_grammar(
207
+ SamplingParams(
208
+ structured_outputs=StructuredOutputsParams(structural_tag=payload)
209
+ )
210
+ )
211
+ except (TypeError, ValueError) as exc:
212
+ raise VLLMValidationError(
213
+ f"Invalid {parameter} structural_tag specification.",
214
+ parameter=parameter,
215
+ ) from exc
216
+
217
+
218
+def validate_structured_outputs_structural_tag(
219
+ structured_outputs: Any,
220
+) -> None:
221
+ from vllm.sampling_params import StructuredOutputsParams
222
+
223
+ if isinstance(structured_outputs, StructuredOutputsParams):
224
+ structural_tag = structured_outputs.structural_tag
225
+ elif isinstance(structured_outputs, dict):
226
+ structural_tag = structured_outputs.get("structural_tag")
227
+ else:
228
+ return
229
+ if structural_tag is not None:
230
+ validate_structural_tag_payload(
231
+ structural_tag,
232
+ parameter="structured_outputs",
233
+ )
234
+
235
+
236
class StreamOptions(OpenAIBaseModel):
237
include_usage: bool | None = False
238
continuous_usage_stats: bool | None = False