| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Replay tests for DelegatingParser with engine adapters. |
| 4 | |
| 5 | Exercises DelegatingParser in engine-adapter mode to verify that delegated |
| 6 | routing produces correct output across chunk sizes. |
| 7 | See test_replay.py for tests that target engine parsers directly. |
| 8 | |
| 9 | Parser discovery is automatic: any engine parser in ``registered_adapters`` |
| 10 | that has both tool and reasoning adapters and a builder in |
| 11 | ``trace_builder._BUILDERS`` is picked up with zero manual wiring. |
| 12 | """ |
| 13 | |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | from typing import NamedTuple |
| 17 | |
| 18 | import pytest |
| 19 | from pydantic import TypeAdapter |
| 20 | |
| 21 | from tests.parser.engine.replay_harness import ( |
| 22 | CHUNK_SIZES, |
| 23 | DUMMY_TOOLS, |
| 24 | MockTokenizer, |
| 25 | _test_request, |
| 26 | assert_no_terminal_leakage, |
| 27 | assert_parse_output, |
| 28 | collect_output, |
| 29 | make_mock_tokenizer, |
| 30 | parse_non_streaming, |
| 31 | replay_streaming, |
| 32 | ) |
| 33 | from tests.parser.engine.trace_builder import _BUILDERS, build_samples |
| 34 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 35 | ChatCompletionToolsParam, |
| 36 | ) |
| 37 | from vllm.parser.abstract_parser import DelegatingParser, Parser |
| 38 | from vllm.parser.engine import registered_adapters as _adapters_mod |
| 39 | from vllm.parser.engine.adapters import ( |
| 40 | ParserEngineReasoningAdapter, |
| 41 | ParserEngineToolAdapter, |
| 42 | ) |
| 43 | from vllm.parser.mistral import MistralParser |
| 44 | |
| 45 | _TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam]) |
| 46 | |
| 47 | # ── Pairing discovery ──────────────────────────────────────────────── |
| 48 | |
| 49 | |
| 50 | class _PairingInfo(NamedTuple): |
| 51 | parser_cls: type[Parser] |
| 52 | name: str |
| 53 | samples: tuple |
| 54 | |
| 55 | |
| 56 | def _discover_pairings() -> list[_PairingInfo]: |
| 57 | """Discover valid delegating pairings from registered engine adapters. |
| 58 | |
| 59 | Groups tool and reasoning adapters by their engine class, then builds |
| 60 | a DelegatingParser subclass for each engine that has both adapters |
| 61 | and a test builder. |
| 62 | """ |
| 63 | bare_tok = MockTokenizer(vocab={}, tokens=[]) |
| 64 | engines: dict[type, dict[str, type]] = {} |
| 65 | for obj in vars(_adapters_mod).values(): |
| 66 | if not isinstance(obj, type): |
| 67 | continue |
| 68 | if ( |
| 69 | issubclass(obj, ParserEngineToolAdapter) |
| 70 | and obj is not ParserEngineToolAdapter |
| 71 | ): |
| 72 | tool_adapter: type[ParserEngineToolAdapter] = obj |
| 73 | engines.setdefault(tool_adapter._parser_engine_cls, {})["tool"] = obj |
| 74 | elif ( |
| 75 | issubclass(obj, ParserEngineReasoningAdapter) |
| 76 | and obj is not ParserEngineReasoningAdapter |
| 77 | ): |
| 78 | reasoning_adapter: type[ParserEngineReasoningAdapter] = obj |
| 79 | engines.setdefault(reasoning_adapter._parser_engine_cls, {})[ |
| 80 | "reasoning" |
| 81 | ] = obj |
| 82 | |
| 83 | found: list[_PairingInfo] = [] |
| 84 | missing_builders: list[str] = [] |
| 85 | for engine_cls, adapters in engines.items(): |
| 86 | if "tool" not in adapters or "reasoning" not in adapters: |
| 87 | continue |
| 88 | if engine_cls is MistralParser: |
| 89 | # Mistral uses brace-balanced JSON tool args with no TOOL_END |
| 90 | # token, so it does not fit this TOOL_END-based replay harness. |
| 91 | # It is covered by tests/parser/mistral/ instead. |
| 92 | continue |
| 93 | cfg = engine_cls(bare_tok, None).parser_engine_config |
| 94 | if cfg.name not in _BUILDERS: |
| 95 | missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})") |
| 96 | continue |
| 97 | if cfg.name == "inkling": |
| 98 | # Inkling uses typed structural blocks and opts out of token-id |
| 99 | # terminal matching; combined-parser replay coverage lives in |
| 100 | # test_inkling.py. |
| 101 | continue |
| 102 | if cfg.name == "granite": |
| 103 | # Granite has a JSON-array tool body with no TOOL_END terminal; |
| 104 | # its replay coverage lives in test_granite.py. |
| 105 | continue |
| 106 | |
| 107 | parser_cls = type( |
| 108 | f"_Delegating{engine_cls.__name__}", |
| 109 | (DelegatingParser,), |
| 110 | { |
| 111 | "reasoning_parser_cls": adapters["reasoning"], |
| 112 | "tool_parser_cls": adapters["tool"], |
| 113 | }, |
| 114 | ) |
| 115 | found.append( |
| 116 | _PairingInfo( |
| 117 | parser_cls=parser_cls, |
| 118 | name=cfg.name, |
| 119 | samples=build_samples(cfg.name), |
| 120 | ) |
| 121 | ) |
| 122 | if missing_builders: |
| 123 | raise RuntimeError( |
| 124 | f"Engine adapters in registered_adapters have no test builder " |
| 125 | f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. " |
| 126 | f"Add a builder to _BUILDERS for each new parser." |
| 127 | ) |
| 128 | found.sort(key=lambda p: p.name) |
| 129 | return found |
| 130 | |
| 131 | |
| 132 | _PAIRINGS = _discover_pairings() |
| 133 | |
| 134 | _ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples] |
| 135 | |
| 136 | |
| 137 | @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}") |
| 138 | @pytest.mark.parametrize( |
| 139 | "parser_cls,sample", |
| 140 | _ALL_SAMPLES, |
| 141 | ids=lambda v: v.id if hasattr(v, "id") else "", |
| 142 | ) |
| 143 | def test_delegating_replay(parser_cls, sample, chunk_size): |
| 144 | tokenizer = make_mock_tokenizer(sample) |
| 145 | validated_tools = ( |
| 146 | _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None |
| 147 | ) |
| 148 | parser = parser_cls( |
| 149 | tokenizer, |
| 150 | validated_tools, |
| 151 | chat_template_kwargs=sample.chat_template_kwargs, |
| 152 | ) |
| 153 | |
| 154 | deltas = replay_streaming( |
| 155 | parser, |
| 156 | sample.tokens, |
| 157 | chunk_size=chunk_size, |
| 158 | finished_on_last=True, |
| 159 | tools=sample.tools, |
| 160 | prompt_token_ids=sample.prompt_token_ids, |
| 161 | ) |
| 162 | output = collect_output(deltas) |
| 163 | assert_parse_output(output, sample) |
| 164 | |
| 165 | |
| 166 | _TOOL_CALL_SAMPLES = [ |
| 167 | (p.parser_cls, p.name, s) |
| 168 | for p in _PAIRINGS |
| 169 | for s in p.samples |
| 170 | if s.expected_tool_calls |
| 171 | ] |
| 172 | |
| 173 | |
| 174 | @pytest.mark.parametrize( |
| 175 | "parser_cls,parser_name,sample", |
| 176 | _TOOL_CALL_SAMPLES, |
| 177 | ids=lambda v: v.id if hasattr(v, "id") else "", |
| 178 | ) |
| 179 | def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample): |
| 180 | """Non-streaming parse() with tool_choice='none' via DelegatingParser |
| 181 | must not leak special tokens into content.""" |
| 182 | tokenizer = make_mock_tokenizer(sample) |
| 183 | validated_tools = ( |
| 184 | _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None |
| 185 | ) |
| 186 | parser = parser_cls( |
| 187 | tokenizer, |
| 188 | validated_tools, |
| 189 | chat_template_kwargs=sample.chat_template_kwargs, |
| 190 | ) |
| 191 | |
| 192 | request = _test_request(tools=DUMMY_TOOLS) |
| 193 | request.tool_choice = "none" |
| 194 | |
| 195 | output = parse_non_streaming(parser, sample, request) |
| 196 | |
| 197 | assert output.tool_calls == [], ( |
| 198 | f"Expected no tool calls but got {output.tool_calls}" |
| 199 | ) |
| 200 | |
| 201 | cfg = parser._tool_parser._parser_engine.parser_engine_config |
| 202 | terminals = sorted( |
| 203 | v |
| 204 | for v in cfg.terminal_literals | set(cfg.token_id_terminals.values()) |
| 205 | if len(v) > 1 |
| 206 | ) |
| 207 | assert_no_terminal_leakage( |
| 208 | output, |
| 209 | terminals, |
| 210 | context=f"parser={parser_name}", |
| 211 | ) |