| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | """Tests for the /derender endpoints (postprocessing counterpart to /render).""" |
| 5 | |
| 6 | import httpx |
| 7 | import pytest |
| 8 | import pytest_asyncio |
| 9 | |
| 10 | from tests.utils import RemoteLaunchRenderServer |
| 11 | from vllm.tokenizers import get_tokenizer |
| 12 | |
| 13 | MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" |
| 14 | |
| 15 | |
| 16 | @pytest.fixture(scope="module") |
| 17 | def server(): |
| 18 | with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server: |
| 19 | yield remote_server |
| 20 | |
| 21 | |
| 22 | @pytest_asyncio.fixture |
| 23 | async def client(server): |
| 24 | async with httpx.AsyncClient( |
| 25 | base_url=server.url_for(""), timeout=30.0 |
| 26 | ) as http_client: |
| 27 | yield http_client |
| 28 | |
| 29 | |
| 30 | @pytest.fixture(scope="module") |
| 31 | def tokenizer(): |
| 32 | return get_tokenizer(MODEL_NAME) |
| 33 | |
| 34 | |
| 35 | # --------------------------------------------------------------------------- |
| 36 | # Helpers |
| 37 | # --------------------------------------------------------------------------- |
| 38 | |
| 39 | |
| 40 | async def _render_chat(client: httpx.AsyncClient) -> dict: |
| 41 | """Render a minimal chat request and return the GenerateRequest dict.""" |
| 42 | resp = await client.post( |
| 43 | "/v1/chat/completions/render", |
| 44 | json={ |
| 45 | "model": MODEL_NAME, |
| 46 | "messages": [{"role": "user", "content": "Hello"}], |
| 47 | }, |
| 48 | ) |
| 49 | assert resp.status_code == 200 |
| 50 | return resp.json() |
| 51 | |
| 52 | |
| 53 | def _make_generate_response( |
| 54 | token_ids: list[int] | None, |
| 55 | request_id: str = "chatcmpl-test-id", |
| 56 | finish_reason: str = "stop", |
| 57 | logprobs: dict | None = None, |
| 58 | prompt_logprobs: list | None = None, |
| 59 | kv_transfer_params: dict | None = None, |
| 60 | metrics: dict | None = None, |
| 61 | ) -> dict: |
| 62 | choice: dict = { |
| 63 | "index": 0, |
| 64 | "token_ids": token_ids, |
| 65 | "finish_reason": finish_reason, |
| 66 | "logprobs": logprobs, |
| 67 | } |
| 68 | return { |
| 69 | "request_id": request_id, |
| 70 | "choices": [choice], |
| 71 | "prompt_logprobs": prompt_logprobs, |
| 72 | "kv_transfer_params": kv_transfer_params, |
| 73 | "metrics": metrics, |
| 74 | } |
| 75 | |
| 76 | |
| 77 | def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict: |
| 78 | entry = { |
| 79 | "token": f"token_id:{token_id}", |
| 80 | "logprob": -1.0, |
| 81 | "bytes": None, |
| 82 | "top_logprobs": [ |
| 83 | {"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None} |
| 84 | ], |
| 85 | } |
| 86 | return {"content": [entry]} |
| 87 | |
| 88 | |
| 89 | # --------------------------------------------------------------------------- |
| 90 | # Chat derender tests |
| 91 | # --------------------------------------------------------------------------- |
| 92 | |
| 93 | |
| 94 | @pytest.mark.asyncio |
| 95 | async def test_derender_chat_roundtrip(client): |
| 96 | """Render then derender: decoded content should be a non-empty string.""" |
| 97 | gen_req = await _render_chat(client) |
| 98 | # Use the first 5 rendered token IDs as synthetic "generated" tokens. |
| 99 | synthetic_ids = gen_req["token_ids"][:5] |
| 100 | |
| 101 | response = await client.post( |
| 102 | "/v1/chat/completions/derender", |
| 103 | json={ |
| 104 | "model": MODEL_NAME, |
| 105 | "generate_response": _make_generate_response(synthetic_ids), |
| 106 | }, |
| 107 | ) |
| 108 | assert response.status_code == 200 |
| 109 | data = response.json() |
| 110 | assert data["object"] == "chat.completion" |
| 111 | assert len(data["choices"]) == 1 |
| 112 | assert data["choices"][0]["message"]["content"] |
| 113 | assert data["choices"][0]["message"]["role"] == "assistant" |
| 114 | |
| 115 | |
| 116 | @pytest.mark.asyncio |
| 117 | async def test_derender_chat_usage(client): |
| 118 | """Supplied prompt_tokens flows through into usage correctly.""" |
| 119 | gen_req = await _render_chat(client) |
| 120 | synthetic_ids = gen_req["token_ids"][:3] |
| 121 | |
| 122 | response = await client.post( |
| 123 | "/v1/chat/completions/derender", |
| 124 | json={ |
| 125 | "model": MODEL_NAME, |
| 126 | "generate_response": _make_generate_response(synthetic_ids), |
| 127 | "prompt_tokens": 10, |
| 128 | }, |
| 129 | ) |
| 130 | assert response.status_code == 200 |
| 131 | usage = response.json()["usage"] |
| 132 | assert usage["prompt_tokens"] == 10 |
| 133 | assert usage["completion_tokens"] == len(synthetic_ids) |
| 134 | assert usage["total_tokens"] == 10 + len(synthetic_ids) |
| 135 | |
| 136 | |
| 137 | @pytest.mark.asyncio |
| 138 | async def test_derender_chat_usage_default(client): |
| 139 | """Omitting prompt_tokens gives usage.prompt_tokens == 0.""" |
| 140 | gen_req = await _render_chat(client) |
| 141 | synthetic_ids = gen_req["token_ids"][:3] |
| 142 | |
| 143 | response = await client.post( |
| 144 | "/v1/chat/completions/derender", |
| 145 | json={ |
| 146 | "model": MODEL_NAME, |
| 147 | "generate_response": _make_generate_response(synthetic_ids), |
| 148 | }, |
| 149 | ) |
| 150 | assert response.status_code == 200 |
| 151 | usage = response.json()["usage"] |
| 152 | assert usage["prompt_tokens"] == 0 |
| 153 | |
| 154 | |
| 155 | @pytest.mark.asyncio |
| 156 | async def test_derender_chat_logprobs(client): |
| 157 | """token_id:N placeholders in content.token are resolved to real strings.""" |
| 158 | gen_req = await _render_chat(client) |
| 159 | synthetic_ids = gen_req["token_ids"][:3] |
| 160 | token_id = synthetic_ids[0] |
| 161 | |
| 162 | response = await client.post( |
| 163 | "/v1/chat/completions/derender", |
| 164 | json={ |
| 165 | "model": MODEL_NAME, |
| 166 | "generate_response": _make_generate_response( |
| 167 | synthetic_ids, |
| 168 | logprobs=_make_logprobs_with_placeholders(token_id), |
| 169 | ), |
| 170 | }, |
| 171 | ) |
| 172 | assert response.status_code == 200 |
| 173 | data = response.json() |
| 174 | logprobs = data["choices"][0]["logprobs"] |
| 175 | assert logprobs is not None |
| 176 | content = logprobs["content"] |
| 177 | assert content is not None and len(content) == 1 |
| 178 | token_str = content[0]["token"] |
| 179 | assert not token_str.startswith("token_id:"), ( |
| 180 | f"Placeholder was not resolved: {token_str!r}" |
| 181 | ) |
| 182 | |
| 183 | |
| 184 | @pytest.mark.asyncio |
| 185 | async def test_derender_chat_logprobs_bytes(client): |
| 186 | """Resolved logprob entries have bytes populated as list[int].""" |
| 187 | gen_req = await _render_chat(client) |
| 188 | synthetic_ids = gen_req["token_ids"][:3] |
| 189 | token_id = synthetic_ids[0] |
| 190 | |
| 191 | response = await client.post( |
| 192 | "/v1/chat/completions/derender", |
| 193 | json={ |
| 194 | "model": MODEL_NAME, |
| 195 | "generate_response": _make_generate_response( |
| 196 | synthetic_ids, |
| 197 | logprobs=_make_logprobs_with_placeholders(token_id), |
| 198 | ), |
| 199 | }, |
| 200 | ) |
| 201 | assert response.status_code == 200 |
| 202 | content = response.json()["choices"][0]["logprobs"]["content"] |
| 203 | bytes_field = content[0]["bytes"] |
| 204 | assert isinstance(bytes_field, list) |
| 205 | assert len(bytes_field) > 0 |
| 206 | assert all(isinstance(b, int) for b in bytes_field) |
| 207 | |
| 208 | |
| 209 | @pytest.mark.asyncio |
| 210 | async def test_derender_chat_top_logprobs(client): |
| 211 | """top_logprobs entries also have their placeholders resolved.""" |
| 212 | gen_req = await _render_chat(client) |
| 213 | synthetic_ids = gen_req["token_ids"][:3] |
| 214 | token_id = synthetic_ids[0] |
| 215 | |
| 216 | response = await client.post( |
| 217 | "/v1/chat/completions/derender", |
| 218 | json={ |
| 219 | "model": MODEL_NAME, |
| 220 | "generate_response": _make_generate_response( |
| 221 | synthetic_ids, |
| 222 | logprobs=_make_logprobs_with_placeholders(token_id), |
| 223 | ), |
| 224 | }, |
| 225 | ) |
| 226 | assert response.status_code == 200 |
| 227 | content = response.json()["choices"][0]["logprobs"]["content"] |
| 228 | top = content[0]["top_logprobs"] |
| 229 | assert len(top) == 1 |
| 230 | assert not top[0]["token"].startswith("token_id:"), ( |
| 231 | f"top_logprobs placeholder not resolved: {top[0]['token']!r}" |
| 232 | ) |
| 233 | |
| 234 | |
| 235 | @pytest.mark.asyncio |
| 236 | async def test_derender_chat_prompt_logprobs_passthrough(client): |
| 237 | """prompt_logprobs on GenerateResponse passes through unchanged.""" |
| 238 | gen_req = await _render_chat(client) |
| 239 | synthetic_ids = gen_req["token_ids"][:3] |
| 240 | # prompt_logprobs is a list[dict[int, Logprob] | None]; use None entries. |
| 241 | prompt_logprobs = [None, None] |
| 242 | |
| 243 | response = await client.post( |
| 244 | "/v1/chat/completions/derender", |
| 245 | json={ |
| 246 | "model": MODEL_NAME, |
| 247 | "generate_response": _make_generate_response( |
| 248 | synthetic_ids, prompt_logprobs=prompt_logprobs |
| 249 | ), |
| 250 | }, |
| 251 | ) |
| 252 | assert response.status_code == 200 |
| 253 | assert response.json()["prompt_logprobs"] == prompt_logprobs |
| 254 | |
| 255 | |
| 256 | @pytest.mark.asyncio |
| 257 | async def test_derender_chat_kv_transfer_params_passthrough(client): |
| 258 | """kv_transfer_params passes through to the ChatCompletionResponse.""" |
| 259 | gen_req = await _render_chat(client) |
| 260 | synthetic_ids = gen_req["token_ids"][:3] |
| 261 | kv = {"key": "value"} |
| 262 | |
| 263 | response = await client.post( |
| 264 | "/v1/chat/completions/derender", |
| 265 | json={ |
| 266 | "model": MODEL_NAME, |
| 267 | "generate_response": _make_generate_response( |
| 268 | synthetic_ids, kv_transfer_params=kv |
| 269 | ), |
| 270 | }, |
| 271 | ) |
| 272 | assert response.status_code == 200 |
| 273 | assert response.json()["kv_transfer_params"] == kv |
| 274 | |
| 275 | |
| 276 | @pytest.mark.asyncio |
| 277 | async def test_derender_chat_metrics_passthrough(client): |
| 278 | gen_req = await _render_chat(client) |
| 279 | metrics = { |
| 280 | "speculative_decoding": { |
| 281 | "mean_acceptance_length": 2.0, |
| 282 | "draft_acceptance_rate": 0.5, |
| 283 | "acceptance_histogram": [0, 1], |
| 284 | "num_spec_steps": 1, |
| 285 | "num_accepted_draft_tokens": 1, |
| 286 | "num_draft_tokens": 2, |
| 287 | "num_spec_tokens": 1, |
| 288 | } |
| 289 | } |
| 290 | response = await client.post( |
| 291 | "/v1/chat/completions/derender", |
| 292 | json={ |
| 293 | "model": MODEL_NAME, |
| 294 | "generate_response": _make_generate_response( |
| 295 | gen_req["token_ids"][:3], metrics=metrics |
| 296 | ), |
| 297 | }, |
| 298 | ) |
| 299 | assert response.status_code == 200 |
| 300 | actual = response.json()["metrics"] |
| 301 | assert set(actual) == { |
| 302 | "time_to_first_token_ms", |
| 303 | "generation_time_ms", |
| 304 | "queue_time_ms", |
| 305 | "mean_itl_ms", |
| 306 | "tokens_per_second", |
| 307 | "speculative_decoding", |
| 308 | } |
| 309 | assert all( |
| 310 | actual[name] is None for name in actual if name != "speculative_decoding" |
| 311 | ) |
| 312 | assert actual["speculative_decoding"] == { |
| 313 | **metrics["speculative_decoding"], |
| 314 | "per_step_accepted": None, |
| 315 | "per_step_drafted": None, |
| 316 | } |
| 317 | |
| 318 | |
| 319 | @pytest.mark.asyncio |
| 320 | async def test_derender_chat_empty_token_ids(client): |
| 321 | """Empty token_ids list returns 400.""" |
| 322 | response = await client.post( |
| 323 | "/v1/chat/completions/derender", |
| 324 | json={ |
| 325 | "model": MODEL_NAME, |
| 326 | "generate_response": _make_generate_response([]), |
| 327 | }, |
| 328 | ) |
| 329 | assert response.status_code == 400 |
| 330 | |
| 331 | |
| 332 | @pytest.mark.asyncio |
| 333 | async def test_derender_chat_null_token_ids(client): |
| 334 | """Null token_ids returns 400.""" |
| 335 | response = await client.post( |
| 336 | "/v1/chat/completions/derender", |
| 337 | json={ |
| 338 | "model": MODEL_NAME, |
| 339 | "generate_response": _make_generate_response(None), |
| 340 | }, |
| 341 | ) |
| 342 | assert response.status_code == 400 |
| 343 | |
| 344 | |
| 345 | @pytest.mark.asyncio |
| 346 | async def test_derender_chat_unknown_model(client): |
| 347 | """Unknown model returns 404.""" |
| 348 | gen_req = await _render_chat(client) |
| 349 | synthetic_ids = gen_req["token_ids"][:3] |
| 350 | |
| 351 | response = await client.post( |
| 352 | "/v1/chat/completions/derender", |
| 353 | json={ |
| 354 | "model": "does-not-exist", |
| 355 | "generate_response": _make_generate_response(synthetic_ids), |
| 356 | }, |
| 357 | ) |
| 358 | assert response.status_code == 404 |
| 359 | |
| 360 | |
| 361 | @pytest.mark.asyncio |
| 362 | async def test_derender_chat_model_omitted_resolves_served_name(client): |
| 363 | """Omitting `model` resolves the served name rather than rejecting. |
| 364 | |
| 365 | Mirrors test_serving_chat.py's "full name is returned when no model is |
| 366 | specified" assertion for the derender path. Asserts the resolved value, |
| 367 | not just the status, so the fallback is proven to have fired. |
| 368 | """ |
| 369 | gen_req = await _render_chat(client) |
| 370 | synthetic_ids = gen_req["token_ids"][:3] |
| 371 | |
| 372 | response = await client.post( |
| 373 | "/v1/chat/completions/derender", |
| 374 | json={"generate_response": _make_generate_response(synthetic_ids)}, |
| 375 | ) |
| 376 | assert response.status_code == 200 |
| 377 | assert response.json()["model"] == MODEL_NAME |
| 378 | |
| 379 | |
| 380 | @pytest.mark.asyncio |
| 381 | async def test_derender_chat_leading_space_seeded_from_prompt(client, tokenizer): |
| 382 | """prompt_token_ids keeps the first token's leading space on this |
| 383 | Metaspace tokenizer, matching the coupled path.""" |
| 384 | gen_req = await _render_chat(client) |
| 385 | prompt_token_ids = gen_req["token_ids"] |
| 386 | output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False) |
| 387 | |
| 388 | unseeded = await client.post( |
| 389 | "/v1/chat/completions/derender", |
| 390 | json={ |
| 391 | "model": MODEL_NAME, |
| 392 | "generate_response": _make_generate_response(output_ids), |
| 393 | }, |
| 394 | ) |
| 395 | seeded = await client.post( |
| 396 | "/v1/chat/completions/derender", |
| 397 | json={ |
| 398 | "model": MODEL_NAME, |
| 399 | "generate_response": _make_generate_response(output_ids), |
| 400 | "prompt_token_ids": prompt_token_ids, |
| 401 | }, |
| 402 | ) |
| 403 | assert unseeded.status_code == 200 |
| 404 | assert seeded.status_code == 200 |
| 405 | unseeded_content = unseeded.json()["choices"][0]["message"]["content"] |
| 406 | seeded_content = seeded.json()["choices"][0]["message"]["content"] |
| 407 | |
| 408 | assert not unseeded_content.startswith(" ") |
| 409 | assert seeded_content.startswith(" ") |
| 410 | assert seeded_content.lstrip(" ") == unseeded_content |
| 411 | |
| 412 | |
| 413 | @pytest.mark.asyncio |
| 414 | async def test_derender_chat_oversized_prompt_token_ids_rejected(client): |
| 415 | """prompt_token_ids longer than max_model_len returns 400.""" |
| 416 | gen_req = await _render_chat(client) |
| 417 | synthetic_ids = gen_req["token_ids"][:3] |
| 418 | oversized_prompt_ids = [42] * 1_000_000 |
| 419 | |
| 420 | response = await client.post( |
| 421 | "/v1/chat/completions/derender", |
| 422 | json={ |
| 423 | "model": MODEL_NAME, |
| 424 | "generate_response": _make_generate_response(synthetic_ids), |
| 425 | "prompt_token_ids": oversized_prompt_ids, |
| 426 | }, |
| 427 | ) |
| 428 | assert response.status_code == 400 |
| 429 | assert "max_model_len" in response.json()["error"]["message"] |
| 430 | |
| 431 | |
| 432 | # --------------------------------------------------------------------------- |
| 433 | # Completion derender tests |
| 434 | # --------------------------------------------------------------------------- |
| 435 | |
| 436 | |
| 437 | async def _render_completion(client: httpx.AsyncClient, prompt: str) -> dict: |
| 438 | """Render a completion prompt and return the first GenerateRequest dict.""" |
| 439 | resp = await client.post( |
| 440 | "/v1/completions/render", |
| 441 | json={"model": MODEL_NAME, "prompt": prompt}, |
| 442 | ) |
| 443 | assert resp.status_code == 200 |
| 444 | data = resp.json() |
| 445 | assert isinstance(data, list) and len(data) >= 1 |
| 446 | return data[0] |
| 447 | |
| 448 | |
| 449 | def _make_completion_generate_response( |
| 450 | token_ids: list[int], |
| 451 | request_id: str, |
| 452 | kv_transfer_params: dict | None = None, |
| 453 | logprobs: dict | None = None, |
| 454 | ) -> dict: |
| 455 | return { |
| 456 | "request_id": request_id, |
| 457 | "choices": [ |
| 458 | { |
| 459 | "index": 0, |
| 460 | "token_ids": token_ids, |
| 461 | "finish_reason": "stop", |
| 462 | "logprobs": logprobs, |
| 463 | } |
| 464 | ], |
| 465 | "prompt_logprobs": None, |
| 466 | "kv_transfer_params": kv_transfer_params, |
| 467 | } |
| 468 | |
| 469 | |
| 470 | @pytest.mark.asyncio |
| 471 | async def test_derender_completion_roundtrip(client): |
| 472 | """Two prompts rendered, two GenerateResponses → two choices with indices 0, 1.""" |
| 473 | gr1 = await _render_completion(client, "Hello world") |
| 474 | gr2 = await _render_completion(client, "Goodbye world") |
| 475 | |
| 476 | ids1 = gr1["token_ids"][:4] |
| 477 | ids2 = gr2["token_ids"][:4] |
| 478 | |
| 479 | response = await client.post( |
| 480 | "/v1/completions/derender", |
| 481 | json={ |
| 482 | "model": MODEL_NAME, |
| 483 | "generate_responses": [ |
| 484 | _make_completion_generate_response(ids1, gr1["request_id"]), |
| 485 | _make_completion_generate_response(ids2, gr2["request_id"]), |
| 486 | ], |
| 487 | }, |
| 488 | ) |
| 489 | assert response.status_code == 200 |
| 490 | data = response.json() |
| 491 | assert data["object"] == "text_completion" |
| 492 | choices = data["choices"] |
| 493 | assert len(choices) == 2 |
| 494 | assert choices[0]["index"] == 0 |
| 495 | assert choices[1]["index"] == 1 |
| 496 | assert choices[0]["text"] |
| 497 | assert choices[1]["text"] |
| 498 | |
| 499 | |
| 500 | @pytest.mark.asyncio |
| 501 | async def test_derender_completion_usage_aggregation(client): |
| 502 | """prompt_tokens=[5, 10] is aggregated correctly into usage.""" |
| 503 | gr1 = await _render_completion(client, "Hello") |
| 504 | gr2 = await _render_completion(client, "World") |
| 505 | |
| 506 | ids1 = gr1["token_ids"][:3] |
| 507 | ids2 = gr2["token_ids"][:4] |
| 508 | |
| 509 | response = await client.post( |
| 510 | "/v1/completions/derender", |
| 511 | json={ |
| 512 | "model": MODEL_NAME, |
| 513 | "generate_responses": [ |
| 514 | _make_completion_generate_response(ids1, gr1["request_id"]), |
| 515 | _make_completion_generate_response(ids2, gr2["request_id"]), |
| 516 | ], |
| 517 | "prompt_tokens": [5, 10], |
| 518 | }, |
| 519 | ) |
| 520 | assert response.status_code == 200 |
| 521 | usage = response.json()["usage"] |
| 522 | assert usage["prompt_tokens"] == 15 |
| 523 | assert usage["completion_tokens"] == len(ids1) + len(ids2) |
| 524 | assert usage["total_tokens"] == 15 + len(ids1) + len(ids2) |
| 525 | |
| 526 | |
| 527 | @pytest.mark.asyncio |
| 528 | async def test_derender_completion_prompt_tokens_length_mismatch(client): |
| 529 | """len(prompt_tokens) != len(generate_responses) returns 400.""" |
| 530 | gr1 = await _render_completion(client, "Hello") |
| 531 | ids1 = gr1["token_ids"][:3] |
| 532 | |
| 533 | response = await client.post( |
| 534 | "/v1/completions/derender", |
| 535 | json={ |
| 536 | "model": MODEL_NAME, |
| 537 | "generate_responses": [ |
| 538 | _make_completion_generate_response(ids1, gr1["request_id"]), |
| 539 | ], |
| 540 | "prompt_tokens": [5, 10], |
| 541 | }, |
| 542 | ) |
| 543 | assert response.status_code == 400 |
| 544 | |
| 545 | |
| 546 | @pytest.mark.asyncio |
| 547 | async def test_derender_completion_prompt_token_ids_length_mismatch(client): |
| 548 | """len(prompt_token_ids) != len(generate_responses) returns 400, the |
| 549 | same as the existing prompt_tokens length check.""" |
| 550 | gr1 = await _render_completion(client, "Hello") |
| 551 | ids1 = gr1["token_ids"][:3] |
| 552 | |
| 553 | response = await client.post( |
| 554 | "/v1/completions/derender", |
| 555 | json={ |
| 556 | "model": MODEL_NAME, |
| 557 | "generate_responses": [ |
| 558 | _make_completion_generate_response(ids1, gr1["request_id"]), |
| 559 | ], |
| 560 | "prompt_token_ids": [[1, 2], [3, 4]], |
| 561 | }, |
| 562 | ) |
| 563 | assert response.status_code == 400 |
| 564 | |
| 565 | |
| 566 | @pytest.mark.asyncio |
| 567 | async def test_derender_completion_leading_space_seeded_from_prompt(client, tokenizer): |
| 568 | """Completions counterpart of |
| 569 | test_derender_chat_leading_space_seeded_from_prompt.""" |
| 570 | gr1 = await _render_completion(client, "Hello world") |
| 571 | prompt_token_ids = gr1["token_ids"] |
| 572 | output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False) |
| 573 | |
| 574 | unseeded = await client.post( |
| 575 | "/v1/completions/derender", |
| 576 | json={ |
| 577 | "model": MODEL_NAME, |
| 578 | "generate_responses": [ |
| 579 | _make_completion_generate_response(output_ids, gr1["request_id"]), |
| 580 | ], |
| 581 | }, |
| 582 | ) |
| 583 | seeded = await client.post( |
| 584 | "/v1/completions/derender", |
| 585 | json={ |
| 586 | "model": MODEL_NAME, |
| 587 | "generate_responses": [ |
| 588 | _make_completion_generate_response(output_ids, gr1["request_id"]), |
| 589 | ], |
| 590 | "prompt_token_ids": [prompt_token_ids], |
| 591 | }, |
| 592 | ) |
| 593 | assert unseeded.status_code == 200 |
| 594 | assert seeded.status_code == 200 |
| 595 | unseeded_text = unseeded.json()["choices"][0]["text"] |
| 596 | seeded_text = seeded.json()["choices"][0]["text"] |
| 597 | |
| 598 | assert not unseeded_text.startswith(" ") |
| 599 | assert seeded_text.startswith(" ") |
| 600 | assert seeded_text.lstrip(" ") == unseeded_text |
| 601 | |
| 602 | |
| 603 | @pytest.mark.asyncio |
| 604 | async def test_derender_completion_oversized_prompt_token_ids_rejected(client): |
| 605 | """A prompt_token_ids entry longer than max_model_len returns 400.""" |
| 606 | gr1 = await _render_completion(client, "Hello") |
| 607 | ids1 = gr1["token_ids"][:3] |
| 608 | oversized_prompt_ids = [42] * 1_000_000 |
| 609 | |
| 610 | response = await client.post( |
| 611 | "/v1/completions/derender", |
| 612 | json={ |
| 613 | "model": MODEL_NAME, |
| 614 | "generate_responses": [ |
| 615 | _make_completion_generate_response(ids1, gr1["request_id"]), |
| 616 | ], |
| 617 | "prompt_token_ids": [oversized_prompt_ids], |
| 618 | }, |
| 619 | ) |
| 620 | assert response.status_code == 400 |
| 621 | assert "max_model_len" in response.json()["error"]["message"] |
| 622 | |
| 623 | |
| 624 | @pytest.mark.asyncio |
| 625 | async def test_derender_completion_empty_generate_responses(client): |
| 626 | """Empty generate_responses list returns 400.""" |
| 627 | response = await client.post( |
| 628 | "/v1/completions/derender", |
| 629 | json={"model": MODEL_NAME, "generate_responses": []}, |
| 630 | ) |
| 631 | assert response.status_code == 400 |
| 632 | |
| 633 | |
| 634 | @pytest.mark.asyncio |
| 635 | async def test_derender_completion_logprobs(client): |
| 636 | """token_id:N placeholders in logprobs are resolved; CompletionLogProbs |
| 637 | flat-list structure is returned with non-empty tokens and text_offsets.""" |
| 638 | gr1 = await _render_completion(client, "Hello world") |
| 639 | ids1 = gr1["token_ids"][:3] |
| 640 | token_id = ids1[0] |
| 641 | |
| 642 | response = await client.post( |
| 643 | "/v1/completions/derender", |
| 644 | json={ |
| 645 | "model": MODEL_NAME, |
| 646 | "generate_responses": [ |
| 647 | _make_completion_generate_response( |
| 648 | ids1, |
| 649 | gr1["request_id"], |
| 650 | logprobs=_make_logprobs_with_placeholders(token_id), |
| 651 | ), |
| 652 | ], |
| 653 | }, |
| 654 | ) |
| 655 | assert response.status_code == 200 |
| 656 | logprobs = response.json()["choices"][0]["logprobs"] |
| 657 | assert logprobs is not None |
| 658 | tokens = logprobs["tokens"] |
| 659 | assert len(tokens) == 1 |
| 660 | assert not tokens[0].startswith("token_id:"), ( |
| 661 | f"Placeholder was not resolved: {tokens[0]!r}" |
| 662 | ) |
| 663 | assert len(logprobs["token_logprobs"]) == 1 |
| 664 | assert isinstance(logprobs["token_logprobs"][0], float) |
| 665 | assert len(logprobs["text_offset"]) == 1 |
| 666 | assert logprobs["text_offset"][0] == 0 |
| 667 | |
| 668 | |
| 669 | @pytest.mark.asyncio |
| 670 | async def test_derender_completion_kv_transfer_params_passthrough(client): |
| 671 | """kv_transfer_params passes through to CompletionResponse.""" |
| 672 | gr1 = await _render_completion(client, "Hello") |
| 673 | ids1 = gr1["token_ids"][:3] |
| 674 | kv = {"node": "abc"} |
| 675 | |
| 676 | response = await client.post( |
| 677 | "/v1/completions/derender", |
| 678 | json={ |
| 679 | "model": MODEL_NAME, |
| 680 | "generate_responses": [ |
| 681 | _make_completion_generate_response( |
| 682 | ids1, gr1["request_id"], kv_transfer_params=kv |
| 683 | ), |
| 684 | ], |
| 685 | }, |
| 686 | ) |
| 687 | assert response.status_code == 200 |
| 688 | assert response.json()["kv_transfer_params"] == kv |
| 689 | |
| 690 | |
| 691 | # --------------------------------------------------------------------------- |
| 692 | # Resource bounds regression tests |
| 693 | # --------------------------------------------------------------------------- |
| 694 | |
| 695 | |
| 696 | @pytest.mark.asyncio |
| 697 | async def test_derender_chat_bounded_payload_succeeds(client): |
| 698 | """Normal bounded derender payload succeeds (positive control).""" |
| 699 | gen_req = await _render_chat(client) |
| 700 | synthetic_ids = gen_req["token_ids"][:5] |
| 701 | |
| 702 | response = await client.post( |
| 703 | "/v1/chat/completions/derender", |
| 704 | json={ |
| 705 | "model": MODEL_NAME, |
| 706 | "generate_response": _make_generate_response(synthetic_ids), |
| 707 | }, |
| 708 | ) |
| 709 | assert response.status_code == 200 |
| 710 | data = response.json() |
| 711 | assert len(data["choices"]) == 1 |
| 712 | assert data["choices"][0]["message"]["content"] |
| 713 | |
| 714 | |
| 715 | @pytest.mark.asyncio |
| 716 | async def test_derender_chat_oversized_token_ids_rejected(client): |
| 717 | """token_ids longer than max_model_len returns 400.""" |
| 718 | response = await client.get("/v1/models") |
| 719 | assert response.status_code == 200 |
| 720 | |
| 721 | # Use a token_ids list that exceeds any reasonable max_model_len. |
| 722 | # The tiny-random model has max_model_len of 2048. |
| 723 | oversized_ids = [42] * 1_000_000 |
| 724 | |
| 725 | response = await client.post( |
| 726 | "/v1/chat/completions/derender", |
| 727 | json={ |
| 728 | "model": MODEL_NAME, |
| 729 | "generate_response": _make_generate_response(oversized_ids), |
| 730 | }, |
| 731 | ) |
| 732 | assert response.status_code == 400 |
| 733 | assert "max_model_len" in response.json()["error"]["message"] |
| 734 | |
| 735 | |
| 736 | @pytest.mark.asyncio |
| 737 | async def test_derender_chat_too_many_choices_rejected(client): |
| 738 | """Choices count exceeding VLLM_MAX_N_SEQUENCES returns 400.""" |
| 739 | # Default VLLM_MAX_N_SEQUENCES is 16384; use a larger count. |
| 740 | oversized_choices = [ |
| 741 | {"index": i, "token_ids": [42], "finish_reason": "stop"} for i in range(20_000) |
| 742 | ] |
| 743 | response = await client.post( |
| 744 | "/v1/chat/completions/derender", |
| 745 | json={ |
| 746 | "model": MODEL_NAME, |
| 747 | "generate_response": { |
| 748 | "request_id": "test-choices-bound", |
| 749 | "choices": oversized_choices, |
| 750 | }, |
| 751 | }, |
| 752 | ) |
| 753 | assert response.status_code == 400 |
| 754 | assert "choices count" in response.json()["error"]["message"] |
| 755 | |
| 756 | |
| 757 | @pytest.mark.asyncio |
| 758 | async def test_derender_completion_too_many_generate_responses_rejected(client): |
| 759 | """generate_responses count exceeding limit returns 400.""" |
| 760 | oversized_responses = [ |
| 761 | { |
| 762 | "request_id": f"gen-{i}", |
| 763 | "choices": [{"index": 0, "token_ids": [42], "finish_reason": "stop"}], |
| 764 | } |
| 765 | for i in range(20_000) |
| 766 | ] |
| 767 | response = await client.post( |
| 768 | "/v1/completions/derender", |
| 769 | json={ |
| 770 | "model": MODEL_NAME, |
| 771 | "generate_responses": oversized_responses, |
| 772 | }, |
| 773 | ) |
| 774 | assert response.status_code == 400 |
| 775 | assert "generate_responses count" in response.json()["error"]["message"] |
| 776 | |
| 777 | |
| 778 | @pytest.mark.asyncio |
| 779 | async def test_derender_chat_negative_token_ids_rejected(client): |
| 780 | """Negative token_ids are rejected at the protocol validation level.""" |
| 781 | response = await client.post( |
| 782 | "/v1/chat/completions/derender", |
| 783 | json={ |
| 784 | "model": MODEL_NAME, |
| 785 | "generate_response": _make_generate_response([-1, 42, 100]), |
| 786 | }, |
| 787 | ) |
| 788 | # vLLM's validation_exception_handler converts Pydantic errors to 400 |
| 789 | assert response.status_code == 400 |
| 790 | |
| 791 | |
| 792 | @pytest.mark.asyncio |
| 793 | async def test_derender_chat_oversized_logprobs_rejected(client): |
| 794 | """logprobs.content longer than max_model_len returns 400.""" |
| 795 | oversized_logprobs: dict = { |
| 796 | "content": [ |
| 797 | {"token": "x", "logprob": -1.0, "bytes": None, "top_logprobs": []} |
| 798 | for _ in range(1_000_000) |
| 799 | ] |
| 800 | } |
| 801 | response = await client.post( |
| 802 | "/v1/chat/completions/derender", |
| 803 | json={ |
| 804 | "model": MODEL_NAME, |
| 805 | "generate_response": { |
| 806 | "request_id": "test-logprobs-bound", |
| 807 | "choices": [ |
| 808 | { |
| 809 | "index": 0, |
| 810 | "token_ids": [42], |
| 811 | "finish_reason": "stop", |
| 812 | "logprobs": oversized_logprobs, |
| 813 | } |
| 814 | ], |
| 815 | }, |
| 816 | }, |
| 817 | ) |
| 818 | assert response.status_code == 400 |
| 819 | assert "logprobs.content length" in response.json()["error"]["message"] |
| 820 | |
| 821 | |
| 822 | @pytest.mark.asyncio |
| 823 | async def test_derender_chat_oversized_top_logprobs_rejected(client): |
| 824 | """top_logprobs count exceeding max_logprobs (default 20) returns 400.""" |
| 825 | oversized_top_logprobs = { |
| 826 | "content": [ |
| 827 | { |
| 828 | "token": "x", |
| 829 | "logprob": -1.0, |
| 830 | "bytes": None, |
| 831 | "top_logprobs": [ |
| 832 | {"token": f"t{i}", "logprob": -float(i), "bytes": None} |
| 833 | for i in range(25) |
| 834 | ], |
| 835 | } |
| 836 | ] |
| 837 | } |
| 838 | response = await client.post( |
| 839 | "/v1/chat/completions/derender", |
| 840 | json={ |
| 841 | "model": MODEL_NAME, |
| 842 | "generate_response": { |
| 843 | "request_id": "test-top-logprobs-bound", |
| 844 | "choices": [ |
| 845 | { |
| 846 | "index": 0, |
| 847 | "token_ids": [42], |
| 848 | "finish_reason": "stop", |
| 849 | "logprobs": oversized_top_logprobs, |
| 850 | } |
| 851 | ], |
| 852 | }, |
| 853 | }, |
| 854 | ) |
| 855 | assert response.status_code == 400 |
| 856 | msg = response.json()["error"]["message"] |
| 857 | assert "top_logprobs count" in msg |
| 858 | assert "max_logprobs" in msg |
| 859 | |
| 860 | |
| 861 | @pytest.mark.asyncio |
| 862 | async def test_derender_completion_oversized_token_ids_rejected(client): |
| 863 | """Completion endpoint also rejects oversized token_ids.""" |
| 864 | oversized_ids = [42] * 1_000_000 |
| 865 | response = await client.post( |
| 866 | "/v1/completions/derender", |
| 867 | json={ |
| 868 | "model": MODEL_NAME, |
| 869 | "generate_responses": [ |
| 870 | { |
| 871 | "request_id": "gen-0", |
| 872 | "choices": [ |
| 873 | { |
| 874 | "index": 0, |
| 875 | "token_ids": oversized_ids, |
| 876 | "finish_reason": "stop", |
| 877 | } |
| 878 | ], |
| 879 | } |
| 880 | ], |
| 881 | }, |
| 882 | ) |
| 883 | assert response.status_code == 400 |
| 884 | assert "max_model_len" in response.json()["error"]["message"] |
| 885 | |
| 886 | |
| 887 | # --------------------------------------------------------------------------- |
| 888 | # E2E: render -> derender roundtrip with parser (reasoning + tool calls) |
| 889 | # --------------------------------------------------------------------------- |
| 890 | |
| 891 | PARSER_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" |
| 892 | |
| 893 | _E2E_TOOLS = [ |
| 894 | { |
| 895 | "type": "function", |
| 896 | "function": { |
| 897 | "name": "get_weather", |
| 898 | "description": "Get weather for a city", |
| 899 | "parameters": { |
| 900 | "type": "object", |
| 901 | "properties": {"city": {"type": "string"}}, |
| 902 | }, |
| 903 | }, |
| 904 | } |
| 905 | ] |
| 906 | |
| 907 | |
| 908 | @pytest.fixture(scope="module") |
| 909 | def parser_server(): |
| 910 | args = [ |
| 911 | "--enable-auto-tool-choice", |
| 912 | "--tool-call-parser", |
| 913 | "hermes", |
| 914 | "--reasoning-parser", |
| 915 | "deepseek_r1", |
| 916 | ] |
| 917 | with RemoteLaunchRenderServer(PARSER_MODEL, args) as remote_server: |
| 918 | yield remote_server |
| 919 | |
| 920 | |
| 921 | @pytest_asyncio.fixture |
| 922 | async def parser_client(parser_server): |
| 923 | async with httpx.AsyncClient( |
| 924 | base_url=parser_server.url_for(""), timeout=60.0 |
| 925 | ) as http_client: |
| 926 | yield http_client |
| 927 | |
| 928 | |
| 929 | @pytest.fixture(scope="module") |
| 930 | def parser_tokenizer(): |
| 931 | return get_tokenizer(PARSER_MODEL) |
| 932 | |
| 933 | |
| 934 | def _encode(tokenizer, text: str) -> list[int]: |
| 935 | return tokenizer.encode(text, add_special_tokens=False) |
| 936 | |
| 937 | |
| 938 | def _decoded(tokenizer, token_ids: list[int]) -> str: |
| 939 | return tokenizer.decode(token_ids, skip_special_tokens=True) |
| 940 | |
| 941 | |
| 942 | def _require_markers_survive(tokenizer, text: str, *markers: str) -> list[int]: |
| 943 | """Encode text and skip the test if any marker is lost in roundtrip.""" |
| 944 | ids = _encode(tokenizer, text) |
| 945 | decoded = tokenizer.decode(ids, skip_special_tokens=False) |
| 946 | for m in markers: |
| 947 | if m not in decoded: |
| 948 | pytest.skip(f"Marker {m!r} lost in encode->decode roundtrip") |
| 949 | return ids |
| 950 | |
| 951 | |
| 952 | async def _e2e_render_chat( |
| 953 | client: httpx.AsyncClient, |
| 954 | model: str, |
| 955 | messages: list[dict], |
| 956 | ) -> dict: |
| 957 | resp = await client.post( |
| 958 | "/v1/chat/completions/render", |
| 959 | json={"model": model, "messages": messages}, |
| 960 | ) |
| 961 | assert resp.status_code == 200, resp.text |
| 962 | return resp.json() |
| 963 | |
| 964 | |
| 965 | def _e2e_generate_response( |
| 966 | token_ids: list[int], |
| 967 | request_id: str = "chatcmpl-e2e-test", |
| 968 | ) -> dict: |
| 969 | return { |
| 970 | "request_id": request_id, |
| 971 | "choices": [ |
| 972 | { |
| 973 | "index": 0, |
| 974 | "token_ids": token_ids, |
| 975 | "finish_reason": "stop", |
| 976 | } |
| 977 | ], |
| 978 | } |
| 979 | |
| 980 | |
| 981 | @pytest.mark.asyncio |
| 982 | async def test_e2e_plain_roundtrip(parser_client, parser_tokenizer): |
| 983 | """Plain text without reasoning markers roundtrips correctly. |
| 984 | |
| 985 | Markerless output has no ``</think>``, which deepseek_r1 classifies |
| 986 | wholly as reasoning, so the text lands there rather than in content. |
| 987 | What this pins is detokenization fidelity through the parser path. |
| 988 | """ |
| 989 | messages = [{"role": "user", "content": "What is 2+2?"}] |
| 990 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 991 | |
| 992 | answer = "The answer is four." |
| 993 | output_ids = _encode(parser_tokenizer, answer) |
| 994 | expected = _decoded(parser_tokenizer, output_ids) |
| 995 | |
| 996 | resp = await parser_client.post( |
| 997 | "/v1/chat/completions/derender", |
| 998 | json={ |
| 999 | "model": PARSER_MODEL, |
| 1000 | "generate_response": _e2e_generate_response(output_ids), |
| 1001 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1002 | "chat_request": {"model": PARSER_MODEL, "messages": messages}, |
| 1003 | }, |
| 1004 | ) |
| 1005 | assert resp.status_code == 200, resp.text |
| 1006 | message = resp.json()["choices"][0]["message"] |
| 1007 | assert message["reasoning"] == expected |
| 1008 | assert message["content"] is None |
| 1009 | |
| 1010 | |
| 1011 | @pytest.mark.asyncio |
| 1012 | async def test_e2e_token_identity(parser_client, parser_tokenizer): |
| 1013 | """encode(derender(token_ids)) == token_ids (RL invariant). |
| 1014 | |
| 1015 | Markerless output comes back as reasoning (see |
| 1016 | ``test_e2e_plain_roundtrip``), so re-encode that. |
| 1017 | """ |
| 1018 | messages = [{"role": "user", "content": "Hi"}] |
| 1019 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 1020 | |
| 1021 | answer = "Hello! How can I help?" |
| 1022 | output_ids = _encode(parser_tokenizer, answer) |
| 1023 | |
| 1024 | resp = await parser_client.post( |
| 1025 | "/v1/chat/completions/derender", |
| 1026 | json={ |
| 1027 | "model": PARSER_MODEL, |
| 1028 | "generate_response": _e2e_generate_response(output_ids), |
| 1029 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1030 | "chat_request": {"model": PARSER_MODEL, "messages": messages}, |
| 1031 | }, |
| 1032 | ) |
| 1033 | assert resp.status_code == 200 |
| 1034 | reasoning = resp.json()["choices"][0]["message"]["reasoning"] |
| 1035 | re_encoded = _encode(parser_tokenizer, reasoning) |
| 1036 | assert output_ids == re_encoded |
| 1037 | |
| 1038 | |
| 1039 | @pytest.mark.asyncio |
| 1040 | async def test_e2e_non_ascii_roundtrip(parser_client, parser_tokenizer): |
| 1041 | """CJK + emoji roundtrip without U+FFFD. |
| 1042 | |
| 1043 | Markerless output comes back as reasoning (see |
| 1044 | ``test_e2e_plain_roundtrip``). |
| 1045 | """ |
| 1046 | messages = [{"role": "user", "content": "Reply in Chinese"}] |
| 1047 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 1048 | |
| 1049 | answer = "你好世界 😀" |
| 1050 | output_ids = _encode(parser_tokenizer, answer) |
| 1051 | |
| 1052 | resp = await parser_client.post( |
| 1053 | "/v1/chat/completions/derender", |
| 1054 | json={ |
| 1055 | "model": PARSER_MODEL, |
| 1056 | "generate_response": _e2e_generate_response(output_ids), |
| 1057 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1058 | "chat_request": {"model": PARSER_MODEL, "messages": messages}, |
| 1059 | }, |
| 1060 | ) |
| 1061 | assert resp.status_code == 200 |
| 1062 | reasoning = resp.json()["choices"][0]["message"]["reasoning"] |
| 1063 | assert "�" not in reasoning |
| 1064 | |
| 1065 | |
| 1066 | @pytest.mark.asyncio |
| 1067 | async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer): |
| 1068 | """<think>...</think> splits into reasoning + content.""" |
| 1069 | messages = [{"role": "user", "content": "What is 2+3?"}] |
| 1070 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 1071 | |
| 1072 | reasoning_text = "The user wants 2 plus 3. That is 5." |
| 1073 | answer_text = "The answer is 5." |
| 1074 | output_text = f"<think>{reasoning_text}</think>{answer_text}" |
| 1075 | output_ids = _require_markers_survive(parser_tokenizer, output_text, "</think>") |
| 1076 | |
| 1077 | resp = await parser_client.post( |
| 1078 | "/v1/chat/completions/derender", |
| 1079 | json={ |
| 1080 | "model": PARSER_MODEL, |
| 1081 | "generate_response": _e2e_generate_response(output_ids), |
| 1082 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1083 | "chat_request": { |
| 1084 | "model": PARSER_MODEL, |
| 1085 | "messages": messages, |
| 1086 | "include_reasoning": True, |
| 1087 | }, |
| 1088 | }, |
| 1089 | ) |
| 1090 | assert resp.status_code == 200, resp.text |
| 1091 | msg = resp.json()["choices"][0]["message"] |
| 1092 | assert msg["reasoning"] is not None |
| 1093 | assert reasoning_text in msg["reasoning"] |
| 1094 | assert answer_text in msg["content"] |
| 1095 | assert "<think>" not in msg["content"] |
| 1096 | |
| 1097 | |
| 1098 | @pytest.mark.asyncio |
| 1099 | async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer): |
| 1100 | """<tool_call> extracted into tool_calls field.""" |
| 1101 | messages = [{"role": "user", "content": "Weather in Paris?"}] |
| 1102 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 1103 | |
| 1104 | output_text = ( |
| 1105 | "<think>Let me check the weather.</think>" |
| 1106 | '<tool_call>\n{"name": "get_weather", ' |
| 1107 | '"arguments": {"city": "Paris"}}\n</tool_call>' |
| 1108 | ) |
| 1109 | output_ids = _require_markers_survive( |
| 1110 | parser_tokenizer, |
| 1111 | output_text, |
| 1112 | "</think>", |
| 1113 | "<tool_call>", |
| 1114 | "</tool_call>", |
| 1115 | ) |
| 1116 | |
| 1117 | resp = await parser_client.post( |
| 1118 | "/v1/chat/completions/derender", |
| 1119 | json={ |
| 1120 | "model": PARSER_MODEL, |
| 1121 | "generate_response": _e2e_generate_response(output_ids), |
| 1122 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1123 | "chat_request": { |
| 1124 | "model": PARSER_MODEL, |
| 1125 | "messages": messages, |
| 1126 | "tools": _E2E_TOOLS, |
| 1127 | "tool_choice": "auto", |
| 1128 | }, |
| 1129 | }, |
| 1130 | ) |
| 1131 | assert resp.status_code == 200, resp.text |
| 1132 | choice = resp.json()["choices"][0] |
| 1133 | assert choice["message"]["tool_calls"] |
| 1134 | assert choice["message"]["tool_calls"][0]["function"]["name"] == "get_weather" |
| 1135 | |
| 1136 | |
| 1137 | @pytest.mark.asyncio |
| 1138 | async def test_e2e_parsed_reasoning_and_tool_call(parser_client, parser_tokenizer): |
| 1139 | """Reasoning + tool call in the same output.""" |
| 1140 | messages = [{"role": "user", "content": "Weather in Paris?"}] |
| 1141 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 1142 | |
| 1143 | reasoning_text = "I should look up the weather." |
| 1144 | tool_text = ( |
| 1145 | '<tool_call>\n{"name": "get_weather", ' |
| 1146 | '"arguments": {"city": "Paris"}}\n</tool_call>' |
| 1147 | ) |
| 1148 | output_text = f"<think>{reasoning_text}</think>{tool_text}" |
| 1149 | output_ids = _require_markers_survive( |
| 1150 | parser_tokenizer, output_text, "</think>", "<tool_call>" |
| 1151 | ) |
| 1152 | |
| 1153 | resp = await parser_client.post( |
| 1154 | "/v1/chat/completions/derender", |
| 1155 | json={ |
| 1156 | "model": PARSER_MODEL, |
| 1157 | "generate_response": _e2e_generate_response(output_ids), |
| 1158 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1159 | "chat_request": { |
| 1160 | "model": PARSER_MODEL, |
| 1161 | "messages": messages, |
| 1162 | "tools": _E2E_TOOLS, |
| 1163 | "tool_choice": "auto", |
| 1164 | "include_reasoning": True, |
| 1165 | }, |
| 1166 | }, |
| 1167 | ) |
| 1168 | assert resp.status_code == 200, resp.text |
| 1169 | choice = resp.json()["choices"][0] |
| 1170 | assert choice["message"]["reasoning"] is not None |
| 1171 | assert reasoning_text in choice["message"]["reasoning"] |
| 1172 | assert choice["message"]["tool_calls"] |
| 1173 | |
| 1174 | |
| 1175 | @pytest.mark.asyncio |
| 1176 | async def test_e2e_no_chat_request_rejected(parser_client, parser_tokenizer): |
| 1177 | """Without chat_request a parser configured model rejects with 400 |
| 1178 | rather than silently falling back to plain detokenization. This is to |
| 1179 | prevent the leak of raw reasoning/tool markup into content.""" |
| 1180 | messages = [{"role": "user", "content": "Hello"}] |
| 1181 | gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) |
| 1182 | |
| 1183 | answer = "Hi there!" |
| 1184 | output_ids = _encode(parser_tokenizer, answer) |
| 1185 | |
| 1186 | resp = await parser_client.post( |
| 1187 | "/v1/chat/completions/derender", |
| 1188 | json={ |
| 1189 | "model": PARSER_MODEL, |
| 1190 | "generate_response": _e2e_generate_response(output_ids), |
| 1191 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1192 | }, |
| 1193 | ) |
| 1194 | assert resp.status_code == 400 |
| 1195 | assert "chat_request" in resp.json()["error"]["message"] |
| 1196 | |
| 1197 | |
| 1198 | # --------------------------------------------------------------------------- |
| 1199 | # E2E: HarmonyParser + GPT-OSS |
| 1200 | # --------------------------------------------------------------------------- |
| 1201 | |
| 1202 | HARMONY_MODEL = "openai/gpt-oss-20b" |
| 1203 | |
| 1204 | |
| 1205 | def _ensure_harmony_vocab(): |
| 1206 | """Pre-cache the o200k_base BPE file needed by openai-harmony. |
| 1207 | |
| 1208 | The Rust tiktoken-rs backend downloads from Azure Blob Storage, which |
| 1209 | may be unreachable in some environments. When the cache is cold we |
| 1210 | fetch the file ourselves and place it in ``/tmp/tiktoken-rs-cache/`` |
| 1211 | using the SHA-1(URL) filename that tiktoken-rs expects. |
| 1212 | """ |
| 1213 | import hashlib |
| 1214 | import urllib.request |
| 1215 | from pathlib import Path |
| 1216 | |
| 1217 | url = "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" |
| 1218 | cache_dir = Path("/tmp/tiktoken-rs-cache") |
| 1219 | cache_key = hashlib.sha1(url.encode()).hexdigest() |
| 1220 | cache_file = cache_dir / cache_key |
| 1221 | if not cache_file.exists(): |
| 1222 | cache_dir.mkdir(parents=True, exist_ok=True) |
| 1223 | urllib.request.urlretrieve(url, cache_file) |
| 1224 | |
| 1225 | |
| 1226 | @pytest.fixture(scope="module") |
| 1227 | def harmony_server(): |
| 1228 | _ensure_harmony_vocab() |
| 1229 | args = [ |
| 1230 | "--trust-remote-code", |
| 1231 | "--enable-auto-tool-choice", |
| 1232 | "--tool-call-parser", |
| 1233 | "openai", |
| 1234 | "--reasoning-parser", |
| 1235 | "openai_gptoss", |
| 1236 | ] |
| 1237 | with RemoteLaunchRenderServer(HARMONY_MODEL, args) as remote_server: |
| 1238 | yield remote_server |
| 1239 | |
| 1240 | |
| 1241 | @pytest_asyncio.fixture |
| 1242 | async def harmony_client(harmony_server): |
| 1243 | async with httpx.AsyncClient( |
| 1244 | base_url=harmony_server.url_for(""), timeout=60.0 |
| 1245 | ) as http_client: |
| 1246 | yield http_client |
| 1247 | |
| 1248 | |
| 1249 | @pytest.fixture(scope="module") |
| 1250 | def harmony_tokenizer(): |
| 1251 | return get_tokenizer(HARMONY_MODEL, trust_remote_code=True) |
| 1252 | |
| 1253 | |
| 1254 | def _harmony_extract_assistant_ids( |
| 1255 | tokenizer, assistant_msg: dict, user_content: str = "test" |
| 1256 | ) -> list[int]: |
| 1257 | """Extract assistant token IDs via apply_chat_template diff.""" |
| 1258 | prompt = [{"role": "user", "content": user_content}] |
| 1259 | full = prompt + [assistant_msg] |
| 1260 | text_prompt = tokenizer.apply_chat_template( |
| 1261 | prompt, add_generation_prompt=True, tokenize=False |
| 1262 | ) |
| 1263 | text_full = tokenizer.apply_chat_template( |
| 1264 | full, add_generation_prompt=False, tokenize=False |
| 1265 | ) |
| 1266 | prompt_ids = tokenizer.encode(text_prompt) |
| 1267 | full_ids = tokenizer.encode(text_full) |
| 1268 | assistant_ids = list(full_ids[len(prompt_ids) :]) |
| 1269 | if not assistant_ids: |
| 1270 | pytest.skip("Could not extract assistant tokens for Harmony") |
| 1271 | return assistant_ids |
| 1272 | |
| 1273 | |
| 1274 | @pytest.mark.asyncio |
| 1275 | async def test_e2e_harmony_plain_roundtrip(harmony_client, harmony_tokenizer): |
| 1276 | """GPT-OSS content-only roundtrip.""" |
| 1277 | messages = [{"role": "user", "content": "What is 2+2?"}] |
| 1278 | gen_req = await _e2e_render_chat(harmony_client, HARMONY_MODEL, messages) |
| 1279 | |
| 1280 | assistant_msg = {"role": "assistant", "content": "Four."} |
| 1281 | output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg) |
| 1282 | |
| 1283 | resp = await harmony_client.post( |
| 1284 | "/v1/chat/completions/derender", |
| 1285 | json={ |
| 1286 | "model": HARMONY_MODEL, |
| 1287 | "generate_response": _e2e_generate_response(output_ids), |
| 1288 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1289 | "chat_request": { |
| 1290 | "model": HARMONY_MODEL, |
| 1291 | "messages": messages, |
| 1292 | }, |
| 1293 | }, |
| 1294 | ) |
| 1295 | assert resp.status_code == 200, resp.text |
| 1296 | content = resp.json()["choices"][0]["message"]["content"] |
| 1297 | assert content is not None and len(content) > 0 |
| 1298 | assert "Four" in content |
| 1299 | |
| 1300 | |
| 1301 | @pytest.mark.asyncio |
| 1302 | async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer): |
| 1303 | """GPT-OSS reasoning: analysis channel extracted.""" |
| 1304 | messages = [{"role": "user", "content": "Add 2 and 3."}] |
| 1305 | gen_req = await _e2e_render_chat(harmony_client, HARMONY_MODEL, messages) |
| 1306 | |
| 1307 | reasoning_text = "The user wants 2 plus 3." |
| 1308 | answer_text = "The answer is 5." |
| 1309 | assistant_msg = { |
| 1310 | "role": "assistant", |
| 1311 | "thinking": reasoning_text, |
| 1312 | "content": answer_text, |
| 1313 | } |
| 1314 | output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg) |
| 1315 | |
| 1316 | decoded = harmony_tokenizer.decode(output_ids) |
| 1317 | if reasoning_text not in decoded: |
| 1318 | pytest.skip("Harmony template did not render thinking") |
| 1319 | |
| 1320 | resp = await harmony_client.post( |
| 1321 | "/v1/chat/completions/derender", |
| 1322 | json={ |
| 1323 | "model": HARMONY_MODEL, |
| 1324 | "generate_response": _e2e_generate_response(output_ids), |
| 1325 | "prompt_tokens": len(gen_req["token_ids"]), |
| 1326 | "chat_request": { |
| 1327 | "model": HARMONY_MODEL, |
| 1328 | "messages": messages, |
| 1329 | "include_reasoning": True, |
| 1330 | }, |
| 1331 | }, |
| 1332 | ) |
| 1333 | assert resp.status_code == 200, resp.text |
| 1334 | msg = resp.json()["choices"][0]["message"] |
| 1335 | assert msg["reasoning"] is not None |
| 1336 | assert reasoning_text in msg["reasoning"] |
| 1337 | assert answer_text in (msg["content"] or "") |