| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | """Tests for the /render endpoints that expose prompt preprocessing.""" |
| 5 | |
| 6 | from http import HTTPStatus |
| 7 | from types import SimpleNamespace |
| 8 | from unittest.mock import AsyncMock, MagicMock |
| 9 | |
| 10 | import httpx |
| 11 | import pytest |
| 12 | import pytest_asyncio |
| 13 | |
| 14 | from tests.utils import RemoteLaunchRenderServer |
| 15 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 16 | from vllm.entrypoints.scale_out.render.api_router import router |
| 17 | from vllm.entrypoints.scale_out.render.serving import ServingRender |
| 18 | from vllm.entrypoints.serve.engine.protocol import ErrorResponse |
| 19 | from vllm.renderers.online_renderer import OnlineRenderer |
| 20 | |
| 21 | MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" |
| 22 | |
| 23 | |
| 24 | def _build_responses_serving_render() -> ServingRender: |
| 25 | serving = ServingRender.__new__(ServingRender) |
| 26 | serving.model_config = SimpleNamespace( |
| 27 | max_model_len=100, |
| 28 | is_encoder_decoder=False, |
| 29 | ) |
| 30 | serving.default_sampling_params = {} |
| 31 | serving.override_max_tokens = None |
| 32 | serving.tool_server = MagicMock() |
| 33 | serving.online_renderer = MagicMock() |
| 34 | serving.online_renderer.create_error_response = ( |
| 35 | OnlineRenderer.create_error_response.__get__(serving.online_renderer) |
| 36 | ) |
| 37 | serving.online_renderer.validate_chat_template = ( |
| 38 | OnlineRenderer.validate_chat_template.__get__(serving.online_renderer) |
| 39 | ) |
| 40 | serving.online_renderer.trust_request_chat_template = True |
| 41 | serving._check_model = AsyncMock(return_value=None) |
| 42 | return serving |
| 43 | |
| 44 | |
| 45 | @pytest.mark.skip_global_cleanup |
| 46 | def test_responses_render_route_is_registered(): |
| 47 | assert any(route.path == "/v1/responses/render" for route in router.routes) |
| 48 | |
| 49 | |
| 50 | @pytest.mark.skip_global_cleanup |
| 51 | def test_responses_render_route_documents_not_implemented(): |
| 52 | route = next( |
| 53 | route for route in router.routes if route.path == "/v1/responses/render" |
| 54 | ) |
| 55 | |
| 56 | assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses |
| 57 | |
| 58 | |
| 59 | @pytest.mark.asyncio |
| 60 | @pytest.mark.skip_global_cleanup |
| 61 | async def test_render_responses_returns_generate_request_without_stored_state(): |
| 62 | serving = _build_responses_serving_render() |
| 63 | serving.online_renderer.render_responses = AsyncMock( |
| 64 | return_value=MagicMock( |
| 65 | messages=[{"role": "user", "content": "Test prompt"}], |
| 66 | engine_input={"prompt_token_ids": [7, 8, 9]}, |
| 67 | ) |
| 68 | ) |
| 69 | request = ResponsesRequest( |
| 70 | model=MODEL_NAME, |
| 71 | input="Test prompt", |
| 72 | max_output_tokens=12, |
| 73 | stream=True, |
| 74 | cache_salt="request-salt", |
| 75 | priority=3, |
| 76 | kv_transfer_params={"do_remote_prefill": True}, |
| 77 | ec_transfer_params={"remote": True}, |
| 78 | ) |
| 79 | |
| 80 | response = await serving.render_responses_request(request) |
| 81 | |
| 82 | assert response.token_ids == [7, 8, 9] |
| 83 | assert response.request_id == request.request_id |
| 84 | assert response.sampling_params.max_tokens == 12 |
| 85 | assert response.model == MODEL_NAME |
| 86 | assert response.stream is True |
| 87 | assert response.cache_salt == "request-salt" |
| 88 | assert response.priority == 3 |
| 89 | assert response.kv_transfer_params == {"do_remote_prefill": True} |
| 90 | assert response.ec_transfer_params == {"remote": True} |
| 91 | serving.online_renderer.render_responses.assert_awaited_once_with( |
| 92 | request, |
| 93 | previous_messages=None, |
| 94 | previous_response_outputs=None, |
| 95 | tool_server=serving.tool_server, |
| 96 | skip_mm_cache=True, |
| 97 | ) |
| 98 | |
| 99 | |
| 100 | @pytest.mark.asyncio |
| 101 | @pytest.mark.skip_global_cleanup |
| 102 | async def test_render_responses_rejects_previous_response_id(): |
| 103 | serving = _build_responses_serving_render() |
| 104 | serving.online_renderer.render_responses = AsyncMock() |
| 105 | request = ResponsesRequest( |
| 106 | model=MODEL_NAME, |
| 107 | input="Test prompt", |
| 108 | previous_response_id="resp_previous", |
| 109 | ) |
| 110 | |
| 111 | response = await serving.render_responses_request(request) |
| 112 | |
| 113 | assert isinstance(response, ErrorResponse) |
| 114 | assert response.error.code == 400 |
| 115 | assert response.error.param == "previous_response_id" |
| 116 | serving.online_renderer.render_responses.assert_not_awaited() |
| 117 | |
| 118 | |
| 119 | @pytest.mark.asyncio |
| 120 | @pytest.mark.skip_global_cleanup |
| 121 | async def test_render_responses_rejects_empty_token_ids(): |
| 122 | serving = _build_responses_serving_render() |
| 123 | serving.online_renderer.render_responses = AsyncMock( |
| 124 | return_value=MagicMock( |
| 125 | messages=[], |
| 126 | engine_input={"prompt_token_ids": []}, |
| 127 | ) |
| 128 | ) |
| 129 | |
| 130 | response = await serving.render_responses_request( |
| 131 | ResponsesRequest(model=MODEL_NAME, input="Test prompt") |
| 132 | ) |
| 133 | |
| 134 | assert isinstance(response, ErrorResponse) |
| 135 | assert response.error.message == "No token_ids rendered" |
| 136 | |
| 137 | |
| 138 | @pytest.fixture(scope="module") |
| 139 | def server(): |
| 140 | args: list[str] = ["--trust-request-chat-template"] |
| 141 | |
| 142 | with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server: |
| 143 | yield remote_server |
| 144 | |
| 145 | |
| 146 | @pytest_asyncio.fixture |
| 147 | async def client(server): |
| 148 | async with httpx.AsyncClient( |
| 149 | base_url=server.url_for(""), timeout=30.0 |
| 150 | ) as http_client: |
| 151 | yield http_client |
| 152 | |
| 153 | |
| 154 | @pytest.mark.asyncio |
| 155 | async def test_responses_render_basic(client): |
| 156 | response = await client.post( |
| 157 | "/v1/responses/render", |
| 158 | json={ |
| 159 | "model": MODEL_NAME, |
| 160 | "input": "When should a Responses handler return an empty string?", |
| 161 | "max_output_tokens": 7, |
| 162 | }, |
| 163 | ) |
| 164 | |
| 165 | assert response.status_code == 200 |
| 166 | data = response.json() |
| 167 | assert data["model"] == MODEL_NAME |
| 168 | assert data["request_id"].startswith("resp_") |
| 169 | assert data["sampling_params"]["max_tokens"] == 7 |
| 170 | assert data["token_ids"] |
| 171 | |
| 172 | |
| 173 | @pytest.mark.asyncio |
| 174 | async def test_responses_render_includes_prior_input_items(client): |
| 175 | current_turn = { |
| 176 | "role": "user", |
| 177 | "content": "Which color should I remember?", |
| 178 | } |
| 179 | current_turn_response = await client.post( |
| 180 | "/v1/responses/render", |
| 181 | json={"model": MODEL_NAME, "input": [current_turn]}, |
| 182 | ) |
| 183 | multi_turn_response = await client.post( |
| 184 | "/v1/responses/render", |
| 185 | json={ |
| 186 | "model": MODEL_NAME, |
| 187 | "input": [ |
| 188 | {"role": "user", "content": "Remember the color cobalt."}, |
| 189 | {"role": "assistant", "content": "I will remember cobalt."}, |
| 190 | current_turn, |
| 191 | ], |
| 192 | }, |
| 193 | ) |
| 194 | |
| 195 | assert current_turn_response.status_code == 200 |
| 196 | assert multi_turn_response.status_code == 200 |
| 197 | current_turn_token_ids = current_turn_response.json()["token_ids"] |
| 198 | multi_turn_token_ids = multi_turn_response.json()["token_ids"] |
| 199 | assert len(multi_turn_token_ids) > len(current_turn_token_ids) |
| 200 | |
| 201 | |
| 202 | @pytest.mark.asyncio |
| 203 | async def test_responses_render_rejects_previous_response_id_over_http(client): |
| 204 | response = await client.post( |
| 205 | "/v1/responses/render", |
| 206 | json={ |
| 207 | "model": MODEL_NAME, |
| 208 | "input": "Continue the previous response.", |
| 209 | "previous_response_id": "resp_previous", |
| 210 | }, |
| 211 | ) |
| 212 | |
| 213 | assert response.status_code == 400 |
| 214 | error = response.json()["error"] |
| 215 | assert error["type"] == "invalid_request_error" |
| 216 | assert error["param"] == "previous_response_id" |
| 217 | |
| 218 | |
| 219 | @pytest.mark.asyncio |
| 220 | async def test_completion_render_basic(client): |
| 221 | """Test basic completion render endpoint.""" |
| 222 | # Make request to render endpoint |
| 223 | response = await client.post( |
| 224 | "/v1/completions/render", |
| 225 | json={ |
| 226 | "model": MODEL_NAME, |
| 227 | "prompt": "When should a chat-completions handler return an empty string?", |
| 228 | }, |
| 229 | ) |
| 230 | |
| 231 | assert response.status_code == 200 |
| 232 | data = response.json() |
| 233 | |
| 234 | # Verify response structure - list of GenerateRequest |
| 235 | assert isinstance(data, list) |
| 236 | assert len(data) > 0 |
| 237 | |
| 238 | # Verify first prompt is a GenerateRequest |
| 239 | first_prompt = data[0] |
| 240 | assert "token_ids" in first_prompt |
| 241 | assert "sampling_params" in first_prompt |
| 242 | assert "model" in first_prompt |
| 243 | assert "request_id" in first_prompt |
| 244 | assert isinstance(first_prompt["token_ids"], list) |
| 245 | assert len(first_prompt["token_ids"]) > 0 |
| 246 | assert first_prompt["model"] == MODEL_NAME |
| 247 | assert first_prompt["request_id"].startswith("cmpl-") |
| 248 | |
| 249 | |
| 250 | @pytest.mark.asyncio |
| 251 | async def test_chat_completion_render_basic(client): |
| 252 | """Test basic chat completion render endpoint.""" |
| 253 | # Make request to render endpoint |
| 254 | response = await client.post( |
| 255 | "/v1/chat/completions/render", |
| 256 | json={ |
| 257 | "model": MODEL_NAME, |
| 258 | "messages": [ |
| 259 | { |
| 260 | "role": "user", |
| 261 | "content": ( |
| 262 | "Returning an empty string for the prompt may be confusing." |
| 263 | ), |
| 264 | } |
| 265 | ], |
| 266 | }, |
| 267 | ) |
| 268 | |
| 269 | assert response.status_code == 200 |
| 270 | data = response.json() |
| 271 | |
| 272 | # Verify response structure - should be a GenerateRequest |
| 273 | assert isinstance(data, dict) |
| 274 | assert "token_ids" in data |
| 275 | assert isinstance(data["token_ids"], list) |
| 276 | assert len(data["token_ids"]) > 0 |
| 277 | |
| 278 | # Verify token IDs are integers and BOS token is present |
| 279 | token_ids = data["token_ids"] |
| 280 | assert all(isinstance(tid, int) for tid in token_ids) |
| 281 | assert token_ids[0] == 1 |
| 282 | |
| 283 | |
| 284 | @pytest.mark.asyncio |
| 285 | async def test_completion_render_multiple_prompts(client): |
| 286 | """Test completion render with multiple prompts.""" |
| 287 | response = await client.post( |
| 288 | "/v1/completions/render", |
| 289 | json={ |
| 290 | "model": MODEL_NAME, |
| 291 | "prompt": ["Hello world", "Goodbye world"], |
| 292 | }, |
| 293 | ) |
| 294 | |
| 295 | assert response.status_code == 200 |
| 296 | data = response.json() |
| 297 | |
| 298 | # Should return two GenerateRequest items |
| 299 | assert isinstance(data, list) |
| 300 | assert len(data) == 2 |
| 301 | |
| 302 | # Verify both prompts have GenerateRequest fields |
| 303 | for prompt in data: |
| 304 | assert "token_ids" in prompt |
| 305 | assert "sampling_params" in prompt |
| 306 | assert "model" in prompt |
| 307 | assert "request_id" in prompt |
| 308 | assert len(prompt["token_ids"]) > 0 |
| 309 | assert prompt["request_id"].startswith("cmpl-") |
| 310 | |
| 311 | |
| 312 | @pytest.mark.asyncio |
| 313 | async def test_chat_completion_render_multi_turn(client): |
| 314 | """Test chat completion render with multi-turn conversation.""" |
| 315 | response = await client.post( |
| 316 | "/v1/chat/completions/render", |
| 317 | json={ |
| 318 | "model": MODEL_NAME, |
| 319 | "messages": [ |
| 320 | {"role": "user", "content": "Hello"}, |
| 321 | {"role": "assistant", "content": "Hi there!"}, |
| 322 | {"role": "user", "content": "How are you?"}, |
| 323 | ], |
| 324 | }, |
| 325 | ) |
| 326 | |
| 327 | assert response.status_code == 200 |
| 328 | data = response.json() |
| 329 | |
| 330 | # Verify tokenization occurred |
| 331 | assert isinstance(data, dict) |
| 332 | assert "token_ids" in data |
| 333 | assert isinstance(data["token_ids"], list) |
| 334 | assert len(data["token_ids"]) > 0 |
| 335 | |
| 336 | |
| 337 | @pytest.mark.asyncio |
| 338 | async def test_chat_completion_render_with_stream_true(client): |
| 339 | """Render accepts stream params but still returns JSON (non-streamed).""" |
| 340 | response = await client.post( |
| 341 | "/v1/chat/completions/render", |
| 342 | json={ |
| 343 | "model": MODEL_NAME, |
| 344 | "stream": True, |
| 345 | "stream_options": { |
| 346 | "include_usage": True, |
| 347 | "continuous_usage_stats": True, |
| 348 | }, |
| 349 | "messages": [ |
| 350 | { |
| 351 | "role": "user", |
| 352 | "content": "Stream options should be accepted by /render.", |
| 353 | } |
| 354 | ], |
| 355 | }, |
| 356 | ) |
| 357 | |
| 358 | assert response.status_code == 200 |
| 359 | assert response.headers.get("content-type", "").startswith("application/json") |
| 360 | |
| 361 | data = response.json() |
| 362 | assert isinstance(data, dict) |
| 363 | assert "token_ids" in data |
| 364 | assert isinstance(data["token_ids"], list) |
| 365 | assert len(data["token_ids"]) > 0 |
| 366 | |
| 367 | # /render should preserve stream fields on the returned token-in request. |
| 368 | assert data.get("stream") is True |
| 369 | assert isinstance(data.get("stream_options"), dict) |
| 370 | assert data["stream_options"].get("include_usage") is True |
| 371 | assert data["stream_options"].get("continuous_usage_stats") is True |
| 372 | |
| 373 | |
| 374 | @pytest.mark.asyncio |
| 375 | async def test_completion_render_error_invalid_model(client): |
| 376 | """Test completion render with invalid model returns error.""" |
| 377 | response = await client.post( |
| 378 | "/v1/completions/render", |
| 379 | json={ |
| 380 | "model": "invalid-model-name", |
| 381 | "prompt": "Hello", |
| 382 | }, |
| 383 | ) |
| 384 | |
| 385 | assert response.status_code == 404 |
| 386 | data = response.json() |
| 387 | assert "error" in data |
| 388 | |
| 389 | |
| 390 | @pytest.mark.asyncio |
| 391 | async def test_chat_completion_render_error_invalid_model(client): |
| 392 | """Test chat completion render with invalid model returns error.""" |
| 393 | response = await client.post( |
| 394 | "/v1/chat/completions/render", |
| 395 | json={ |
| 396 | "model": "invalid-model-name", |
| 397 | "messages": [{"role": "user", "content": "Hello"}], |
| 398 | }, |
| 399 | ) |
| 400 | |
| 401 | assert response.status_code == 404 |
| 402 | data = response.json() |
| 403 | assert "error" in data |
| 404 | |
| 405 | |
| 406 | @pytest.mark.asyncio |
| 407 | async def test_completion_render_no_generation(client): |
| 408 | """Verify render endpoint does not generate text.""" |
| 409 | # This test verifies that calling render is fast (no generation) |
| 410 | import time |
| 411 | |
| 412 | start = time.perf_counter() |
| 413 | response = await client.post( |
| 414 | "/v1/completions/render", |
| 415 | json={ |
| 416 | "model": MODEL_NAME, |
| 417 | "prompt": "Tell me a very long story about " * 10, |
| 418 | }, |
| 419 | ) |
| 420 | elapsed = time.perf_counter() - start |
| 421 | |
| 422 | assert response.status_code == 200 |
| 423 | # Render should be fast (< 1 second) since no generation |
| 424 | assert elapsed < 1.0 |
| 425 | |
| 426 | |
| 427 | @pytest.mark.asyncio |
| 428 | async def test_chat_completion_render_with_sampling_params(client): |
| 429 | """Verify sampling params are correctly returned by /render.""" |
| 430 | response = await client.post( |
| 431 | "/v1/chat/completions/render", |
| 432 | json={ |
| 433 | "model": MODEL_NAME, |
| 434 | "messages": [{"role": "user", "content": "Test sampling params"}], |
| 435 | "temperature": 0.123, |
| 436 | "top_p": 0.456, |
| 437 | "frequency_penalty": 1.1, |
| 438 | }, |
| 439 | ) |
| 440 | |
| 441 | assert response.status_code == 200 |
| 442 | data = response.json() |
| 443 | |
| 444 | assert "sampling_params" in data |
| 445 | sampling_params = data["sampling_params"] |
| 446 | |
| 447 | assert sampling_params.get("temperature") == 0.123 |
| 448 | assert sampling_params.get("top_p") == 0.456 |
| 449 | assert sampling_params.get("frequency_penalty") == 1.1 |
| 450 | |
| 451 | # Check that internal fields are not present |
| 452 | assert "_all_stop_token_ids" not in sampling_params |
| 453 | |
| 454 | |
| 455 | @pytest.mark.asyncio |
| 456 | async def test_completion_render_emits_token_offsets(client): |
| 457 | """With return_token_offsets, /v1/completions/render returns per-token |
| 458 | (start, end) char offsets aligned with token_ids.""" |
| 459 | prompt = "Hello, world." |
| 460 | response = await client.post( |
| 461 | "/v1/completions/render", |
| 462 | json={ |
| 463 | "model": MODEL_NAME, |
| 464 | "prompt": prompt, |
| 465 | "return_token_offsets": True, |
| 466 | }, |
| 467 | ) |
| 468 | |
| 469 | assert response.status_code == 200 |
| 470 | data = response.json() |
| 471 | assert isinstance(data, list) |
| 472 | offsets = data[0]["token_offsets"] |
| 473 | assert offsets is not None |
| 474 | assert len(offsets) == len(data[0]["token_ids"]) |
| 475 | for start, end in offsets: |
| 476 | assert isinstance(start, int) and isinstance(end, int) |
| 477 | assert 0 <= start <= end <= len(prompt) |
| 478 | |
| 479 | |
| 480 | @pytest.mark.asyncio |
| 481 | async def test_completion_render_default_no_token_offsets(client): |
| 482 | """Without the flag, token_offsets must be null (existing responses |
| 483 | unchanged).""" |
| 484 | response = await client.post( |
| 485 | "/v1/completions/render", |
| 486 | json={ |
| 487 | "model": MODEL_NAME, |
| 488 | "prompt": "Hello, world.", |
| 489 | }, |
| 490 | ) |
| 491 | |
| 492 | assert response.status_code == 200 |
| 493 | data = response.json() |
| 494 | assert data[0]["token_offsets"] is None |
| 495 | |
| 496 | |
| 497 | @pytest.mark.asyncio |
| 498 | async def test_chat_render_emits_token_offsets(client): |
| 499 | """With return_token_offsets, /v1/chat/completions/render returns |
| 500 | per-token offsets relative to the templated prompt string.""" |
| 501 | response = await client.post( |
| 502 | "/v1/chat/completions/render", |
| 503 | json={ |
| 504 | "model": MODEL_NAME, |
| 505 | "messages": [{"role": "user", "content": "Hello, world."}], |
| 506 | "return_token_offsets": True, |
| 507 | }, |
| 508 | ) |
| 509 | |
| 510 | assert response.status_code == 200 |
| 511 | data = response.json() |
| 512 | assert isinstance(data, dict) |
| 513 | offsets = data["token_offsets"] |
| 514 | assert offsets is not None |
| 515 | assert len(offsets) == len(data["token_ids"]) |
| 516 | for start, end in offsets: |
| 517 | assert isinstance(start, int) and isinstance(end, int) |
| 518 | assert 0 <= start <= end |
| 519 | |
| 520 | |
| 521 | @pytest.mark.asyncio |
| 522 | async def test_chat_render_default_no_token_offsets(client): |
| 523 | """Without the flag, chat render token_offsets must be null.""" |
| 524 | response = await client.post( |
| 525 | "/v1/chat/completions/render", |
| 526 | json={ |
| 527 | "model": MODEL_NAME, |
| 528 | "messages": [{"role": "user", "content": "Hello, world."}], |
| 529 | }, |
| 530 | ) |
| 531 | |
| 532 | assert response.status_code == 200 |
| 533 | data = response.json() |
| 534 | assert data["token_offsets"] is None |
| 535 | |
| 536 | |
| 537 | @pytest.mark.asyncio |
| 538 | async def test_completion_render_truncated_token_offsets(client): |
| 539 | """Truncation must shorten token_offsets together with token_ids. |
| 540 | |
| 541 | An explicit truncation_side turns off tokenizer-level truncation, so the |
| 542 | tokenizer returns offsets for the whole prompt and they are reduced |
| 543 | afterwards -- separately from token_ids. GenerateRequest documents that the |
| 544 | two lists have equal length. |
| 545 | """ |
| 546 | prompt = "The quick brown fox jumps over the lazy dog." |
| 547 | keep = 4 |
| 548 | response = await client.post( |
| 549 | "/v1/completions/render", |
| 550 | json={ |
| 551 | "model": MODEL_NAME, |
| 552 | "prompt": prompt, |
| 553 | "return_token_offsets": True, |
| 554 | "truncate_prompt_tokens": keep, |
| 555 | "truncation_side": "left", |
| 556 | }, |
| 557 | ) |
| 558 | |
| 559 | assert response.status_code == 200 |
| 560 | data = response.json() |
| 561 | token_ids = data[0]["token_ids"] |
| 562 | offsets = data[0]["token_offsets"] |
| 563 | |
| 564 | assert len(token_ids) == keep |
| 565 | assert len(offsets) == len(token_ids) |
| 566 | |
| 567 | # Equal length is not enough: truncating from the left keeps the *last* |
| 568 | # tokens, so the surviving offsets must cover the end of the prompt. |
| 569 | assert offsets[-1][1] == len(prompt) |
| 570 | assert offsets[0][0] > 0 |
| 571 | |
| 572 | |
| 573 | @pytest.mark.asyncio |
| 574 | async def test_completion_render_multiple_prompts_token_offsets(client): |
| 575 | """Each prompt in a batch gets its own offsets aligned with its tokens.""" |
| 576 | prompts = ["Hello, world.", "Goodbye, world."] |
| 577 | response = await client.post( |
| 578 | "/v1/completions/render", |
| 579 | json={ |
| 580 | "model": MODEL_NAME, |
| 581 | "prompt": prompts, |
| 582 | "return_token_offsets": True, |
| 583 | }, |
| 584 | ) |
| 585 | |
| 586 | assert response.status_code == 200 |
| 587 | data = response.json() |
| 588 | assert len(data) == len(prompts) |
| 589 | for item, prompt in zip(data, prompts): |
| 590 | offsets = item["token_offsets"] |
| 591 | assert offsets is not None |
| 592 | assert len(offsets) == len(item["token_ids"]) |
| 593 | for start, end in offsets: |
| 594 | assert 0 <= start <= end <= len(prompt) |
| 595 | |
| 596 | |
| 597 | @pytest.mark.asyncio |
| 598 | async def test_messages_render_basic(client): |
| 599 | """Test basic Anthropic Messages render endpoint.""" |
| 600 | response = await client.post( |
| 601 | "/v1/messages/render", |
| 602 | json={ |
| 603 | "model": MODEL_NAME, |
| 604 | "max_tokens": 16, |
| 605 | "messages": [{"role": "user", "content": "Render this Anthropic message."}], |
| 606 | }, |
| 607 | ) |
| 608 | |
| 609 | assert response.status_code == 200 |
| 610 | data = response.json() |
| 611 | |
| 612 | # Single GenerateRequest, like chat render. |
| 613 | assert isinstance(data, dict) |
| 614 | assert "token_ids" in data |
| 615 | assert "sampling_params" in data |
| 616 | assert "model" in data |
| 617 | assert data["model"] == MODEL_NAME |
| 618 | |
| 619 | token_ids = data["token_ids"] |
| 620 | assert isinstance(token_ids, list) |
| 621 | assert len(token_ids) > 0 |
| 622 | assert all(isinstance(tid, int) for tid in token_ids) |
| 623 | assert token_ids[0] == 1 # BOS |
| 624 | |
| 625 | |
| 626 | @pytest.mark.asyncio |
| 627 | async def test_messages_render_system_and_multi_turn(client): |
| 628 | """System field + multi-turn messages render to a single prompt.""" |
| 629 | response = await client.post( |
| 630 | "/v1/messages/render", |
| 631 | json={ |
| 632 | "model": MODEL_NAME, |
| 633 | "max_tokens": 16, |
| 634 | "system": "You are a helpful assistant.", |
| 635 | "messages": [ |
| 636 | {"role": "user", "content": "Hello"}, |
| 637 | {"role": "assistant", "content": "Hi! How can I help?"}, |
| 638 | {"role": "user", "content": "What is 2 + 2?"}, |
| 639 | ], |
| 640 | }, |
| 641 | ) |
| 642 | |
| 643 | assert response.status_code == 200 |
| 644 | data = response.json() |
| 645 | assert isinstance(data, dict) |
| 646 | assert len(data["token_ids"]) > 0 |
| 647 | assert data["token_ids"][0] == 1 # BOS |
| 648 | |
| 649 | |
| 650 | @pytest.mark.asyncio |
| 651 | async def test_messages_render_merges_inline_system(client): |
| 652 | """Inline system messages merge into the leading system block. |
| 653 | |
| 654 | The model's chat template rejects non-leading system messages, so the |
| 655 | /v1/messages server path detects merge_inline_system=True and render must |
| 656 | produce the same tokens as the manually pre-merged request. |
| 657 | """ |
| 658 | inline = await client.post( |
| 659 | "/v1/messages/render", |
| 660 | json={ |
| 661 | "model": MODEL_NAME, |
| 662 | "max_tokens": 16, |
| 663 | "system": "You are a helpful assistant.", |
| 664 | "messages": [ |
| 665 | {"role": "user", "content": "Hello"}, |
| 666 | {"role": "assistant", "content": "Hi! How can I help?"}, |
| 667 | {"role": "system", "content": "Be brief."}, |
| 668 | {"role": "user", "content": "What is 2 + 2?"}, |
| 669 | ], |
| 670 | }, |
| 671 | ) |
| 672 | assert inline.status_code == 200 |
| 673 | |
| 674 | merged = await client.post( |
| 675 | "/v1/messages/render", |
| 676 | json={ |
| 677 | "model": MODEL_NAME, |
| 678 | "max_tokens": 16, |
| 679 | "system": "You are a helpful assistant.Be brief.", |
| 680 | "messages": [ |
| 681 | {"role": "user", "content": "Hello"}, |
| 682 | {"role": "assistant", "content": "Hi! How can I help?"}, |
| 683 | {"role": "user", "content": "What is 2 + 2?"}, |
| 684 | ], |
| 685 | }, |
| 686 | ) |
| 687 | assert merged.status_code == 200 |
| 688 | |
| 689 | assert inline.json()["token_ids"] == merged.json()["token_ids"] |
| 690 | |
| 691 | |
| 692 | @pytest.mark.asyncio |
| 693 | async def test_messages_render_error_invalid_model(client): |
| 694 | """Messages render with an invalid model returns an error.""" |
| 695 | response = await client.post( |
| 696 | "/v1/messages/render", |
| 697 | json={ |
| 698 | "model": "invalid-model-name", |
| 699 | "max_tokens": 16, |
| 700 | "messages": [{"role": "user", "content": "Hello"}], |
| 701 | }, |
| 702 | ) |
| 703 | |
| 704 | assert response.status_code == 404 |
| 705 | data = response.json() |
| 706 | assert "error" in data |