| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """End-to-end tests for the TranslateGemma extra-fields-on-content-parts pathway. |
| 4 | |
| 5 | TranslateGemma's bundled chat template reads ``source_lang_code`` and |
| 6 | ``target_lang_code`` off each content part to assemble the translation prompt. |
| 7 | This PR makes sure those extra fields survive request parsing and reach the |
| 8 | template; these tests verify the full pathway with the real model on both |
| 9 | the text and image content branches of the template. |
| 10 | """ |
| 11 | |
| 12 | import json |
| 13 | |
| 14 | import openai |
| 15 | import pytest |
| 16 | import pytest_asyncio |
| 17 | |
| 18 | from tests.utils import RemoteOpenAIServer |
| 19 | from vllm.assets.image import ImageAsset |
| 20 | from vllm.multimodal.utils import encode_image_url |
| 21 | |
| 22 | MODEL_NAME = "google/translategemma-4b-it" |
| 23 | |
| 24 | |
| 25 | @pytest.fixture(scope="module") |
| 26 | def server(): |
| 27 | args = [ |
| 28 | "--max-model-len", |
| 29 | "2048", |
| 30 | "--max-num-seqs", |
| 31 | "16", |
| 32 | "--enforce-eager", |
| 33 | "--chat-template-content-format", |
| 34 | "openai", |
| 35 | "--limit-mm-per-prompt", |
| 36 | json.dumps({"image": 1}), |
| 37 | ] |
| 38 | with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: |
| 39 | yield remote_server |
| 40 | |
| 41 | |
| 42 | @pytest_asyncio.fixture |
| 43 | async def client(server): |
| 44 | async with server.get_async_client() as async_client: |
| 45 | yield async_client |
| 46 | |
| 47 | |
| 48 | @pytest.fixture(scope="module") |
| 49 | def stop_sign_image_url(): |
| 50 | return encode_image_url(ImageAsset("stop_sign").pil_image) |
| 51 | |
| 52 | |
| 53 | @pytest.mark.asyncio |
| 54 | async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI): |
| 55 | """En -> es translation through TranslateGemma's bundled chat template's |
| 56 | ``text`` content branch, which depends on ``source_lang_code`` / |
| 57 | ``target_lang_code`` extra fields being preserved on the content part.""" |
| 58 | completion = await client.chat.completions.create( |
| 59 | model=MODEL_NAME, |
| 60 | messages=[ |
| 61 | { |
| 62 | "role": "user", |
| 63 | "content": [ |
| 64 | { |
| 65 | "type": "text", |
| 66 | "text": "The quick brown fox jumps over the lazy dog.", |
| 67 | "source_lang_code": "en", |
| 68 | "target_lang_code": "es", |
| 69 | } |
| 70 | ], |
| 71 | } |
| 72 | ], |
| 73 | max_tokens=64, |
| 74 | temperature=0.0, |
| 75 | ) |
| 76 | |
| 77 | content = completion.choices[0].message.content |
| 78 | assert content is not None and content.strip(), ( |
| 79 | "expected a non-empty Spanish translation" |
| 80 | ) |
| 81 | # If extra fields had been stripped, the bundled template would not |
| 82 | # produce a translation prompt and the output would not be Spanish. |
| 83 | spanish_markers = (" el ", " la ", " los ", " las ", " perro", " zorro") |
| 84 | lowered = f" {content.lower()} " |
| 85 | assert any(m in lowered for m in spanish_markers), ( |
| 86 | f"output does not look like Spanish: {content!r}" |
| 87 | ) |
| 88 | |
| 89 | |
| 90 | @pytest.mark.asyncio |
| 91 | async def test_translategemma_image_extra_lang_code_fields( |
| 92 | client: openai.AsyncOpenAI, |
| 93 | stop_sign_image_url: str, |
| 94 | ): |
| 95 | """En -> es OCR-translation through TranslateGemma's bundled chat |
| 96 | template's ``image`` content branch. Exercises the multimodal branch of |
| 97 | ``_collect_extra_fields`` in ``_parse_chat_message_content_part``: the |
| 98 | ``source_lang_code`` / ``target_lang_code`` extras must survive parsing |
| 99 | and end up alongside the ``{"type": "image"}`` placeholder so the |
| 100 | template's image branch (``content["type"] == 'image'``) renders the |
| 101 | right OCR-translation prompt.""" |
| 102 | completion = await client.chat.completions.create( |
| 103 | model=MODEL_NAME, |
| 104 | messages=[ |
| 105 | { |
| 106 | "role": "user", |
| 107 | "content": [ |
| 108 | { |
| 109 | "type": "image_url", |
| 110 | "image_url": {"url": stop_sign_image_url}, |
| 111 | "source_lang_code": "en", |
| 112 | "target_lang_code": "es", |
| 113 | } |
| 114 | ], |
| 115 | } |
| 116 | ], |
| 117 | max_tokens=64, |
| 118 | temperature=0.0, |
| 119 | ) |
| 120 | |
| 121 | content = completion.choices[0].message.content |
| 122 | assert content is not None and content.strip(), ( |
| 123 | "expected a non-empty translation of the stop-sign image" |
| 124 | ) |
| 125 | # Common Spanish renderings of "STOP" on a road sign. If extras were |
| 126 | # stripped we would either fail the template or get back the original |
| 127 | # English word, neither of which match this set. |
| 128 | spanish_stop_terms = ("detén", "deteng", "deten", "alto", "pare", "para") |
| 129 | lowered = content.lower() |
| 130 | assert "stop" not in lowered, ( |
| 131 | f"output appears to echo the English source: {content!r}" |
| 132 | ) |
| 133 | assert any(t in lowered for t in spanish_stop_terms), ( |
| 134 | f"output is not a recognizable Spanish translation of 'STOP': {content!r}" |
| 135 | ) |