[Model] Support TranslateGemma-12b-it (#41599)
Signed-off-by: Zhang Jian <jianmusings@gmail.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Zhang Jian committed
Jul 17, 2026 at 15:17 UTC
26c909ed74a6298952d0c3191fbfdf2b513d9e1d
2 files changed
+174
-3
tests/entrypoints/openai/chat_completion/test_extra_content_fields.py
new
+135
@@ -0,0 +1,135 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""End-to-end tests for the TranslateGemma extra-fields-on-content-parts pathway.
4
+
5
+TranslateGemma's bundled chat template reads ``source_lang_code`` and
6
+``target_lang_code`` off each content part to assemble the translation prompt.
7
+This PR makes sure those extra fields survive request parsing and reach the
8
+template; these tests verify the full pathway with the real model on both
9
+the text and image content branches of the template.
10
+"""
11
+
12
+import json
13
+
14
+import openai
15
+import pytest
16
+import pytest_asyncio
17
+
18
+from tests.utils import RemoteOpenAIServer
19
+from vllm.assets.image import ImageAsset
20
+from vllm.multimodal.utils import encode_image_url
21
+
22
+MODEL_NAME = "google/translategemma-4b-it"
23
+
24
+
25
+@pytest.fixture(scope="module")
26
+def server():
27
+ args = [
28
+ "--max-model-len",
29
+ "2048",
30
+ "--max-num-seqs",
31
+ "16",
32
+ "--enforce-eager",
33
+ "--chat-template-content-format",
34
+ "openai",
35
+ "--limit-mm-per-prompt",
36
+ json.dumps({"image": 1}),
37
+ ]
38
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
39
+ yield remote_server
40
+
41
+
42
+@pytest_asyncio.fixture
43
+async def client(server):
44
+ async with server.get_async_client() as async_client:
45
+ yield async_client
46
+
47
+
48
+@pytest.fixture(scope="module")
49
+def stop_sign_image_url():
50
+ return encode_image_url(ImageAsset("stop_sign").pil_image)
51
+
52
+
53
+@pytest.mark.asyncio
54
+async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI):
55
+ """en -> es translation through TranslateGemma's bundled chat template's
56
+ ``text`` content branch, which depends on ``source_lang_code`` /
57
+ ``target_lang_code`` extra fields being preserved on the content part."""
58
+ completion = await client.chat.completions.create(
59
+ model=MODEL_NAME,
60
+ messages=[
61
+ {
62
+ "role": "user",
63
+ "content": [
64
+ {
65
+ "type": "text",
66
+ "text": "The quick brown fox jumps over the lazy dog.",
67
+ "source_lang_code": "en",
68
+ "target_lang_code": "es",
69
+ }
70
+ ],
71
+ }
72
+ ],
73
+ max_tokens=64,
74
+ temperature=0.0,
75
+ )
76
+
77
+ content = completion.choices[0].message.content
78
+ assert content is not None and content.strip(), (
79
+ "expected a non-empty Spanish translation"
80
+ )
81
+ # If extra fields had been stripped, the bundled template would not
82
+ # produce a translation prompt and the output would not be Spanish.
83
+ spanish_markers = (" el ", " la ", " los ", " las ", " perro", " zorro")
84
+ lowered = f" {content.lower()} "
85
+ assert any(m in lowered for m in spanish_markers), (
86
+ f"output does not look like Spanish: {content!r}"
87
+ )
88
+
89
+
90
+@pytest.mark.asyncio
91
+async def test_translategemma_image_extra_lang_code_fields(
92
+ client: openai.AsyncOpenAI,
93
+ stop_sign_image_url: str,
94
+):
95
+ """en -> es OCR-translation through TranslateGemma's bundled chat
96
+ template's ``image`` content branch. Exercises the multimodal branch of
97
+ ``_collect_extra_fields`` in ``_parse_chat_message_content_part``: the
98
+ ``source_lang_code`` / ``target_lang_code`` extras must survive parsing
99
+ and end up alongside the ``{"type": "image"}`` placeholder so the
100
+ template's image branch (``content["type"] == 'image'``) renders the
101
+ right OCR-translation prompt."""
102
+ completion = await client.chat.completions.create(
103
+ model=MODEL_NAME,
104
+ messages=[
105
+ {
106
+ "role": "user",
107
+ "content": [
108
+ {
109
+ "type": "image_url",
110
+ "image_url": {"url": stop_sign_image_url},
111
+ "source_lang_code": "en",
112
+ "target_lang_code": "es",
113
+ }
114
+ ],
115
+ }
116
+ ],
117
+ max_tokens=64,
118
+ temperature=0.0,
119
+ )
120
+
121
+ content = completion.choices[0].message.content
122
+ assert content is not None and content.strip(), (
123
+ "expected a non-empty translation of the stop-sign image"
124
+ )
125
+ # Common Spanish renderings of "STOP" on a road sign. If extras were
126
+ # stripped we would either fail the template or get back the original
127
+ # English word, neither of which match this set.
128
+ spanish_stop_terms = ("detén", "deteng", "deten", "alto", "pare", "para")
129
+ lowered = content.lower()
130
+ assert "stop" not in lowered, (
131
+ f"output appears to echo the English source: {content!r}"
132
+ )
133
+ assert any(t in lowered for t in spanish_stop_terms), (
134
+ f"output is not a recognizable Spanish translation of 'STOP': {content!r}"
135
+ )
vllm/entrypoints/chat_utils.py
+39
-3
@@ -3,6 +3,7 @@
3
4
import asyncio
5
import json
6
+import types
7
from abc import ABC, abstractmethod
8
from collections import Counter, defaultdict
9
from collections.abc import Awaitable, Callable, Iterable
@@ -10,7 +11,19 @@ from dataclasses import dataclass
11
from functools import cached_property, lru_cache, partial
12
from itertools import accumulate
13
from pathlib import Path
13
-from typing import TYPE_CHECKING, Any, Final, Generic, Literal, TypeAlias, TypeVar, cast
14
+from typing import (
15
+ TYPE_CHECKING,
16
+ Any,
17
+ Final,
18
+ Generic,
19
+ Literal,
20
+ TypeAlias,
21
+ TypeVar,
22
+ Union,
23
+ cast,
24
+ get_args,
25
+ get_origin,
26
+)
27
28
from openai.types.chat import (
29
ChatCompletionAssistantMessageParam,
@@ -1460,6 +1473,25 @@ MM_PARSER_MAP: dict[
1473
}
1474
1475
1476
+def _collect_known_content_part_fields() -> frozenset[str]:
1477
+ fields: set[str] = set()
1478
+ stack: list[Any] = [ChatCompletionContentPartParam]
1479
+ while stack:
1480
+ node = stack.pop()
1481
+ if get_origin(node) in (Union, types.UnionType):
1482
+ stack.extend(get_args(node))
1483
+ elif hasattr(node, "__required_keys__"):
1484
+ fields |= node.__required_keys__ | node.__optional_keys__
1485
+ return frozenset(fields)
1486
+
1487
+
1488
+_KNOWN_CONTENT_PART_FIELDS = _collect_known_content_part_fields()
1489
+
1490
+
1491
+def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]:
1492
+ return {k: v for k, v in part.items() if k not in _KNOWN_CONTENT_PART_FIELDS}
1493
+
1494
+
1495
def _parse_chat_message_content_mm_part(
1496
part: ChatCompletionContentPartParam,
1497
) -> tuple[str, _ContentPart]:
@@ -1669,7 +1701,9 @@ def _parse_chat_message_content_part(
1701
str_content = cast(str, content)
1702
_reject_reserved_placeholder_in_text(str_content, mm_parser.model_config)
1703
if wrap_dicts:
1672
- return {"type": "text", "text": str_content}
1704
+ result: dict[str, Any] = {"type": "text", "text": str_content}
1705
+ result.update(_collect_extra_fields(cast(dict[str, Any], part)))
1706
+ return result
1707
else:
1708
return str_content
1709
@@ -1734,7 +1768,9 @@ def _parse_chat_message_content_part(
1768
# emit the single sentinel token as text so the template renders
1769
# it inline. The renderer later expands it to N tokens post-tokenize.
1770
return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN}
1737
- return {"type": modality}
1771
+ result = {"type": modality}
1772
+ result.update(_collect_extra_fields(cast(dict[str, Any], part)))
1773
+ return result
1774
if modality == "prompt_embeds":
1775
# Emit the renderer token inline regardless of `interleave_strings`,
1776
# prompt_embeds are spliced at the token offset so position matters.