@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d46934b229
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Frontend] Gemma3n audio `transcriptions`/`translations` endpoint (#23735)

Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Nicolò Lucchesi committed Sep 1, 2025 at 12:07 UTC d46934b2297eb3dbda24c3bf26f6655d88ba99bb
9 files changed +189 -63
tests/entrypoints/openai/conftest.py new
+27
@@ -0,0 +1,27 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import pytest
4 +
5 +from vllm.assets.audio import AudioAsset
6 +
7 +
8 +@pytest.fixture
9 +def mary_had_lamb():
10 + path = AudioAsset('mary_had_lamb').get_local_path()
11 + with open(str(path), "rb") as f:
12 + yield f
13 +
14 +
15 +@pytest.fixture
16 +def winning_call():
17 + path = AudioAsset('winning_call').get_local_path()
18 + with open(str(path), "rb") as f:
19 + yield f
20 +
21 +
22 +@pytest.fixture
23 +def foscolo():
24 + # Test translation it->en
25 + path = AudioAsset('azacinto_foscolo').get_local_path()
26 + with open(str(path), "rb") as f:
27 + yield f
tests/entrypoints/openai/test_transcription_validation.py
+19 -16
@@ -12,8 +12,6 @@ import pytest
12 import pytest_asyncio
13 import soundfile as sf
14
15 -from vllm.assets.audio import AudioAsset
16 -
15 from ...utils import RemoteOpenAIServer
16
17 MODEL_NAME = "openai/whisper-large-v3-turbo"
@@ -24,20 +22,6 @@ MISTRAL_FORMAT_ARGS = [
22 ]
23
24
27 -@pytest.fixture
28 -def mary_had_lamb():
29 - path = AudioAsset('mary_had_lamb').get_local_path()
30 - with open(str(path), "rb") as f:
31 - yield f
32 -
33 -
34 -@pytest.fixture
35 -def winning_call():
36 - path = AudioAsset('winning_call').get_local_path()
37 - with open(str(path), "rb") as f:
38 - yield f
39 -
40 -
25 @pytest.fixture(scope="module")
26 def server():
27 with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as remote_server:
@@ -76,6 +60,25 @@ async def test_basic_audio(mary_had_lamb, model_name):
60 assert out_usage["seconds"] == 16, out_usage["seconds"]
61
62
63 +@pytest.mark.asyncio
64 +async def test_basic_audio_gemma(foscolo):
65 + # Gemma accuracy on some of the audio samples we use is particularly bad,
66 + # hence we use a different one here. WER is evaluated separately.
67 + model_name = "google/gemma-3n-E2B-it"
68 + server_args = ["--enforce-eager"]
69 +
70 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
71 + client = remote_server.get_async_client()
72 + transcription = await client.audio.transcriptions.create(
73 + model=model_name,
74 + file=foscolo,
75 + language="it",
76 + response_format="text",
77 + temperature=0.0)
78 + out = json.loads(transcription)['text']
79 + assert "da cui vergine nacque Venere" in out
80 +
81 +
82 @pytest.mark.asyncio
83 async def test_non_asr_model(winning_call):
84 # text to text model
tests/entrypoints/openai/test_translation_validation.py
+44 -34
@@ -12,32 +12,24 @@ import pytest
12 import pytest_asyncio
13 import soundfile as sf
14
15 -from vllm.assets.audio import AudioAsset
16 -
15 from ...utils import RemoteOpenAIServer
16
19 -MODEL_NAME = "openai/whisper-small"
17 SERVER_ARGS = ["--enforce-eager"]
18
19
23 -@pytest.fixture
24 -def foscolo():
25 - # Test translation it->en
26 - path = AudioAsset('azacinto_foscolo').get_local_path()
27 - with open(str(path), "rb") as f:
28 - yield f
29 -
30 -
31 -@pytest.fixture(scope="module")
32 -def server():
33 - with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as remote_server:
34 - yield remote_server
20 +@pytest.fixture(scope="module",
21 + params=["openai/whisper-small", "google/gemma-3n-E2B-it"])
22 +def server(request):
23 + # Parametrize over model name
24 + with RemoteOpenAIServer(request.param, SERVER_ARGS) as remote_server:
25 + yield remote_server, request.param
26
27
28 @pytest_asyncio.fixture
38 -async def client(server):
29 +async def client_and_model(server):
30 + server, model_name = server
31 async with server.get_async_client() as async_client:
40 - yield async_client
32 + yield async_client, model_name
33
34
35 @pytest.mark.asyncio
@@ -56,27 +48,29 @@ async def test_non_asr_model(foscolo):
48
49 # NOTE: (NickLucche) the large-v3-turbo model was not trained on translation!
50 @pytest.mark.asyncio
59 -async def test_basic_audio(foscolo, client):
51 +async def test_basic_audio(foscolo, client_and_model):
52 + client, model_name = client_and_model
53 translation = await client.audio.translations.create(
61 - model=MODEL_NAME,
54 + model=model_name,
55 file=foscolo,
56 response_format="text",
64 - # TODO remove once language detection is implemented
65 - extra_body=dict(language="it"),
57 + # TODO remove `language="it"` once language detection is implemented
58 + extra_body=dict(language="it", to_language="en"),
59 temperature=0.0)
60 out = json.loads(translation)['text'].strip().lower()
61 assert "greek sea" in out
62
63
64 @pytest.mark.asyncio
72 -async def test_audio_prompt(foscolo, client):
65 +async def test_audio_prompt(foscolo, client_and_model):
66 + client, model_name = client_and_model
67 # Condition whisper on starting text
68 prompt = "Nor have I ever"
69 transcription = await client.audio.translations.create(
76 - model=MODEL_NAME,
70 + model=model_name,
71 file=foscolo,
72 prompt=prompt,
79 - extra_body=dict(language="it"),
73 + extra_body=dict(language="it", to_language="en"),
74 response_format="text",
75 temperature=0.0)
76 out = json.loads(transcription)['text']
@@ -85,22 +79,27 @@ async def test_audio_prompt(foscolo, client):
79
80
81 @pytest.mark.asyncio
88 -async def test_streaming_response(foscolo, client, server):
82 +async def test_streaming_response(foscolo, client_and_model, server):
83 + client, model_name = client_and_model
84 translation = ""
85 res_no_stream = await client.audio.translations.create(
91 - model=MODEL_NAME,
86 + model=model_name,
87 file=foscolo,
88 response_format="json",
94 - extra_body=dict(language="it"),
89 + extra_body=dict(language="it", to_language="en", seed=42),
90 temperature=0.0)
91 +
92 # Stream via HTTPX since OpenAI translation client doesn't expose streaming
93 + server, model_name = server
94 url = server.url_for("v1/audio/translations")
95 headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
96 data = {
100 - "model": MODEL_NAME,
97 + "model": model_name,
98 "language": "it",
99 + "to_language": "en",
100 "stream": True,
101 "temperature": 0.0,
102 + "seed": 42,
103 }
104 foscolo.seek(0)
105 async with httpx.AsyncClient() as http_client:
@@ -121,16 +120,24 @@ async def test_streaming_response(foscolo, client, server):
120 text = chunk["choices"][0].get("delta", {}).get("content")
121 translation += text or ""
122
124 - assert translation == res_no_stream.text
123 + res_stream = translation.split()
124 + # NOTE There's a small non-deterministic issue here, likely in the attn
125 + # computation, which will cause a few tokens to be different, while still
126 + # being very close semantically.
127 + assert sum([
128 + x == y for x, y in zip(res_stream, res_no_stream.text.split())
129 + ]) >= len(res_stream) * 0.9
130
131
132 @pytest.mark.asyncio
128 -async def test_stream_options(foscolo, client, server):
133 +async def test_stream_options(foscolo, server):
134 + server, model_name = server
135 url = server.url_for("v1/audio/translations")
136 headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
137 data = {
132 - "model": MODEL_NAME,
138 + "model": model_name,
139 "language": "it",
140 + "to_language": "en",
141 "stream": True,
142 "stream_include_usage": True,
143 "stream_continuous_usage_stats": True,
@@ -164,7 +171,10 @@ async def test_stream_options(foscolo, client, server):
171
172
173 @pytest.mark.asyncio
167 -async def test_long_audio_request(foscolo, client):
174 +async def test_long_audio_request(foscolo, client_and_model):
175 + client, model_name = client_and_model
176 + if model_name == "google/gemma-3n-E2B-it":
177 + pytest.skip("Gemma3n does not support long audio requests")
178 foscolo.seek(0)
179 audio, sr = librosa.load(foscolo)
180 repeated_audio = np.tile(audio, 2)
@@ -173,9 +183,9 @@ async def test_long_audio_request(foscolo, client):
183 sf.write(buffer, repeated_audio, sr, format='WAV')
184 buffer.seek(0)
185 translation = await client.audio.translations.create(
176 - model=MODEL_NAME,
186 + model=model_name,
187 file=buffer,
178 - extra_body=dict(language="it"),
188 + extra_body=dict(language="it", to_language="en"),
189 response_format="text",
190 temperature=0.0)
191 out = json.loads(translation)['text'].strip().lower()
vllm/entrypoints/openai/protocol.py
+19
@@ -2175,6 +2175,13 @@ class TranscriptionRequest(OpenAIBaseModel):
2175 )
2176 # --8<-- [end:transcription-extra-params]
2177
2178 + to_language: Optional[str] = None
2179 + """The language of the output audio we transcribe to.
2180 +
2181 + Please note that this is not currently used by supported models at this
2182 + time, but it is a placeholder for future use, matching translation api.
2183 + """
2184 +
2185 # --8<-- [start:transcription-sampling-params]
2186 temperature: float = Field(default=0.0)
2187 """The sampling temperature, between 0 and 1.
@@ -2408,6 +2415,9 @@ class TranslationRequest(OpenAIBaseModel):
2415
2416 # TODO support additional sampling parameters
2417 # --8<-- [start:translation-sampling-params]
2418 + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2419 + """The seed to use for sampling."""
2420 +
2421 temperature: float = Field(default=0.0)
2422 """The sampling temperature, between 0 and 1.
2423
@@ -2427,6 +2437,14 @@ class TranslationRequest(OpenAIBaseModel):
2437 will improve accuracy.
2438 """
2439
2440 + to_language: Optional[str] = None
2441 + """The language of the input audio we translate to.
2442 +
2443 + Please note that this is not supported by all models, refer to the specific
2444 + model documentation for more details.
2445 + For instance, Whisper only supports `to_language=en`.
2446 + """
2447 +
2448 stream: Optional[bool] = False
2449 """Custom field not present in the original OpenAI definition. When set,
2450 it will enable output to be streamed in a similar fashion as the Chat
@@ -2458,6 +2476,7 @@ class TranslationRequest(OpenAIBaseModel):
2476
2477 return SamplingParams.from_optional(temperature=temperature,
2478 max_tokens=max_tokens,
2479 + seed=self.seed,
2480 output_kind=RequestOutputKind.DELTA
2481 if self.stream \
2482 else RequestOutputKind.FINAL_ONLY)
vllm/entrypoints/openai/speech_to_text.py
+6 -1
@@ -89,6 +89,9 @@ class OpenAISpeechToText(OpenAIServing):
89 ) -> tuple[list[PromptType], float]:
90 # Validate request
91 language = self.model_cls.validate_language(request.language)
92 + # Skip to_language validation to avoid extra logging for Whisper.
93 + to_language = self.model_cls.validate_language(request.to_language) \
94 + if request.to_language else None
95
96 if len(audio_data) / 1024**2 > self.max_audio_filesize_mb:
97 raise ValueError("Maximum file size exceeded.")
@@ -112,7 +115,9 @@ class OpenAISpeechToText(OpenAIServing):
115 model_config=self.model_config,
116 language=language,
117 task_type=self.task_type,
115 - request_prompt=request.prompt)
118 + request_prompt=request.prompt,
119 + to_language=to_language,
120 + )
121 prompts.append(prompt)
122 return prompts, duration
123
vllm/model_executor/models/gemma3n_mm.py
+61 -4
@@ -1,8 +1,9 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from collections.abc import Iterable, Mapping, Sequence
4 -from typing import Any, Optional, TypedDict, Union, cast
4 +from typing import Any, Literal, Optional, TypedDict, Union, cast
5
6 +import numpy as np
7 import torch
8 from torch import nn
9 from transformers import AutoModel, BatchFeature
@@ -13,7 +14,8 @@ from transformers.models.gemma3n import (Gemma3nAudioConfig,
14 Gemma3nVisionConfig)
15 from transformers.models.siglip import SiglipImageProcessorFast
16
16 -from vllm.config import VllmConfig
17 +from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
18 +from vllm.inputs.data import PromptType
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.layernorm import RMSNorm
21 from vllm.model_executor.layers.linear import RowParallelLinear
@@ -21,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23 VocabParallelEmbedding)
24 from vllm.model_executor.models.gemma3n import Gemma3nForCausalLM
25 from vllm.model_executor.models.module_mapping import MultiModelKeys
26 +from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.multimodal import MULTIMODAL_REGISTRY
29 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
@@ -40,7 +43,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 from vllm.multimodal.profiling import BaseDummyInputsBuilder
44 from vllm.sequence import IntermediateTensors
45
43 -from .interfaces import MultiModalEmbeddings, SupportsMultiModal
46 +from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,
47 + SupportsTranscription)
48 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
49 init_vllm_registered_model, maybe_prefix,
50 merge_multimodal_embeddings)
@@ -410,7 +414,10 @@ class Gemma3nMultimodalEmbedder(nn.Module):
414 @MULTIMODAL_REGISTRY.register_processor(Gemma3nMultiModalProcessor,
415 info=Gemma3nProcessingInfo,
416 dummy_inputs=Gemma3nDummyInputsBuilder)
413 -class Gemma3nForConditionalGeneration(nn.Module, SupportsMultiModal):
417 +class Gemma3nForConditionalGeneration(nn.Module, SupportsMultiModal,
418 + SupportsTranscription):
419 + supported_languages = ISO639_1_SUPPORTED_LANGS
420 +
421 packed_modules_mapping = {
422 "qkv_proj": [
423 "q_proj",
@@ -694,3 +701,53 @@ class Gemma3nForConditionalGeneration(nn.Module, SupportsMultiModal):
701 return "<audio_soft_token>"
702 else:
703 raise ValueError(f"Unsupported modality: {modality}")
704 +
705 + @classmethod
706 + def get_generation_prompt(cls, audio: np.ndarray,
707 + stt_config: SpeechToTextConfig,
708 + model_config: ModelConfig,
709 + language: Optional[str],
710 + task_type: Literal["transcribe", "translate"],
711 + request_prompt: str,
712 + to_language: Optional[str]) -> PromptType:
713 + """
714 + Gemma3n supports "free-form" transcription.
715 + We fix its prompt here to standardize transcriptions/translations
716 + requests.
717 + """
718 + # Transcribe this audio [into <>] | for transcription
719 + # Translate this audio [from <> into <>] | for translation
720 + prompt = "<start_of_turn>user\n"
721 + prompt += "Transcribe" if task_type == "transcribe" else "Translate"
722 + prompt += " this audio"
723 +
724 + # We assume the language is a valid ISO 639-1 code.
725 + full_lang_name = cls.supported_languages.get(language, "")
726 + # Translation only for now
727 + full_lang_name_to = cls.supported_languages.get(to_language, "")
728 +
729 + if task_type == "transcribe" and full_lang_name:
730 + prompt += f" into {full_lang_name}"
731 + elif task_type == "translate":
732 + if full_lang_name:
733 + prompt += f" from {full_lang_name}"
734 + if full_lang_name_to:
735 + prompt += f" into {full_lang_name_to}"
736 +
737 + prompt += ": <audio_soft_token><end_of_turn>\n<start_of_turn>model\n"
738 +
739 + audio = (audio, stt_config.sample_rate)
740 + prompts_dict = {"multi_modal_data": {"audio": audio}, "prompt": prompt}
741 + return cast(PromptType, prompts_dict)
742 +
743 + @classmethod
744 + def get_speech_to_text_config(cls, model_config: ModelConfig,
745 + task_type: str) -> SpeechToTextConfig:
746 + return SpeechToTextConfig(
747 + # Let's set this to 30 as suggested in the docs for now, although
748 + # the model is only limited by its context length.
749 + max_audio_clip_s=30,
750 + sample_rate=16000,
751 + # TODO enable chunking after more thorough testing.
752 + min_energy_split_window_size=None,
753 + )
vllm/model_executor/models/interfaces.py
+4 -2
@@ -700,8 +700,10 @@ class SupportsTranscription(Protocol):
700 def get_generation_prompt(cls, audio: np.ndarray,
701 stt_config: SpeechToTextConfig,
702 model_config: ModelConfig,
703 - language: Optional[str], task_type: str,
704 - request_prompt: str) -> PromptType:
703 + language: Optional[str],
704 + task_type: Literal["transcribe", "translate"],
705 + request_prompt: str,
706 + to_language: Optional[str]) -> PromptType:
707 """Get the prompt for the ASR model.
708 The model has control over the construction, as long as it
709 returns a valid PromptType."""
vllm/model_executor/models/voxtral.py
+5 -3
@@ -5,7 +5,7 @@ import math
5 from collections.abc import Iterable, Mapping, Sequence
6 from functools import cached_property
7 from math import ceil
8 -from typing import Optional, Union, cast
8 +from typing import Literal, Optional, Union, cast
9
10 import numpy as np
11 import regex as re
@@ -455,8 +455,10 @@ class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,
455 def get_generation_prompt(cls, audio: np.ndarray,
456 model_config: ModelConfig,
457 stt_config: SpeechToTextConfig,
458 - language: Optional[str], task_type: str,
459 - request_prompt: str) -> PromptType:
458 + language: Optional[str],
459 + task_type: Literal["transcribe", "translate"],
460 + request_prompt: str,
461 + to_language: Optional[str]) -> PromptType:
462 tokenizer = cached_tokenizer_from_config(model_config)
463 audio = Audio(audio, int(stt_config.sample_rate),
464 format="wav") # lossless
vllm/model_executor/models/whisper.py
+4 -3
@@ -4,7 +4,7 @@
4 import math
5 from collections.abc import Iterable, Mapping, Sequence
6 from contextlib import nullcontext
7 -from typing import Optional, TypedDict, Union, cast
7 +from typing import Literal, Optional, TypedDict, Union, cast
8
9 import numpy as np
10 import torch
@@ -783,8 +783,9 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
783 model_config: ModelConfig, # not needed here
784 stt_config: SpeechToTextConfig,
785 language: Optional[str],
786 - task_type: str,
787 - request_prompt: str) -> PromptType:
786 + task_type: Literal["transcribe", "translate"],
787 + request_prompt: str,
788 + to_language: Optional[str]) -> PromptType:
789 if language is None:
790 raise ValueError(
791 "Language must be specified when creating the Whisper prompt")