@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e795d723ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 04:45 UTC

[Frontend] Add `/v1/audio/translations` OpenAI API endpoint (#19615)

Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Nicolò Lucchesi committed Jun 25, 2025 at 19:54 UTC e795d723ed8b03135978399c41a93437951176ed
10 files changed +1127 -461
docs/serving/openai_compatible_server.md
+30
@@ -57,6 +57,8 @@ We currently support the following OpenAI APIs:
57 - Only applicable to [embedding models](../models/pooling_models.md) (`--task embed`).
58 - [Transcriptions API][transcriptions-api] (`/v1/audio/transcriptions`)
59 - Only applicable to Automatic Speech Recognition (ASR) models (OpenAI Whisper) (`--task generate`).
60 +- [Translation API][translations-api] (`/v1/audio/translations`)
61 + - Only applicable to Automatic Speech Recognition (ASR) models (OpenAI Whisper) (`--task generate`).
62
63 In addition, we have the following custom APIs:
64
@@ -374,6 +376,34 @@ The following extra parameters are supported:
376 ```python
377 --8<-- "vllm/entrypoints/openai/protocol.py:transcription-extra-params"
378 ```
379 +
380 +[](){ #translations-api }
381 +
382 +### Translations API
383 +
384 +Our Translation API is compatible with [OpenAI's Translations API](https://platform.openai.com/docs/api-reference/audio/createTranslation);
385 +you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
386 +Whisper models can translate audio from one of the 55 non-English supported languages into English.
387 +Please mind that the popular `openai/whisper-large-v3-turbo` model does not support translating.
388 +
389 +!!! note
390 + To use the Translation API, please install with extra audio dependencies using `pip install vllm[audio]`.
391 +
392 +Code example: <gh-file:examples/online_serving/openai_translation_client.py>
393 +
394 +#### Extra Parameters
395 +
396 +The following [sampling parameters][sampling-params] are supported.
397 +
398 +```python
399 +--8<-- "vllm/entrypoints/openai/protocol.py:translation-sampling-params"
400 +```
401 +
402 +The following extra parameters are supported:
403 +
404 +```python
405 +--8<-- "vllm/entrypoints/openai/protocol.py:translation-extra-params"
406 +```
407
408 [](){ #tokenizer-api }
409
examples/online_serving/openai_transcription_client.py
+20 -21
@@ -26,23 +26,12 @@ from openai import OpenAI
26
27 from vllm.assets.audio import AudioAsset
28
29 -mary_had_lamb = AudioAsset("mary_had_lamb").get_local_path()
30 -winning_call = AudioAsset("winning_call").get_local_path()
29
32 -# Modify OpenAI's API key and API base to use vLLM's API server.
33 -openai_api_key = "EMPTY"
34 -openai_api_base = "http://localhost:8000/v1"
35 -client = OpenAI(
36 - api_key=openai_api_key,
37 - base_url=openai_api_base,
38 -)
39 -
40 -
41 -def sync_openai():
30 +def sync_openai(audio_path: str, client: OpenAI):
31 """
32 Perform synchronous transcription using OpenAI-compatible API.
33 """
45 - with open(str(mary_had_lamb), "rb") as f:
34 + with open(audio_path, "rb") as f:
35 transcription = client.audio.transcriptions.create(
36 file=f,
37 model="openai/whisper-large-v3",
@@ -58,8 +47,7 @@ def sync_openai():
47 print("transcription result:", transcription.text)
48
49
61 -# OpenAI Transcription API client does not support streaming.
62 -async def stream_openai_response():
50 +async def stream_openai_response(audio_path: str, base_url: str, api_key: str):
51 """
52 Perform streaming transcription using vLLM's raw HTTP streaming API.
53 """
@@ -68,11 +56,12 @@ async def stream_openai_response():
56 "stream": True,
57 "model": "openai/whisper-large-v3",
58 }
71 - url = openai_api_base + "/audio/transcriptions"
72 - headers = {"Authorization": f"Bearer {openai_api_key}"}
59 + url = base_url + "/audio/transcriptions"
60 + headers = {"Authorization": f"Bearer {api_key}"}
61 print("transcription result:", end=" ")
62 + # OpenAI Transcription API client does not support streaming.
63 async with httpx.AsyncClient() as client:
75 - with open(str(winning_call), "rb") as f:
64 + with open(audio_path, "rb") as f:
65 async with client.stream(
66 "POST", url, files={"file": f}, data=data, headers=headers
67 ) as response:
@@ -93,10 +82,20 @@ async def stream_openai_response():
82
83
84 def main():
96 - sync_openai()
97 -
85 + mary_had_lamb = str(AudioAsset("mary_had_lamb").get_local_path())
86 + winning_call = str(AudioAsset("winning_call").get_local_path())
87 +
88 + # Modify OpenAI's API key and API base to use vLLM's API server.
89 + openai_api_key = "EMPTY"
90 + openai_api_base = "http://localhost:8000/v1"
91 + client = OpenAI(
92 + api_key=openai_api_key,
93 + base_url=openai_api_base,
94 + )
95 +
96 + sync_openai(mary_had_lamb, client)
97 # Run the asynchronous function
99 - asyncio.run(stream_openai_response())
98 + asyncio.run(stream_openai_response(winning_call, openai_api_base, openai_api_key))
99
100
101 if __name__ == "__main__":
examples/online_serving/openai_translation_client.py new
+75
@@ -0,0 +1,75 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import asyncio
4 +import json
5 +
6 +import httpx
7 +from openai import OpenAI
8 +
9 +from vllm.assets.audio import AudioAsset
10 +
11 +
12 +def sync_openai(audio_path: str, client: OpenAI):
13 + with open(audio_path, "rb") as f:
14 + translation = client.audio.translations.create(
15 + file=f,
16 + model="openai/whisper-large-v3",
17 + response_format="json",
18 + temperature=0.0,
19 + # Additional params not provided by OpenAI API.
20 + extra_body=dict(
21 + language="it",
22 + seed=4419,
23 + repetition_penalty=1.3,
24 + ),
25 + )
26 + print("translation result:", translation.text)
27 +
28 +
29 +async def stream_openai_response(audio_path: str, base_url: str, api_key: str):
30 + data = {
31 + "language": "it",
32 + "stream": True,
33 + "model": "openai/whisper-large-v3",
34 + }
35 + url = base_url + "/audio/translations"
36 + headers = {"Authorization": f"Bearer {api_key}"}
37 + print("translation result:", end=" ")
38 + # OpenAI translation API client does not support streaming.
39 + async with httpx.AsyncClient() as client:
40 + with open(audio_path, "rb") as f:
41 + async with client.stream(
42 + "POST", url, files={"file": f}, data=data, headers=headers
43 + ) as response:
44 + async for line in response.aiter_lines():
45 + # Each line is a JSON object prefixed with 'data: '
46 + if line:
47 + if line.startswith("data: "):
48 + line = line[len("data: ") :]
49 + # Last chunk, stream ends
50 + if line.strip() == "[DONE]":
51 + break
52 + # Parse the JSON response
53 + chunk = json.loads(line)
54 + # Extract and print the content
55 + content = chunk["choices"][0].get("delta", {}).get("content")
56 + print(content, end="")
57 +
58 +
59 +def main():
60 + foscolo = str(AudioAsset("azacinto_foscolo").get_local_path())
61 +
62 + # Modify OpenAI's API key and API base to use vLLM's API server.
63 + openai_api_key = "EMPTY"
64 + openai_api_base = "http://localhost:8000/v1"
65 + client = OpenAI(
66 + api_key=openai_api_key,
67 + base_url=openai_api_base,
68 + )
69 + sync_openai(foscolo, client)
70 + # Run the asynchronous function
71 + asyncio.run(stream_openai_response(foscolo, openai_api_base, openai_api_key))
72 +
73 +
74 +if __name__ == "__main__":
75 + main()
tests/entrypoints/openai/test_transcription_validation.py
+2
@@ -82,6 +82,8 @@ async def test_long_audio_request(mary_had_lamb):
82
83 mary_had_lamb.seek(0)
84 audio, sr = librosa.load(mary_had_lamb)
85 + # Add small silence after each audio for repeatability in the split process
86 + audio = np.pad(audio, (0, 1600))
87 repeated_audio = np.tile(audio, 10)
88 # Repeated audio to buffer
89 buffer = io.BytesIO()
tests/entrypoints/openai/test_translation_validation.py new
+172
@@ -0,0 +1,172 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import io
5 +# imports for guided decoding tests
6 +import json
7 +from unittest.mock import patch
8 +
9 +import librosa
10 +import numpy as np
11 +import pytest
12 +import soundfile as sf
13 +from openai._base_client import AsyncAPIClient
14 +
15 +from vllm.assets.audio import AudioAsset
16 +
17 +from ...utils import RemoteOpenAIServer
18 +
19 +
20 +@pytest.fixture
21 +def foscolo():
22 + # Test translation it->en
23 + path = AudioAsset('azacinto_foscolo').get_local_path()
24 + with open(str(path), "rb") as f:
25 + yield f
26 +
27 +
28 +# NOTE: (NickLucche) the large-v3-turbo model was not trained on translation!
29 +@pytest.mark.asyncio
30 +async def test_basic_audio(foscolo):
31 + model_name = "openai/whisper-small"
32 + server_args = ["--enforce-eager"]
33 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
34 + client = remote_server.get_async_client()
35 + translation = await client.audio.translations.create(
36 + model=model_name,
37 + file=foscolo,
38 + response_format="text",
39 + # TODO remove once language detection is implemented
40 + extra_body=dict(language="it"),
41 + temperature=0.0)
42 + out = json.loads(translation)['text'].strip()
43 + assert "Nor will I ever touch the sacred" in out
44 +
45 +
46 +@pytest.mark.asyncio
47 +async def test_audio_prompt(foscolo):
48 + model_name = "openai/whisper-small"
49 + server_args = ["--enforce-eager"]
50 + # Condition whisper on starting text
51 + prompt = "Nor have I ever"
52 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
53 + client = remote_server.get_async_client()
54 + transcription = await client.audio.translations.create(
55 + model=model_name,
56 + file=foscolo,
57 + prompt=prompt,
58 + extra_body=dict(language="it"),
59 + response_format="text",
60 + temperature=0.0)
61 + out = json.loads(transcription)['text']
62 + assert "Nor will I ever touch the sacred" not in out
63 + assert prompt not in out
64 +
65 +
66 +@pytest.mark.asyncio
67 +async def test_non_asr_model(foscolo):
68 + # text to text model
69 + model_name = "JackFram/llama-68m"
70 + server_args = ["--enforce-eager"]
71 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
72 + client = remote_server.get_async_client()
73 + res = await client.audio.translations.create(model=model_name,
74 + file=foscolo,
75 + temperature=0.0)
76 + assert res.code == 400 and not res.text
77 + assert res.message == "The model does not support Translations API"
78 +
79 +
80 +@pytest.mark.asyncio
81 +async def test_streaming_response(foscolo):
82 + model_name = "openai/whisper-small"
83 + server_args = ["--enforce-eager"]
84 + translation = ""
85 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
86 + client = remote_server.get_async_client()
87 + res_no_stream = await client.audio.translations.create(
88 + model=model_name,
89 + file=foscolo,
90 + response_format="json",
91 + extra_body=dict(language="it"),
92 + temperature=0.0)
93 + # Unfortunately this only works when the openai client is patched
94 + # to use streaming mode, not exposed in the translation api.
95 + original_post = AsyncAPIClient.post
96 +
97 + async def post_with_stream(*args, **kwargs):
98 + kwargs['stream'] = True
99 + return await original_post(*args, **kwargs)
100 +
101 + with patch.object(AsyncAPIClient, "post", new=post_with_stream):
102 + client = remote_server.get_async_client()
103 + res = await client.audio.translations.create(model=model_name,
104 + file=foscolo,
105 + temperature=0.0,
106 + extra_body=dict(
107 + stream=True,
108 + language="it"))
109 + # Reconstruct from chunks and validate
110 + async for chunk in res:
111 + # just a chunk
112 + text = chunk.choices[0]['delta']['content']
113 + translation += text
114 +
115 + assert translation == res_no_stream.text
116 +
117 +
118 +@pytest.mark.asyncio
119 +async def test_stream_options(foscolo):
120 + model_name = "openai/whisper-small"
121 + server_args = ["--enforce-eager"]
122 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
123 + original_post = AsyncAPIClient.post
124 +
125 + async def post_with_stream(*args, **kwargs):
126 + kwargs['stream'] = True
127 + return await original_post(*args, **kwargs)
128 +
129 + with patch.object(AsyncAPIClient, "post", new=post_with_stream):
130 + client = remote_server.get_async_client()
131 + res = await client.audio.translations.create(
132 + model=model_name,
133 + file=foscolo,
134 + temperature=0.0,
135 + extra_body=dict(language="it",
136 + stream=True,
137 + stream_include_usage=True,
138 + stream_continuous_usage_stats=True))
139 + final = False
140 + continuous = True
141 + async for chunk in res:
142 + if not len(chunk.choices):
143 + # final usage sent
144 + final = True
145 + else:
146 + continuous = continuous and hasattr(chunk, 'usage')
147 + assert final and continuous
148 +
149 +
150 +@pytest.mark.asyncio
151 +async def test_long_audio_request(foscolo):
152 + model_name = "openai/whisper-small"
153 + server_args = ["--enforce-eager"]
154 +
155 + foscolo.seek(0)
156 + audio, sr = librosa.load(foscolo)
157 + repeated_audio = np.tile(audio, 2)
158 + # Repeated audio to buffer
159 + buffer = io.BytesIO()
160 + sf.write(buffer, repeated_audio, sr, format='WAV')
161 + buffer.seek(0)
162 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
163 + client = remote_server.get_async_client()
164 + translation = await client.audio.translations.create(
165 + model=model_name,
166 + file=buffer,
167 + extra_body=dict(language="it"),
168 + response_format="text",
169 + temperature=0.0)
170 + out = json.loads(translation)['text'].strip().lower()
171 + # TODO investigate higher model uncertainty in for longer translations.
172 + assert out.count("nor will i ever") == 2
vllm/entrypoints/openai/api_server.py
+54 -1
@@ -73,6 +73,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
73 TokenizeResponse,
74 TranscriptionRequest,
75 TranscriptionResponse,
76 + TranslationRequest,
77 + TranslationResponse,
78 UnloadLoRAAdapterRequest)
79 # yapf: enable
80 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
@@ -88,7 +90,7 @@ from vllm.entrypoints.openai.serving_score import ServingScores
90 from vllm.entrypoints.openai.serving_tokenization import (
91 OpenAIServingTokenization)
92 from vllm.entrypoints.openai.serving_transcription import (
91 - OpenAIServingTranscription)
93 + OpenAIServingTranscription, OpenAIServingTranslation)
94 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
95 from vllm.entrypoints.utils import (cli_env_setup, load_aware_call,
96 with_cancellation)
@@ -401,6 +403,10 @@ def transcription(request: Request) -> OpenAIServingTranscription:
403 return request.app.state.openai_serving_transcription
404
405
406 +def translation(request: Request) -> OpenAIServingTranslation:
407 + return request.app.state.openai_serving_translation
408 +
409 +
410 def engine_client(request: Request) -> EngineClient:
411 return request.app.state.engine_client
412
@@ -774,6 +780,47 @@ async def create_transcriptions(raw_request: Request,
780 return StreamingResponse(content=generator, media_type="text/event-stream")
781
782
783 +@router.post("/v1/audio/translations",
784 + responses={
785 + HTTPStatus.OK.value: {
786 + "content": {
787 + "text/event-stream": {}
788 + }
789 + },
790 + HTTPStatus.BAD_REQUEST.value: {
791 + "model": ErrorResponse
792 + },
793 + HTTPStatus.UNPROCESSABLE_ENTITY.value: {
794 + "model": ErrorResponse
795 + },
796 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
797 + "model": ErrorResponse
798 + },
799 + })
800 +@with_cancellation
801 +@load_aware_call
802 +async def create_translations(request: Annotated[TranslationRequest,
803 + Form()],
804 + raw_request: Request):
805 + handler = translation(raw_request)
806 + if handler is None:
807 + return base(raw_request).create_error_response(
808 + message="The model does not support Translations API")
809 +
810 + audio_data = await request.file.read()
811 + generator = await handler.create_translation(audio_data, request,
812 + raw_request)
813 +
814 + if isinstance(generator, ErrorResponse):
815 + return JSONResponse(content=generator.model_dump(),
816 + status_code=generator.code)
817 +
818 + elif isinstance(generator, TranslationResponse):
819 + return JSONResponse(content=generator.model_dump())
820 +
821 + return StreamingResponse(content=generator, media_type="text/event-stream")
822 +
823 +
824 @router.post("/rerank",
825 dependencies=[Depends(validate_json_request)],
826 responses={
@@ -1248,6 +1295,12 @@ async def init_app_state(
1295 state.openai_serving_models,
1296 request_logger=request_logger,
1297 ) if model_config.runner_type == "transcription" else None
1298 + state.openai_serving_translation = OpenAIServingTranslation(
1299 + engine_client,
1300 + model_config,
1301 + state.openai_serving_models,
1302 + request_logger=request_logger,
1303 + ) if model_config.runner_type == "transcription" else None
1304 state.task = model_config.task
1305
1306 state.enable_server_load_tracking = args.enable_server_load_tracking
vllm/entrypoints/openai/protocol.py
+187
@@ -1947,3 +1947,190 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):
1947
1948 words: Optional[list[TranscriptionWord]] = None
1949 """Extracted words and their corresponding timestamps."""
1950 +
1951 +
1952 +class TranslationResponseStreamChoice(OpenAIBaseModel):
1953 + delta: DeltaMessage
1954 + finish_reason: Optional[str] = None
1955 + stop_reason: Optional[Union[int, str]] = None
1956 +
1957 +
1958 +class TranslationStreamResponse(OpenAIBaseModel):
1959 + id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
1960 + object: Literal["translation.chunk"] = "translation.chunk"
1961 + created: int = Field(default_factory=lambda: int(time.time()))
1962 + model: str
1963 + choices: list[TranslationResponseStreamChoice]
1964 + usage: Optional[UsageInfo] = Field(default=None)
1965 +
1966 +
1967 +class TranslationRequest(OpenAIBaseModel):
1968 + # Ordered by official OpenAI API documentation
1969 + # https://platform.openai.com/docs/api-reference/audio/createTranslation
1970 +
1971 + file: UploadFile
1972 + """
1973 + The audio file object (not file name) to translate, in one of these
1974 + formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
1975 + """
1976 +
1977 + model: Optional[str] = None
1978 + """ID of the model to use.
1979 + """
1980 +
1981 + prompt: str = Field(default="")
1982 + """An optional text to guide the model's style or continue a previous audio
1983 + segment.
1984 +
1985 + The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
1986 + should match the audio language.
1987 + """
1988 +
1989 + response_format: AudioResponseFormat = Field(default="json")
1990 + """
1991 + The format of the output, in one of these options: `json`, `text`, `srt`,
1992 + `verbose_json`, or `vtt`.
1993 + """
1994 +
1995 + # TODO support additional sampling parameters
1996 + # --8<-- [start:translation-sampling-params]
1997 + temperature: float = Field(default=0.0)
1998 + """The sampling temperature, between 0 and 1.
1999 +
2000 + Higher values like 0.8 will make the output more random, while lower values
2001 + like 0.2 will make it more focused / deterministic. If set to 0, the model
2002 + will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
2003 + to automatically increase the temperature until certain thresholds are hit.
2004 + """
2005 + # --8<-- [end:translation-sampling-params]
2006 +
2007 + # --8<-- [start:translation-extra-params]
2008 + language: Optional[str] = None
2009 + """The language of the input audio we translate from.
2010 +
2011 + Supplying the input language in
2012 + [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
2013 + will improve accuracy.
2014 + """
2015 +
2016 + stream: Optional[bool] = False
2017 + """Custom field not present in the original OpenAI definition. When set,
2018 + it will enable output to be streamed in a similar fashion as the Chat
2019 + Completion endpoint.
2020 + """
2021 + # Flattened stream option to simplify form data.
2022 + stream_include_usage: Optional[bool] = False
2023 + stream_continuous_usage_stats: Optional[bool] = False
2024 + # --8<-- [end:translation-extra-params]
2025 +
2026 + # Default sampling parameters for translation requests.
2027 + _DEFAULT_SAMPLING_PARAMS: dict = {
2028 + "temperature": 0,
2029 + }
2030 +
2031 + def to_sampling_params(
2032 + self,
2033 + default_max_tokens: int,
2034 + default_sampling_params: Optional[dict] = None) -> SamplingParams:
2035 + # TODO(#9845): remove max_tokens when field is removed from OpenAI API
2036 + max_tokens = default_max_tokens
2037 +
2038 + if default_sampling_params is None:
2039 + default_sampling_params = {}
2040 + # Default parameters
2041 + if (temperature := self.temperature) is None:
2042 + temperature = default_sampling_params.get(
2043 + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
2044 +
2045 + return SamplingParams.from_optional(temperature=temperature,
2046 + max_tokens=max_tokens,
2047 + output_kind=RequestOutputKind.DELTA
2048 + if self.stream \
2049 + else RequestOutputKind.FINAL_ONLY)
2050 +
2051 + @model_validator(mode="before")
2052 + @classmethod
2053 + def validate_stream_options(cls, data):
2054 + stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
2055 + stream = data.get("stream", False)
2056 + if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2057 + raise ValueError(
2058 + "Stream options can only be defined when `stream=True`.")
2059 +
2060 + return data
2061 +
2062 +
2063 +# Translation response objects
2064 +class TranslationResponse(OpenAIBaseModel):
2065 + text: str
2066 + """The translated text."""
2067 +
2068 +
2069 +class TranslationWord(OpenAIBaseModel):
2070 + end: float
2071 + """End time of the word in seconds."""
2072 +
2073 + start: float
2074 + """Start time of the word in seconds."""
2075 +
2076 + word: str
2077 + """The text content of the word."""
2078 +
2079 +
2080 +class TranslationSegment(OpenAIBaseModel):
2081 + id: int
2082 + """Unique identifier of the segment."""
2083 +
2084 + avg_logprob: float
2085 + """Average logprob of the segment.
2086 +
2087 + If the value is lower than -1, consider the logprobs failed.
2088 + """
2089 +
2090 + compression_ratio: float
2091 + """Compression ratio of the segment.
2092 +
2093 + If the value is greater than 2.4, consider the compression failed.
2094 + """
2095 +
2096 + end: float
2097 + """End time of the segment in seconds."""
2098 +
2099 + no_speech_prob: float
2100 + """Probability of no speech in the segment.
2101 +
2102 + If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
2103 + this segment silent.
2104 + """
2105 +
2106 + seek: int
2107 + """Seek offset of the segment."""
2108 +
2109 + start: float
2110 + """Start time of the segment in seconds."""
2111 +
2112 + temperature: float
2113 + """Temperature parameter used for generating the segment."""
2114 +
2115 + text: str
2116 + """Text content of the segment."""
2117 +
2118 + tokens: list[int]
2119 + """Array of token IDs for the text content."""
2120 +
2121 +
2122 +class TranslationResponseVerbose(OpenAIBaseModel):
2123 + duration: str
2124 + """The duration of the input audio."""
2125 +
2126 + language: str
2127 + """The language of the input audio."""
2128 +
2129 + text: str
2130 + """The translated text."""
2131 +
2132 + segments: Optional[list[TranslationSegment]] = None
2133 + """Segments of the translated text and their corresponding details."""
2134 +
2135 + words: Optional[list[TranslationWord]] = None
2136 + """Extracted words and their corresponding timestamps."""
vllm/entrypoints/openai/serving_engine.py
+4 -4
@@ -58,7 +58,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
58 TokenizeCompletionRequest,
59 TokenizeResponse,
60 TranscriptionRequest,
61 - TranscriptionResponse)
61 + TranscriptionResponse,
62 + TranslationRequest)
63 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
64 from vllm.entrypoints.openai.tool_parsers import ToolParser
65 # yapf: enable
@@ -89,9 +90,8 @@ CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
90
91 ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
92 TokenizeChatRequest]
92 -
93 -AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest,
94 - TranscriptionRequest]
93 +SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]
94 +AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest, SpeechToTextRequest]
95
96 AnyResponse = Union[
97 CompletionResponse,
vllm/entrypoints/openai/serving_transcription.py
+80 -435
@@ -1,155 +1,28 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import asyncio
4 -import io
5 -import math
6 -import time
3 from collections.abc import AsyncGenerator
8 -from math import ceil
9 -from typing import Final, Optional, Union, cast
4 +from typing import Optional, Union
5
11 -import numpy as np
6 from fastapi import Request
7
8 from vllm.config import ModelConfig
9 from vllm.engine.protocol import EngineClient
10 from vllm.entrypoints.logger import RequestLogger
11 from vllm.entrypoints.openai.protocol import (
18 - DeltaMessage, ErrorResponse, RequestResponseMetadata, TranscriptionRequest,
12 + ErrorResponse, RequestResponseMetadata, TranscriptionRequest,
13 TranscriptionResponse, TranscriptionResponseStreamChoice,
20 - TranscriptionStreamResponse, UsageInfo)
21 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
14 + TranscriptionStreamResponse, TranslationRequest, TranslationResponse,
15 + TranslationResponseStreamChoice, TranslationStreamResponse)
16 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
23 -from vllm.inputs.data import PromptType
17 +from vllm.entrypoints.openai.speech_to_text import OpenAISpeechToText
18 from vllm.logger import init_logger
19 from vllm.outputs import RequestOutput
26 -from vllm.transformers_utils.processor import cached_get_processor
27 -from vllm.utils import PlaceholderModule
28 -
29 -try:
30 - import librosa
31 -except ImportError:
32 - librosa = PlaceholderModule("librosa") # type: ignore[assignment]
20
21 logger = init_logger(__name__)
22
36 -# From https://platform.openai.com/docs/guides/speech-to-text/supported-languages#supported-languages
37 -# TODO these configs should live somewhere with the model so we can support
38 -# additional ones
39 -
40 -ISO639_1_SUPPORTED_LANGS = {
41 - "af": "Afrikaans",
42 - "ar": "Arabic",
43 - "hy": "Armenian",
44 - "az": "Azerbaijani",
45 - "be": "Belarusian",
46 - "bs": "Bosnian",
47 - "bg": "Bulgarian",
48 - "ca": "Catalan",
49 - "zh": "Chinese",
50 - "hr": "Croatian",
51 - "cs": "Czech",
52 - "da": "Danish",
53 - "nl": "Dutch",
54 - "en": "English",
55 - "et": "Estonian",
56 - "fi": "Finnish",
57 - "fr": "French",
58 - "gl": "Galician",
59 - "de": "German",
60 - "el": "Greek",
61 - "he": "Hebrew",
62 - "hi": "Hindi",
63 - "hu": "Hungarian",
64 - "is": "Icelandic",
65 - "id": "Indonesian",
66 - "it": "Italian",
67 - "ja": "Japanese",
68 - "kn": "Kannada",
69 - "kk": "Kazakh",
70 - "ko": "Korean",
71 - "lv": "Latvian",
72 - "lt": "Lithuanian",
73 - "mk": "Macedonian",
74 - "ms": "Malay",
75 - "mr": "Marathi",
76 - "mi": "Maori",
77 - "ne": "Nepali",
78 - "no": "Norwegian",
79 - "fa": "Persian",
80 - "pl": "Polish",
81 - "pt": "Portuguese",
82 - "ro": "Romanian",
83 - "ru": "Russian",
84 - "sr": "Serbian",
85 - "sk": "Slovak",
86 - "sl": "Slovenian",
87 - "es": "Spanish",
88 - "sw": "Swahili",
89 - "sv": "Swedish",
90 - "tl": "Tagalog",
91 - "ta": "Tamil",
92 - "th": "Thai",
93 - "tr": "Turkish",
94 - "uk": "Ukrainian",
95 - "ur": "Urdu",
96 - "vi": "Vietnamese",
97 - "cy": "Welsh"
98 -}
99 -ISO639_1_OTHER_LANGS = {
100 - "lo": "Lao",
101 - "jw": "Javanese",
102 - "tk": "Turkmen",
103 - "yi": "Yiddish",
104 - "so": "Somali",
105 - "bn": "Bengali",
106 - "nn": "Norwegian Nynorsk",
107 - "si": "Sinhala",
108 - "yo": "Yoruba",
109 - "sa": "Sanskrit",
110 - "mi": "Māori",
111 - "fo": "Faroese", # codespell:ignore
112 - "mt": "Maltese",
113 - "tg": "Tajik",
114 - "mg": "Malagasy",
115 - "haw": "Hawaiian",
116 - "km": "Khmer",
117 - "br": "Breton",
118 - "ps": "Pashto",
119 - "ln": "Lingala",
120 - "la": "Latin",
121 - "ml": "Malayalam",
122 - "sq": "Albanian",
123 - "su": "Sundanese",
124 - "eu": "Basque",
125 - "ka": "Georgian",
126 - "uz": "Uzbek",
127 - "sn": "Shona",
128 - "ht": "Haitian",
129 - "as": "Assamese",
130 - "mn": "Mongolian",
131 - "te": "Telugu",
132 - "pa": "Panjabi",
133 - "tt": "Tatar",
134 - "gu": "Gujarati",
135 - "oc": "Occitan",
136 - "ha": "Hausa",
137 - "ba": "Bashkir",
138 - "my": "Burmese",
139 - "sd": "Sindhi",
140 - "am": "Amharic",
141 - "lb": "Luxembourgish",
142 - "bo": "Tibetan"
143 -}
144 -
145 -# As per https://platform.openai.com/docs/guides/speech-to-text#overview.
146 -# TODO configurable
147 -MAX_AUDIO_CLIP_FILESIZE_MB = 25
148 -OVERLAP_CHUNK_SECOND = 1
149 -MIN_ENERGY_WINDOW_SIZE = 1600 # 1600 ~ 100ms for 16000 Hz audio
150 -
23
152 -class OpenAIServingTranscription(OpenAIServing):
24 +class OpenAIServingTranscription(OpenAISpeechToText):
25 + """Handles transcription requests."""
26
27 def __init__(
28 self,
@@ -164,70 +37,9 @@ class OpenAIServingTranscription(OpenAIServing):
37 model_config=model_config,
38 models=models,
39 request_logger=request_logger,
167 - return_tokens_as_token_ids=return_tokens_as_token_ids)
168 -
169 - self.default_sampling_params = (
170 - self.model_config.get_diff_sampling_param())
171 - processor = cached_get_processor(model_config.model)
172 - self.max_audio_clip_s = processor.feature_extractor.chunk_length
173 - self.model_sr = processor.feature_extractor.sampling_rate
174 - self.hop_length = processor.feature_extractor.hop_length
175 -
176 - if self.default_sampling_params:
177 - logger.info(
178 - "Overwriting default completion sampling param with: %s",
179 - self.default_sampling_params)
180 -
181 - async def _preprocess_transcription(
182 - self,
183 - request: TranscriptionRequest,
184 - audio_data: bytes,
185 - ) -> tuple[list[PromptType], float]:
186 - # Validate request
187 - # TODO language should be optional and can be guessed.
188 - # For now we default to en. See
189 - # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
190 - lang_token = f"<|{request.language}|>" if request.language else "<|en|>"
191 - if request.language:
192 - if request.language in ISO639_1_SUPPORTED_LANGS:
193 - pass
194 - elif request.language in ISO639_1_OTHER_LANGS:
195 - logger.warning(
196 - "The selected language %s has limited accuracy with"
197 - " reported WER>=0.5. Results may be less accurate "
198 - "for this choice.", request.language)
199 - else:
200 - raise ValueError(
201 - f"Unsupported language: {request.language}."
202 - "Language should be one of:" +
203 - f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +
204 - f"or {list(ISO639_1_OTHER_LANGS.values())}")
205 -
206 - if len(audio_data) / 1024**2 > MAX_AUDIO_CLIP_FILESIZE_MB:
207 - raise ValueError("Maximum file size exceeded.")
208 -
209 - with io.BytesIO(audio_data) as bytes_:
210 - y, sr = librosa.load(bytes_)
211 -
212 - duration = librosa.get_duration(y=y, sr=sr)
213 - chunks = [y] if duration < 30 else self._split_audio(y, sr)
214 - prompts = []
215 - for i, chunk in enumerate(chunks):
216 - prompt = {
217 - "encoder_prompt": {
218 - "prompt": "",
219 - "multi_modal_data": {
220 - "audio": (chunk, sr),
221 - },
222 - },
223 - "decoder_prompt":
224 - f"<|startoftranscript|>{lang_token}<|transcribe|><|notimestamps|>{request.prompt}"
225 - if i == 0 else ""
226 - }
227 - prompts.append(cast(PromptType, prompt))
228 - return prompts, duration
40 + return_tokens_as_token_ids=return_tokens_as_token_ids,
41 + task_type="transcribe")
42
230 - # TODO (varun) : Make verbose response work !
43 async def create_transcription(
44 self, audio_data: bytes, request: TranscriptionRequest,
45 raw_request: Request
@@ -238,250 +50,83 @@ class OpenAIServingTranscription(OpenAIServing):
50 See https://platform.openai.com/docs/api-reference/audio/createTranscription
51 for the API specification. This API mimics the OpenAI transcription API.
52 """
241 - error_check_ret = await self._check_model(request)
242 - if error_check_ret is not None:
243 - return error_check_ret
244 -
245 - # If the engine is dead, raise the engine's DEAD_ERROR.
246 - # This is required for the streaming case, where we return a
247 - # success status before we actually start generating text :).
248 - if self.engine_client.errored:
249 - raise self.engine_client.dead_error
250 -
251 - if request.response_format not in ['text', 'json']:
252 - return self.create_error_response(
253 - "Currently only support response_format `text` or `json`")
254 -
255 - request_id = f"trsc-{self._base_request_id(raw_request)}"
256 -
257 - request_metadata = RequestResponseMetadata(request_id=request_id)
258 - if raw_request:
259 - raw_request.state.request_metadata = request_metadata
260 -
261 - try:
262 - (
263 - lora_request,
264 - prompt_adapter_request,
265 - ) = self._maybe_get_adapters(request)
266 -
267 - if lora_request:
268 - return self.create_error_response(
269 - "Currently do not support LoRA for Transcription.")
270 - if prompt_adapter_request:
271 - return self.create_error_response(
272 - "Currently do not support PromptAdapter for Transcription."
273 - )
274 -
275 - prompts, duration_s = await self._preprocess_transcription(
276 - request=request,
277 - audio_data=audio_data,
278 - )
279 -
280 - except ValueError as e:
281 - logger.exception("Error in preprocessing prompt inputs")
282 - return self.create_error_response(str(e))
283 -
284 - list_result_generator: Optional[list[AsyncGenerator[RequestOutput,
285 - None]]] = None
286 - try:
287 - # Unlike most decoder-only models, whisper generation length is not
288 - # constrained by the size of the input audio, which is mapped to a
289 - # fixed-size log-mel-spectogram.
290 - default_max_tokens = self.model_config.max_model_len
291 - sampling_params = request.to_sampling_params(
292 - default_max_tokens, self.default_sampling_params)
293 -
294 - self._log_inputs(
295 - request_id,
296 - prompts[0]['decoder_prompt'], # type: ignore
297 - params=sampling_params,
298 - lora_request=None,
299 - prompt_adapter_request=None)
300 -
301 - list_result_generator = [
302 - self.engine_client.generate(
303 - prompt,
304 - sampling_params,
305 - request_id,
306 - ) for prompt in prompts
307 - ]
308 - except ValueError as e:
309 - # TODO: Use a vllm-specific Validation Error
310 - return self.create_error_response(str(e))
311 -
312 - if request.stream:
313 - return self.transcription_stream_generator(request,
314 - list_result_generator,
315 - request_id,
316 - request_metadata,
317 - duration_s)
318 - # Non-streaming response.
319 - try:
320 - assert list_result_generator is not None
321 - text = ""
322 - for result_generator in list_result_generator:
323 - async for op in result_generator:
324 - text += op.outputs[0].text
325 - return TranscriptionResponse(text=text)
326 - except asyncio.CancelledError:
327 - return self.create_error_response("Client disconnected")
328 - except ValueError as e:
329 - # TODO: Use a vllm-specific Validation Error
330 - return self.create_error_response(str(e))
53 + return await self._create_speech_to_text(
54 + audio_data=audio_data,
55 + request=request,
56 + raw_request=raw_request,
57 + response_class=TranscriptionResponse,
58 + stream_generator_method=self.transcription_stream_generator,
59 + )
60
61 async def transcription_stream_generator(
62 self, request: TranscriptionRequest,
334 - list_result_generator: list[AsyncGenerator[RequestOutput, None]],
63 + result_generator: list[AsyncGenerator[RequestOutput, None]],
64 request_id: str, request_metadata: RequestResponseMetadata,
65 audio_duration_s: float) -> AsyncGenerator[str, None]:
337 - created_time = int(time.time())
338 - model_name = request.model
339 - chunk_object_type: Final = "transcription.chunk"
340 -
341 - completion_tokens = 0
342 - num_prompt_tokens = 0
343 -
344 - include_usage = request.stream_include_usage \
345 - if request.stream_include_usage else False
346 - include_continuous_usage = request.stream_continuous_usage_stats\
347 - if include_usage and request.stream_continuous_usage_stats\
348 - else False
349 -
350 - try:
351 - for result_generator in list_result_generator:
352 - async for res in result_generator:
353 - # On first result.
354 - if res.prompt_token_ids is not None:
355 - # Do not account the 4-tokens `<|startoftranscript|>..`
356 - # Could be negative when language token
357 - # is not specified.
358 - num_prompt_tokens = max(
359 - len(res.prompt_token_ids) - 4, 0)
360 - # NOTE(NickLucche) user can't pass encoder
361 - # prompts directly at least not to Whisper.
362 - # One indicator of the encoder amount of processing
363 - # is the log-mel spectogram length.
364 - num_prompt_tokens += ceil(
365 - audio_duration_s * self.model_sr / self.hop_length)
366 -
367 - # We need to do it here, because if there are exceptions in
368 - # the result_generator, it needs to be sent as the FIRST
369 - # response (by the try...catch).
370 -
371 - # Just one output (n=1) supported.
372 - assert len(res.outputs) == 1
373 - output = res.outputs[0]
66 + generator = self._speech_to_text_stream_generator(
67 + request=request,
68 + list_result_generator=result_generator,
69 + request_id=request_id,
70 + request_metadata=request_metadata,
71 + audio_duration_s=audio_duration_s,
72 + chunk_object_type="transcription.chunk",
73 + response_stream_choice_class=TranscriptionResponseStreamChoice,
74 + stream_response_class=TranscriptionStreamResponse,
75 + )
76 + async for chunk in generator:
77 + yield chunk
78 +
79 +
80 +class OpenAIServingTranslation(OpenAISpeechToText):
81 + """Handles translation requests."""
82
375 - delta_message = DeltaMessage(content=output.text)
376 - completion_tokens += len(output.token_ids)
377 -
378 - if output.finish_reason is None:
379 - # Still generating, send delta update.
380 - choice_data = TranscriptionResponseStreamChoice(
381 - delta=delta_message)
382 - else:
383 - # Model is finished generating.
384 - choice_data = TranscriptionResponseStreamChoice(
385 - delta=delta_message,
386 - finish_reason=output.finish_reason,
387 - stop_reason=output.stop_reason)
388 -
389 - chunk = TranscriptionStreamResponse(
390 - id=request_id,
391 - object=chunk_object_type,
392 - created=created_time,
393 - choices=[choice_data],
394 - model=model_name)
395 -
396 - # handle usage stats if requested & if continuous
397 - if include_continuous_usage:
398 - chunk.usage = UsageInfo(
399 - prompt_tokens=num_prompt_tokens,
400 - completion_tokens=completion_tokens,
401 - total_tokens=num_prompt_tokens + completion_tokens,
402 - )
403 -
404 - data = chunk.model_dump_json(exclude_unset=True)
405 - yield f"data: {data}\n\n"
406 -
407 - # Once the final token is handled, if stream_options.include_usage
408 - # is sent, send the usage.
409 - if include_usage:
410 - final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
411 - completion_tokens=completion_tokens,
412 - total_tokens=num_prompt_tokens +
413 - completion_tokens)
414 -
415 - final_usage_chunk = TranscriptionStreamResponse(
416 - id=request_id,
417 - object=chunk_object_type,
418 - created=created_time,
419 - choices=[],
420 - model=model_name,
421 - usage=final_usage)
422 - final_usage_data = (final_usage_chunk.model_dump_json(
423 - exclude_unset=True, exclude_none=True))
424 - yield f"data: {final_usage_data}\n\n"
425 -
426 - # report to FastAPI middleware aggregate usage across all choices
427 - request_metadata.final_usage_info = UsageInfo(
428 - prompt_tokens=num_prompt_tokens,
429 - completion_tokens=completion_tokens,
430 - total_tokens=num_prompt_tokens + completion_tokens)
431 -
432 - except Exception as e:
433 - # TODO: Use a vllm-specific Validation Error
434 - logger.exception("Error in chat completion stream generator.")
435 - data = self.create_streaming_error_response(str(e))
436 - yield f"data: {data}\n\n"
437 - # Send the final done message after all response.n are finished
438 - yield "data: [DONE]\n\n"
439 -
440 - def _split_audio(self, audio_data: np.ndarray,
441 - sample_rate: int) -> list[np.ndarray]:
442 - chunk_size = sample_rate * self.max_audio_clip_s
443 - overlap_size = sample_rate * OVERLAP_CHUNK_SECOND
444 - chunks = []
445 - i = 0
446 - while i < audio_data.shape[-1]:
447 - if i + chunk_size >= audio_data.shape[-1]:
448 - # handle last chunk
449 - chunks.append(audio_data[..., i:])
450 - break
451 -
452 - # Find the best split point in the overlap region
453 - search_start = i + chunk_size - overlap_size
454 - search_end = min(i + chunk_size, audio_data.shape[-1])
455 - split_point = self._find_split_point(audio_data, search_start,
456 - search_end)
83 + def __init__(
84 + self,
85 + engine_client: EngineClient,
86 + model_config: ModelConfig,
87 + models: OpenAIServingModels,
88 + *,
89 + request_logger: Optional[RequestLogger],
90 + return_tokens_as_token_ids: bool = False,
91 + ):
92 + super().__init__(engine_client=engine_client,
93 + model_config=model_config,
94 + models=models,
95 + request_logger=request_logger,
96 + return_tokens_as_token_ids=return_tokens_as_token_ids,
97 + task_type="translate")
98
458 - # Extract chunk up to the split point
459 - chunks.append(audio_data[..., i:split_point])
460 - i = split_point
461 - return chunks
99 + async def create_translation(
100 + self, audio_data: bytes, request: TranslationRequest,
101 + raw_request: Request
102 + ) -> Union[TranslationResponse, AsyncGenerator[str, None], ErrorResponse]:
103 + """Translation API similar to OpenAI's API.
104
463 - def _find_split_point(self, wav: np.ndarray, start_idx: int,
464 - end_idx: int) -> int:
465 - """Find the best point to split audio by
466 - looking for silence or low amplitude.
467 - Args:
468 - wav: Audio tensor [1, T]
469 - start_idx: Start index of search region
470 - end_idx: End index of search region
471 - Returns:
472 - Index of best splitting point
105 + See https://platform.openai.com/docs/api-reference/audio/createTranslation
106 + for the API specification. This API mimics the OpenAI translation API.
107 """
474 - segment = wav[start_idx:end_idx]
475 -
476 - # Calculate RMS energy in small windows
477 - min_energy = math.inf
478 - quietest_idx = 0
479 - for i in range(0,
480 - len(segment) - MIN_ENERGY_WINDOW_SIZE,
481 - MIN_ENERGY_WINDOW_SIZE):
482 - window = segment[i:i + MIN_ENERGY_WINDOW_SIZE]
483 - energy = (window**2).mean()**0.5
484 - if energy < min_energy:
485 - quietest_idx = i + start_idx
486 - min_energy = energy
487 - return quietest_idx
108 + return await self._create_speech_to_text(
109 + audio_data=audio_data,
110 + request=request,
111 + raw_request=raw_request,
112 + response_class=TranslationResponse,
113 + stream_generator_method=self.translation_stream_generator,
114 + )
115 +
116 + async def translation_stream_generator(
117 + self, request: TranslationRequest,
118 + result_generator: list[AsyncGenerator[RequestOutput, None]],
119 + request_id: str, request_metadata: RequestResponseMetadata,
120 + audio_duration_s: float) -> AsyncGenerator[str, None]:
121 + generator = self._speech_to_text_stream_generator(
122 + request=request,
123 + list_result_generator=result_generator,
124 + request_id=request_id,
125 + request_metadata=request_metadata,
126 + audio_duration_s=audio_duration_s,
127 + chunk_object_type="translation.chunk",
128 + response_stream_choice_class=TranslationResponseStreamChoice,
129 + stream_response_class=TranslationStreamResponse,
130 + )
131 + async for chunk in generator:
132 + yield chunk
vllm/entrypoints/openai/speech_to_text.py new
+503
@@ -0,0 +1,503 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import asyncio
4 +import io
5 +import math
6 +import time
7 +from collections.abc import AsyncGenerator
8 +from math import ceil
9 +from typing import Callable, Literal, Optional, TypeVar, Union, cast
10 +
11 +import numpy as np
12 +from fastapi import Request
13 +
14 +from vllm.config import ModelConfig
15 +from vllm.engine.protocol import EngineClient
16 +from vllm.entrypoints.logger import RequestLogger
17 +from vllm.entrypoints.openai.protocol import (
18 + DeltaMessage, ErrorResponse, RequestResponseMetadata,
19 + TranscriptionResponse, TranscriptionResponseStreamChoice,
20 + TranscriptionStreamResponse, TranslationResponse,
21 + TranslationResponseStreamChoice, TranslationStreamResponse, UsageInfo)
22 +from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
23 + SpeechToTextRequest)
24 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
25 +from vllm.inputs.data import PromptType
26 +from vllm.logger import init_logger
27 +from vllm.outputs import RequestOutput
28 +from vllm.transformers_utils.processor import cached_get_processor
29 +from vllm.utils import PlaceholderModule
30 +
31 +try:
32 + import librosa
33 +except ImportError:
34 + librosa = PlaceholderModule("librosa") # type: ignore[assignment]
35 +
36 +SpeechToTextResponse = Union[TranscriptionResponse, TranslationResponse]
37 +T = TypeVar("T", bound=SpeechToTextResponse)
38 +
39 +logger = init_logger(__name__)
40 +
41 +# From https://platform.openai.com/docs/guides/speech-to-text/supported-languages
42 +# TODO these configs should live somewhere with the model so we can support
43 +# additional ones
44 +
45 +ISO639_1_SUPPORTED_LANGS = {
46 + "af": "Afrikaans",
47 + "ar": "Arabic",
48 + "hy": "Armenian",
49 + "az": "Azerbaijani",
50 + "be": "Belarusian",
51 + "bs": "Bosnian",
52 + "bg": "Bulgarian",
53 + "ca": "Catalan",
54 + "zh": "Chinese",
55 + "hr": "Croatian",
56 + "cs": "Czech",
57 + "da": "Danish",
58 + "nl": "Dutch",
59 + "en": "English",
60 + "et": "Estonian",
61 + "fi": "Finnish",
62 + "fr": "French",
63 + "gl": "Galician",
64 + "de": "German",
65 + "el": "Greek",
66 + "he": "Hebrew",
67 + "hi": "Hindi",
68 + "hu": "Hungarian",
69 + "is": "Icelandic",
70 + "id": "Indonesian",
71 + "it": "Italian",
72 + "ja": "Japanese",
73 + "kn": "Kannada",
74 + "kk": "Kazakh",
75 + "ko": "Korean",
76 + "lv": "Latvian",
77 + "lt": "Lithuanian",
78 + "mk": "Macedonian",
79 + "ms": "Malay",
80 + "mr": "Marathi",
81 + "mi": "Maori",
82 + "ne": "Nepali",
83 + "no": "Norwegian",
84 + "fa": "Persian",
85 + "pl": "Polish",
86 + "pt": "Portuguese",
87 + "ro": "Romanian",
88 + "ru": "Russian",
89 + "sr": "Serbian",
90 + "sk": "Slovak",
91 + "sl": "Slovenian",
92 + "es": "Spanish",
93 + "sw": "Swahili",
94 + "sv": "Swedish",
95 + "tl": "Tagalog",
96 + "ta": "Tamil",
97 + "th": "Thai",
98 + "tr": "Turkish",
99 + "uk": "Ukrainian",
100 + "ur": "Urdu",
101 + "vi": "Vietnamese",
102 + "cy": "Welsh"
103 +}
104 +ISO639_1_OTHER_LANGS = {
105 + "lo": "Lao",
106 + "jw": "Javanese",
107 + "tk": "Turkmen",
108 + "yi": "Yiddish",
109 + "so": "Somali",
110 + "bn": "Bengali",
111 + "nn": "Norwegian Nynorsk",
112 + "si": "Sinhala",
113 + "yo": "Yoruba",
114 + "sa": "Sanskrit",
115 + "mi": "Māori",
116 + "fo": "Faroese", # codespell:ignore
117 + "mt": "Maltese",
118 + "tg": "Tajik",
119 + "mg": "Malagasy",
120 + "haw": "Hawaiian",
121 + "km": "Khmer",
122 + "br": "Breton",
123 + "ps": "Pashto",
124 + "ln": "Lingala",
125 + "la": "Latin",
126 + "ml": "Malayalam",
127 + "sq": "Albanian",
128 + "su": "Sundanese",
129 + "eu": "Basque",
130 + "ka": "Georgian",
131 + "uz": "Uzbek",
132 + "sn": "Shona",
133 + "ht": "Haitian",
134 + "as": "Assamese",
135 + "mn": "Mongolian",
136 + "te": "Telugu",
137 + "pa": "Panjabi",
138 + "tt": "Tatar",
139 + "gu": "Gujarati",
140 + "oc": "Occitan",
141 + "ha": "Hausa",
142 + "ba": "Bashkir",
143 + "my": "Burmese",
144 + "sd": "Sindhi",
145 + "am": "Amharic",
146 + "lb": "Luxembourgish",
147 + "bo": "Tibetan"
148 +}
149 +
150 +# As per https://platform.openai.com/docs/guides/speech-to-text#overview.
151 +# TODO configurable
152 +MAX_AUDIO_CLIP_FILESIZE_MB = 25
153 +OVERLAP_CHUNK_SECOND = 1
154 +MIN_ENERGY_WINDOW_SIZE = 1600 # 1600 ~ 100ms for 16000 Hz audio
155 +
156 +
157 +class OpenAISpeechToText(OpenAIServing):
158 + """Base class for speech-to-text operations like transcription and
159 + translation."""
160 +
161 + def __init__(
162 + self,
163 + engine_client: EngineClient,
164 + model_config: ModelConfig,
165 + models: OpenAIServingModels,
166 + *,
167 + request_logger: Optional[RequestLogger],
168 + return_tokens_as_token_ids: bool = False,
169 + task_type: Literal["transcribe", "translate"] = "transcribe",
170 + ):
171 + super().__init__(engine_client=engine_client,
172 + model_config=model_config,
173 + models=models,
174 + request_logger=request_logger,
175 + return_tokens_as_token_ids=return_tokens_as_token_ids)
176 +
177 + self.default_sampling_params = (
178 + self.model_config.get_diff_sampling_param())
179 + processor = cached_get_processor(model_config.model)
180 + self.max_audio_clip_s = processor.feature_extractor.chunk_length
181 + self.model_sr = processor.feature_extractor.sampling_rate
182 + self.hop_length = processor.feature_extractor.hop_length
183 + self.task_type = task_type
184 +
185 + if self.default_sampling_params:
186 + logger.info(
187 + "Overwriting default completion sampling param with: %s",
188 + self.default_sampling_params)
189 +
190 + async def _preprocess_speech_to_text(
191 + self,
192 + request: SpeechToTextRequest,
193 + audio_data: bytes,
194 + ) -> tuple[list[PromptType], float]:
195 + # Validate request
196 + # TODO language should be optional and can be guessed.
197 + # For now we default to en. See
198 + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
199 + lang_token = f"<|{request.language}|>" if request.language else "<|en|>"
200 + if request.language:
201 + if request.language in ISO639_1_SUPPORTED_LANGS:
202 + pass
203 + elif request.language in ISO639_1_OTHER_LANGS:
204 + logger.warning(
205 + "The selected language %s has limited accuracy with"
206 + " reported WER>=0.5. Results may be less accurate "
207 + "for this choice.", request.language)
208 + else:
209 + raise ValueError(
210 + f"Unsupported language: {request.language}."
211 + "Language should be one of:" +
212 + f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +
213 + f"or {list(ISO639_1_OTHER_LANGS.values())}")
214 +
215 + if len(audio_data) / 1024**2 > MAX_AUDIO_CLIP_FILESIZE_MB:
216 + raise ValueError("Maximum file size exceeded.")
217 +
218 + with io.BytesIO(audio_data) as bytes_:
219 + # NOTE resample to model SR here for efficiency. This is also a
220 + # pre-requisite for chunking, as it assumes Whisper SR.
221 + y, sr = librosa.load(bytes_, sr=self.model_sr)
222 +
223 + duration = librosa.get_duration(y=y, sr=sr)
224 + chunks = [y] if duration < 30 else self._split_audio(y, int(sr))
225 + prompts = []
226 + for chunk in chunks:
227 + prompt = {
228 + "encoder_prompt": {
229 + "prompt": "",
230 + "multi_modal_data": {
231 + "audio": (chunk, sr),
232 + },
233 + },
234 + "decoder_prompt":
235 + (f"<|startoftranscript|>{lang_token}"
236 + f"<|{self.task_type}|><|notimestamps|>{request.prompt}")
237 + }
238 + prompts.append(cast(PromptType, prompt))
239 + return prompts, duration
240 +
241 + async def _create_speech_to_text(
242 + self,
243 + audio_data: bytes,
244 + request: SpeechToTextRequest,
245 + raw_request: Request,
246 + response_class: type[T],
247 + stream_generator_method: Callable[..., AsyncGenerator[str, None]],
248 + ) -> Union[T, AsyncGenerator[str, None], ErrorResponse]:
249 + """Base method for speech-to-text operations like transcription and
250 + translation."""
251 + error_check_ret = await self._check_model(request)
252 + if error_check_ret is not None:
253 + return error_check_ret
254 +
255 + # If the engine is dead, raise the engine's DEAD_ERROR.
256 + # This is required for the streaming case, where we return a
257 + # success status before we actually start generating text :).
258 + if self.engine_client.errored:
259 + raise self.engine_client.dead_error
260 +
261 + if request.response_format not in ['text', 'json']:
262 + return self.create_error_response(
263 + "Currently only support response_format `text` or `json`")
264 +
265 + request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
266 +
267 + request_metadata = RequestResponseMetadata(request_id=request_id)
268 + if raw_request:
269 + raw_request.state.request_metadata = request_metadata
270 +
271 + try:
272 + (
273 + lora_request,
274 + prompt_adapter_request,
275 + ) = self._maybe_get_adapters(request)
276 +
277 + if lora_request:
278 + return self.create_error_response(
279 + "Currently do not support LoRA for "
280 + f"{self.task_type.title()}.")
281 + if prompt_adapter_request:
282 + return self.create_error_response(
283 + f"Currently do not support PromptAdapter for "
284 + f"{self.task_type.title()}.")
285 +
286 + prompts, duration_s = await self._preprocess_speech_to_text(
287 + request=request,
288 + audio_data=audio_data,
289 + )
290 +
291 + except ValueError as e:
292 + logger.exception("Error in preprocessing prompt inputs")
293 + return self.create_error_response(str(e))
294 +
295 + list_result_generator: Optional[list[AsyncGenerator[RequestOutput,
296 + None]]] = None
297 + try:
298 + # Unlike most decoder-only models, whisper generation length is not
299 + # constrained by the size of the input audio, which is mapped to a
300 + # fixed-size log-mel-spectogram.
301 + default_max_tokens = self.model_config.max_model_len
302 + sampling_params = request.to_sampling_params(
303 + default_max_tokens, self.default_sampling_params)
304 +
305 + self._log_inputs(
306 + request_id,
307 + prompts[0]['decoder_prompt'], # type: ignore
308 + params=sampling_params,
309 + lora_request=None,
310 + prompt_adapter_request=None)
311 +
312 + list_result_generator = [
313 + self.engine_client.generate(
314 + prompt,
315 + sampling_params,
316 + request_id,
317 + ) for prompt in prompts
318 + ]
319 + except ValueError as e:
320 + # TODO: Use a vllm-specific Validation Error
321 + return self.create_error_response(str(e))
322 +
323 + if request.stream:
324 + return stream_generator_method(request, list_result_generator,
325 + request_id, request_metadata,
326 + duration_s)
327 + # Non-streaming response.
328 + try:
329 + assert list_result_generator is not None
330 + text = ""
331 + for result_generator in list_result_generator:
332 + async for op in result_generator:
333 + text += op.outputs[0].text
334 + return cast(T, response_class(text=text))
335 + except asyncio.CancelledError:
336 + return self.create_error_response("Client disconnected")
337 + except ValueError as e:
338 + # TODO: Use a vllm-specific Validation Error
339 + return self.create_error_response(str(e))
340 +
341 + async def _speech_to_text_stream_generator(
342 + self,
343 + request: SpeechToTextRequest,
344 + list_result_generator: list[AsyncGenerator[RequestOutput, None]],
345 + request_id: str,
346 + request_metadata: RequestResponseMetadata,
347 + audio_duration_s: float,
348 + chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
349 + response_stream_choice_class: Union[
350 + type[TranscriptionResponseStreamChoice],
351 + type[TranslationResponseStreamChoice]],
352 + stream_response_class: Union[type[TranscriptionStreamResponse],
353 + type[TranslationStreamResponse]],
354 + ) -> AsyncGenerator[str, None]:
355 + created_time = int(time.time())
356 + model_name = request.model
357 +
358 + completion_tokens = 0
359 + num_prompt_tokens = 0
360 +
361 + include_usage = request.stream_include_usage \
362 + if request.stream_include_usage else False
363 + include_continuous_usage = request.stream_continuous_usage_stats\
364 + if include_usage and request.stream_continuous_usage_stats\
365 + else False
366 +
367 + try:
368 + for result_generator in list_result_generator:
369 + async for res in result_generator:
370 + # On first result.
371 + if res.prompt_token_ids is not None:
372 + # Do not account the 4-tokens `<|startoftranscript|>..`
373 + # Could be negative when language token
374 + # is not specified.
375 + num_prompt_tokens = max(
376 + len(res.prompt_token_ids) - 4, 0)
377 + # NOTE(NickLucche) user can't pass encoder
378 + # prompts directly at least not to Whisper.
379 + # One indicator of the encoder amount of processing
380 + # is the log-mel spectogram length.
381 + num_prompt_tokens += ceil(
382 + audio_duration_s * self.model_sr / self.hop_length)
383 +
384 + # We need to do it here, because if there are exceptions in
385 + # the result_generator, it needs to be sent as the FIRST
386 + # response (by the try...catch).
387 +
388 + # Just one output (n=1) supported.
389 + assert len(res.outputs) == 1
390 + output = res.outputs[0]
391 +
392 + delta_message = DeltaMessage(content=output.text)
393 + completion_tokens += len(output.token_ids)
394 +
395 + if output.finish_reason is None:
396 + # Still generating, send delta update.
397 + choice_data = response_stream_choice_class(
398 + delta=delta_message)
399 + else:
400 + # Model is finished generating.
401 + choice_data = response_stream_choice_class(
402 + delta=delta_message,
403 + finish_reason=output.finish_reason,
404 + stop_reason=output.stop_reason)
405 +
406 + chunk = stream_response_class(id=request_id,
407 + object=chunk_object_type,
408 + created=created_time,
409 + choices=[choice_data],
410 + model=model_name)
411 +
412 + # handle usage stats if requested & if continuous
413 + if include_continuous_usage:
414 + chunk.usage = UsageInfo(
415 + prompt_tokens=num_prompt_tokens,
416 + completion_tokens=completion_tokens,
417 + total_tokens=num_prompt_tokens + completion_tokens,
418 + )
419 +
420 + data = chunk.model_dump_json(exclude_unset=True)
421 + yield f"data: {data}\n\n"
422 +
423 + # Once the final token is handled, if stream_options.include_usage
424 + # is sent, send the usage.
425 + if include_usage:
426 + final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
427 + completion_tokens=completion_tokens,
428 + total_tokens=num_prompt_tokens +
429 + completion_tokens)
430 +
431 + final_usage_chunk = stream_response_class(
432 + id=request_id,
433 + object=chunk_object_type,
434 + created=created_time,
435 + choices=[],
436 + model=model_name,
437 + usage=final_usage)
438 + final_usage_data = (final_usage_chunk.model_dump_json(
439 + exclude_unset=True, exclude_none=True))
440 + yield f"data: {final_usage_data}\n\n"
441 +
442 + # report to FastAPI middleware aggregate usage across all choices
443 + request_metadata.final_usage_info = UsageInfo(
444 + prompt_tokens=num_prompt_tokens,
445 + completion_tokens=completion_tokens,
446 + total_tokens=num_prompt_tokens + completion_tokens)
447 +
448 + except Exception as e:
449 + # TODO: Use a vllm-specific Validation Error
450 + logger.exception("Error in %s stream generator.", self.task_type)
451 + data = self.create_streaming_error_response(str(e))
452 + yield f"data: {data}\n\n"
453 + # Send the final done message after all response.n are finished
454 + yield "data: [DONE]\n\n"
455 +
456 + def _split_audio(self, audio_data: np.ndarray,
457 + sample_rate: int) -> list[np.ndarray]:
458 + chunk_size = sample_rate * self.max_audio_clip_s
459 + overlap_size = sample_rate * OVERLAP_CHUNK_SECOND
460 + chunks = []
461 + i = 0
462 + while i < audio_data.shape[-1]:
463 + if i + chunk_size >= audio_data.shape[-1]:
464 + # handle last chunk
465 + chunks.append(audio_data[..., i:])
466 + break
467 +
468 + # Find the best split point in the overlap region
469 + search_start = i + chunk_size - overlap_size
470 + search_end = min(i + chunk_size, audio_data.shape[-1])
471 + split_point = self._find_split_point(audio_data, search_start,
472 + search_end)
473 +
474 + # Extract chunk up to the split point
475 + chunks.append(audio_data[..., i:split_point])
476 + i = split_point
477 + return chunks
478 +
479 + def _find_split_point(self, wav: np.ndarray, start_idx: int,
480 + end_idx: int) -> int:
481 + """Find the best point to split audio by
482 + looking for silence or low amplitude.
483 + Args:
484 + wav: Audio tensor [1, T]
485 + start_idx: Start index of search region
486 + end_idx: End index of search region
487 + Returns:
488 + Index of best splitting point
489 + """
490 + segment = wav[start_idx:end_idx]
491 +
492 + # Calculate RMS energy in small windows
493 + min_energy = math.inf
494 + quietest_idx = 0
495 + for i in range(0,
496 + len(segment) - MIN_ENERGY_WINDOW_SIZE,
497 + MIN_ENERGY_WINDOW_SIZE):
498 + window = segment[i:i + MIN_ENERGY_WINDOW_SIZE]
499 + energy = (window**2).mean()**0.5
500 + if energy < min_energy:
501 + quietest_idx = i + start_idx
502 + min_energy = energy
503 + return quietest_idx