A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Frontend][Docs] Transcription API streaming (#13301)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Mar 6, 2025 at 11:39 UTC
fa82b9385330319619ddb293a9f01ccd96fd0faf
5 files changed
+297
-26
docs/source/serving/openai_compatible_server.md
+4
@@ -379,6 +379,10 @@ For chat-like input (i.e. if `messages` is passed), these extra parameters are s
379
Our Transcriptions API is compatible with [OpenAI's Transcriptions API](https://platform.openai.com/docs/api-reference/audio/createTranscription);
380
you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
381
382
+:::{note}
383
+To use the Transcriptions API, please install with extra audio dependencies using `pip install vllm[audio]`.
384
+:::
385
+
386
<!-- TODO: api enforced limits + uploading audios -->
387
388
Code example: <gh-file:examples/online_serving/openai_transcription_client.py>
examples/online_serving/openai_transcription_client.py
+51
-8
@@ -1,4 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+import asyncio
3
+import json
4
+
5
+import httpx
6
from openai import OpenAI
7
8
from vllm.assets.audio import AudioAsset
@@ -13,11 +17,50 @@ client = OpenAI(
17
api_key=openai_api_key,
18
base_url=openai_api_base,
19
)
16
-with open(str(mary_had_lamb), "rb") as f:
17
- transcription = client.audio.transcriptions.create(
18
- file=f,
19
- model="openai/whisper-large-v3",
20
- language="en",
21
- response_format="text",
22
- temperature=0.0)
23
- print("transcription result:", transcription)
20
+
21
+
22
+def sync_openai():
23
+ with open(str(mary_had_lamb), "rb") as f:
24
+ transcription = client.audio.transcriptions.create(
25
+ file=f,
26
+ model="openai/whisper-small",
27
+ language="en",
28
+ response_format="json",
29
+ temperature=0.0)
30
+ print("transcription result:", transcription.text)
31
+
32
+
33
+sync_openai()
34
+
35
+
36
+# OpenAI Transcription API client does not support streaming.
37
+async def stream_openai_response():
38
+ data = {
39
+ "language": "en",
40
+ 'stream': True,
41
+ "model": "openai/whisper-large-v3",
42
+ }
43
+ url = openai_api_base + "/audio/transcriptions"
44
+ print("transcription result:", end=' ')
45
+ async with httpx.AsyncClient() as client:
46
+ with open(str(winning_call), "rb") as f:
47
+ async with client.stream('POST', url, files={'file': f},
48
+ data=data) as response:
49
+ async for line in response.aiter_lines():
50
+ # Each line is a JSON object prefixed with 'data: '
51
+ if line:
52
+ if line.startswith('data: '):
53
+ line = line[len('data: '):]
54
+ # Last chunk, stream ends
55
+ if line.strip() == '[DONE]':
56
+ break
57
+ # Parse the JSON response
58
+ chunk = json.loads(line)
59
+ # Extract and print the content
60
+ content = chunk['choices'][0].get('delta',
61
+ {}).get('content')
62
+ print(content, end='')
63
+
64
+
65
+# Run the asynchronous function
66
+asyncio.run(stream_openai_response())
tests/entrypoints/openai/test_transcription_validation.py
+72
@@ -3,12 +3,14 @@
3
# imports for guided decoding tests
4
import io
5
import json
6
+from unittest.mock import patch
7
8
import librosa
9
import numpy as np
10
import openai
11
import pytest
12
import soundfile as sf
13
+from openai._base_client import AsyncAPIClient
14
15
from vllm.assets.audio import AudioAsset
16
@@ -120,3 +122,73 @@ async def test_completion_endpoints():
122
res = await client.completions.create(model=model_name, prompt="Hello")
123
assert res.code == 400
124
assert res.message == "The model does not support Completions API"
125
+
126
+
127
+@pytest.mark.asyncio
128
+async def test_streaming_response(winning_call):
129
+ model_name = "openai/whisper-small"
130
+ server_args = ["--enforce-eager"]
131
+ transcription = ""
132
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
133
+ client = remote_server.get_async_client()
134
+ res_no_stream = await client.audio.transcriptions.create(
135
+ model=model_name,
136
+ file=winning_call,
137
+ response_format="json",
138
+ language="en",
139
+ temperature=0.0)
140
+ # Unfortunately this only works when the openai client is patched
141
+ # to use streaming mode, not exposed in the transcription api.
142
+ original_post = AsyncAPIClient.post
143
+
144
+ async def post_with_stream(*args, **kwargs):
145
+ kwargs['stream'] = True
146
+ return await original_post(*args, **kwargs)
147
+
148
+ with patch.object(AsyncAPIClient, "post", new=post_with_stream):
149
+ client = remote_server.get_async_client()
150
+ res = await client.audio.transcriptions.create(
151
+ model=model_name,
152
+ file=winning_call,
153
+ language="en",
154
+ temperature=0.0,
155
+ extra_body=dict(stream=True))
156
+ # Reconstruct from chunks and validate
157
+ async for chunk in res:
158
+ # just a chunk
159
+ text = chunk.choices[0]['delta']['content']
160
+ transcription += text
161
+
162
+ assert transcription == res_no_stream.text
163
+
164
+
165
+@pytest.mark.asyncio
166
+async def test_stream_options(winning_call):
167
+ model_name = "openai/whisper-small"
168
+ server_args = ["--enforce-eager"]
169
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
170
+ original_post = AsyncAPIClient.post
171
+
172
+ async def post_with_stream(*args, **kwargs):
173
+ kwargs['stream'] = True
174
+ return await original_post(*args, **kwargs)
175
+
176
+ with patch.object(AsyncAPIClient, "post", new=post_with_stream):
177
+ client = remote_server.get_async_client()
178
+ res = await client.audio.transcriptions.create(
179
+ model=model_name,
180
+ file=winning_call,
181
+ language="en",
182
+ temperature=0.0,
183
+ extra_body=dict(stream=True,
184
+ stream_include_usage=True,
185
+ stream_continuous_usage_stats=True))
186
+ final = False
187
+ continuous = True
188
+ async for chunk in res:
189
+ if not len(chunk.choices):
190
+ # final usage sent
191
+ final = True
192
+ else:
193
+ continuous = continuous and hasattr(chunk, 'usage')
194
+ assert final and continuous
vllm/entrypoints/openai/protocol.py
+39
-1
@@ -1285,6 +1285,21 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
1285
usage: Optional[UsageInfo] = Field(default=None)
1286
1287
1288
+class TranscriptionResponseStreamChoice(OpenAIBaseModel):
1289
+ delta: DeltaMessage
1290
+ finish_reason: Optional[str] = None
1291
+ stop_reason: Optional[Union[int, str]] = None
1292
+
1293
+
1294
+class TranscriptionStreamResponse(OpenAIBaseModel):
1295
+ id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
1296
+ object: Literal["transcription.chunk"] = "transcription.chunk"
1297
+ created: int = Field(default_factory=lambda: int(time.time()))
1298
+ model: str
1299
+ choices: list[TranscriptionResponseStreamChoice]
1300
+ usage: Optional[UsageInfo] = Field(default=None)
1301
+
1302
+
1303
class BatchRequestInput(OpenAIBaseModel):
1304
"""
1305
The per-line object of the batch input file.
@@ -1510,6 +1525,15 @@ class TranscriptionRequest(OpenAIBaseModel):
1525
timestamps incurs additional latency.
1526
"""
1527
1528
+ stream: Optional[bool] = False
1529
+ """Custom field not present in the original OpenAI definition. When set,
1530
+ it will enable output to be streamed in a similar fashion as the Chat
1531
+ Completion endpoint.
1532
+ """
1533
+ # Flattened stream option to simplify form data.
1534
+ stream_include_usage: Optional[bool] = False
1535
+ stream_continuous_usage_stats: Optional[bool] = False
1536
+
1537
# Default sampling parameters for transcription requests.
1538
_DEFAULT_SAMPLING_PARAMS: dict = {
1539
"temperature": 0,
@@ -1530,7 +1554,21 @@ class TranscriptionRequest(OpenAIBaseModel):
1554
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
1555
1556
return SamplingParams.from_optional(temperature=temperature,
1533
- max_tokens=max_tokens)
1557
+ max_tokens=max_tokens,
1558
+ output_kind=RequestOutputKind.DELTA
1559
+ if self.stream \
1560
+ else RequestOutputKind.FINAL_ONLY)
1561
+
1562
+ @model_validator(mode="before")
1563
+ @classmethod
1564
+ def validate_stream_options(cls, data):
1565
+ stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
1566
+ stream = data.get("stream", False)
1567
+ if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
1568
+ raise ValueError(
1569
+ "Stream options can only be defined when `stream=True`.")
1570
+
1571
+ return data
1572
1573
1574
# Transcription response objects
vllm/entrypoints/openai/serving_transcription.py
+131
-17
@@ -1,24 +1,26 @@
1
# SPDX-License-Identifier: Apache-2.0
2
import asyncio
3
import io
4
+import time
5
from collections.abc import AsyncGenerator
5
-from typing import Optional, Union, cast
6
+from math import ceil
7
+from typing import Final, Optional, Union, cast
8
9
from fastapi import Request
10
11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.logger import RequestLogger
12
-from vllm.entrypoints.openai.protocol import (ErrorResponse,
13
- RequestResponseMetadata,
14
- TranscriptionRequest,
15
- TranscriptionResponse,
16
- TranscriptionResponseVerbose)
14
+from vllm.entrypoints.openai.protocol import (
15
+ DeltaMessage, ErrorResponse, RequestResponseMetadata, TranscriptionRequest,
16
+ TranscriptionResponse, TranscriptionResponseStreamChoice,
17
+ TranscriptionStreamResponse, UsageInfo)
18
from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
20
from vllm.inputs.data import PromptType
21
from vllm.logger import init_logger
22
from vllm.outputs import RequestOutput
23
+from vllm.transformers_utils.processor import cached_get_processor
24
from vllm.utils import PlaceholderModule
25
26
try:
@@ -140,8 +142,6 @@ ISO639_1_OTHER_LANGS = {
142
# As per https://platform.openai.com/docs/guides/speech-to-text#overview.
143
# TODO configurable
144
MAX_AUDIO_CLIP_FILESIZE_MB = 25
143
-# TODO get from processor.feature_extractor.chunk_length
144
-MAX_AUDIO_CLIP_DURATION_S = 30
145
146
147
class OpenAIServingTranscription(OpenAIServing):
@@ -163,6 +163,11 @@ class OpenAIServingTranscription(OpenAIServing):
163
164
self.default_sampling_params = (
165
self.model_config.get_diff_sampling_param())
166
+ processor = cached_get_processor(model_config.model)
167
+ self.max_audio_clip_s = processor.feature_extractor.chunk_length
168
+ self.model_sr = processor.feature_extractor.sampling_rate
169
+ self.hop_length = processor.feature_extractor.hop_length
170
+
171
if self.default_sampling_params:
172
logger.info(
173
"Overwriting default completion sampling param with: %s",
@@ -172,7 +177,7 @@ class OpenAIServingTranscription(OpenAIServing):
177
self,
178
request: TranscriptionRequest,
179
audio_data: bytes,
175
- ) -> PromptType:
180
+ ) -> tuple[PromptType, float]:
181
# Validate request
182
# TODO language should be optional and can be guessed.
183
# For now we default to en. See
@@ -198,9 +203,11 @@ class OpenAIServingTranscription(OpenAIServing):
203
204
with io.BytesIO(audio_data) as bytes_:
205
y, sr = librosa.load(bytes_)
201
- if librosa.get_duration(y=y, sr=sr) > MAX_AUDIO_CLIP_DURATION_S:
206
+
207
+ duration = librosa.get_duration(y=y, sr=sr)
208
+ if duration > self.max_audio_clip_s:
209
raise ValueError(
203
- f"Maximum clip duration ({MAX_AUDIO_CLIP_DURATION_S}s) "
210
+ f"Maximum clip duration ({self.max_audio_clip_s}s) "
211
"exceeded.")
212
213
prompt = {
@@ -213,13 +220,13 @@ class OpenAIServingTranscription(OpenAIServing):
220
"decoder_prompt":
221
f"<|startoftranscript|>{lang_token}<|transcribe|><|notimestamps|>{request.prompt}"
222
}
216
- return cast(PromptType, prompt)
223
+ return cast(PromptType, prompt), duration
224
225
# TODO (varun) : Make verbose response work !
226
async def create_transcription(
227
self, audio_data: bytes, request: TranscriptionRequest,
228
raw_request: Request
222
- ) -> Union[TranscriptionResponse, TranscriptionResponseVerbose,
229
+ ) -> Union[TranscriptionResponse, AsyncGenerator[str, None],
230
ErrorResponse]:
231
"""Transcription API similar to OpenAI's API.
232
@@ -240,8 +247,7 @@ class OpenAIServingTranscription(OpenAIServing):
247
return self.create_error_response(
248
"Currently only support response_format `text` or `json`")
249
243
- # TODO cmpl->transcription?
244
- request_id = f"cmpl-{self._base_request_id(raw_request)}"
250
+ request_id = f"trsc-{self._base_request_id(raw_request)}"
251
252
request_metadata = RequestResponseMetadata(request_id=request_id)
253
if raw_request:
@@ -261,7 +267,7 @@ class OpenAIServingTranscription(OpenAIServing):
267
"Currently do not support PromptAdapter for Transcription."
268
)
269
264
- prompt = await self._preprocess_transcription(
270
+ prompt, duration_s = await self._preprocess_transcription(
271
request=request,
272
audio_data=audio_data,
273
)
@@ -293,7 +299,12 @@ class OpenAIServingTranscription(OpenAIServing):
299
# TODO: Use a vllm-specific Validation Error
300
return self.create_error_response(str(e))
301
296
- # TODO(rob): figure out a way to pipe streaming in.
302
+ if request.stream:
303
+ return self.transcription_stream_generator(request,
304
+ result_generator,
305
+ request_id,
306
+ request_metadata,
307
+ duration_s)
308
# Non-streaming response.
309
try:
310
assert result_generator is not None
@@ -305,3 +316,106 @@ class OpenAIServingTranscription(OpenAIServing):
316
except ValueError as e:
317
# TODO: Use a vllm-specific Validation Error
318
return self.create_error_response(str(e))
319
+
320
+ async def transcription_stream_generator(
321
+ self, request: TranscriptionRequest,
322
+ result_generator: AsyncGenerator[RequestOutput, None],
323
+ request_id: str, request_metadata: RequestResponseMetadata,
324
+ audio_duration_s: float) -> AsyncGenerator[str, None]:
325
+ created_time = int(time.time())
326
+ model_name = request.model
327
+ chunk_object_type: Final = "transcription.chunk"
328
+
329
+ completion_tokens = 0
330
+ num_prompt_tokens = 0
331
+
332
+ include_usage = request.stream_include_usage \
333
+ if request.stream_include_usage else False
334
+ include_continuous_usage = request.stream_continuous_usage_stats\
335
+ if include_usage and request.stream_continuous_usage_stats\
336
+ else False
337
+
338
+ try:
339
+ async for res in result_generator:
340
+ # On first result.
341
+ if res.prompt_token_ids is not None:
342
+ # Do not account the 4-tokens `<|startoftranscript|>..`
343
+ # Could be negative when language token is not specified.
344
+ num_prompt_tokens = max(len(res.prompt_token_ids) - 4, 0)
345
+ # NOTE(NickLucche) user can't pass encoder prompts directly
346
+ # at least not to Whisper. One indicator of the encoder
347
+ # amount of processing is the log-mel spectogram length.
348
+ num_prompt_tokens += ceil(audio_duration_s *
349
+ self.model_sr / self.hop_length)
350
+
351
+ # We need to do it here, because if there are exceptions in
352
+ # the result_generator, it needs to be sent as the FIRST
353
+ # response (by the try...catch).
354
+
355
+ # Just one output (n=1) supported.
356
+ assert len(res.outputs) == 1
357
+ output = res.outputs[0]
358
+
359
+ delta_message = DeltaMessage(content=output.text)
360
+ completion_tokens += len(output.token_ids)
361
+
362
+ if output.finish_reason is None:
363
+ # Still generating, send delta update.
364
+ choice_data = TranscriptionResponseStreamChoice(
365
+ delta=delta_message)
366
+ else:
367
+ # Model is finished generating.
368
+ choice_data = TranscriptionResponseStreamChoice(
369
+ delta=delta_message,
370
+ finish_reason=output.finish_reason,
371
+ stop_reason=output.stop_reason)
372
+
373
+ chunk = TranscriptionStreamResponse(id=request_id,
374
+ object=chunk_object_type,
375
+ created=created_time,
376
+ choices=[choice_data],
377
+ model=model_name)
378
+
379
+ # handle usage stats if requested & if continuous
380
+ if include_continuous_usage:
381
+ chunk.usage = UsageInfo(
382
+ prompt_tokens=num_prompt_tokens,
383
+ completion_tokens=completion_tokens,
384
+ total_tokens=num_prompt_tokens + completion_tokens,
385
+ )
386
+
387
+ data = chunk.model_dump_json(exclude_unset=True)
388
+ yield f"data: {data}\n\n"
389
+
390
+ # Once the final token is handled, if stream_options.include_usage
391
+ # is sent, send the usage.
392
+ if include_usage:
393
+ final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
394
+ completion_tokens=completion_tokens,
395
+ total_tokens=num_prompt_tokens +
396
+ completion_tokens)
397
+
398
+ final_usage_chunk = TranscriptionStreamResponse(
399
+ id=request_id,
400
+ object=chunk_object_type,
401
+ created=created_time,
402
+ choices=[],
403
+ model=model_name,
404
+ usage=final_usage)
405
+ final_usage_data = (final_usage_chunk.model_dump_json(
406
+ exclude_unset=True, exclude_none=True))
407
+ yield f"data: {final_usage_data}\n\n"
408
+
409
+ # report to FastAPI middleware aggregate usage across all choices
410
+ request_metadata.final_usage_info = UsageInfo(
411
+ prompt_tokens=num_prompt_tokens,
412
+ completion_tokens=completion_tokens,
413
+ total_tokens=num_prompt_tokens + completion_tokens)
414
+
415
+ except Exception as e:
416
+ # TODO: Use a vllm-specific Validation Error
417
+ logger.exception("Error in chat completion stream generator.")
418
+ data = self.create_streaming_error_response(str(e))
419
+ yield f"data: {data}\n\n"
420
+ # Send the final done message after all response.n are finished
421
+ yield "data: [DONE]\n\n"