A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 21:30 UTC
[Frontend] Consolidate Speech to Text entrypoints. (#42370)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
wang.yuqi committed
May 12, 2026 at 15:06 UTC
d37e25ffbed5fb5cb84123ab77cf361d5f91edef
26 files changed
+681
-607
.github/CODEOWNERS
+2
-2
@@ -31,8 +31,8 @@
31
/vllm/entrypoints/cli @hmellor @mgoin @DarkLight1337 @russellb
32
/vllm/entrypoints/mcp @heheda12345
33
/vllm/entrypoints/openai @aarnphm @chaunceyjiang @DarkLight1337 @russellb
34
-/vllm/entrypoints/openai/realtime @njhill
35
-/vllm/entrypoints/openai/speech_to_text @NickLucche
34
+/vllm/entrypoints/speech_to_text/realtime @njhill
35
+/vllm/entrypoints/speech_to_text @NickLucche
36
/vllm/entrypoints/pooling @noooop
37
/vllm/entrypoints/sagemaker @DarkLight1337
38
/vllm/entrypoints/serve @njhill
tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py
+2
-2
@@ -7,8 +7,8 @@ from unittest.mock import AsyncMock, Mock
7
8
import pytest
9
10
-from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionResponse
11
-from vllm.entrypoints.openai.speech_to_text.speech_to_text import OpenAISpeechToText
10
+from vllm.entrypoints.speech_to_text.base.serving import OpenAISpeechToText
11
+from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse
12
13
14
async def _never_finishes():
tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py
+5
-3
@@ -24,12 +24,14 @@ from vllm.entrypoints.openai.engine.protocol import (
24
RequestResponseMetadata,
25
)
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
-from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionRequest
28
-from vllm.entrypoints.openai.speech_to_text.serving import OpenAIServingTranscription
29
-from vllm.entrypoints.openai.speech_to_text.speech_to_text import (
27
+from vllm.entrypoints.speech_to_text.base.serving import (
28
OpenAISpeechToText,
29
asr_inter_chunk_separator,
30
)
31
+from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
32
+from vllm.entrypoints.speech_to_text.transcription.serving import (
33
+ OpenAIServingTranscription,
34
+)
35
from vllm.model_executor.models.interfaces import SupportsTranscription
36
from vllm.outputs import CompletionOutput, RequestOutput
37
vllm/entrypoints/openai/api_server.py
+10
-24
@@ -233,19 +233,12 @@ def build_app(
233
234
attach_render_router(app)
235
236
- if "transcription" in supported_tasks:
237
- from vllm.entrypoints.openai.speech_to_text.api_router import (
238
- attach_router as register_speech_to_text_api_router,
236
+ if "transcription" in supported_tasks or "realtime" in supported_tasks:
237
+ from vllm.entrypoints.speech_to_text.factories import (
238
+ register_speech_to_text_api_routers,
239
)
240
241
- register_speech_to_text_api_router(app)
242
-
243
- if "realtime" in supported_tasks:
244
- from vllm.entrypoints.openai.realtime.api_router import (
245
- attach_router as register_realtime_api_router,
246
- )
247
-
248
- register_realtime_api_router(app)
241
+ register_speech_to_text_api_routers(app, supported_tasks)
242
243
if any(task in POOLING_TASKS for task in supported_tasks):
244
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
@@ -284,11 +277,11 @@ def build_app(
277
278
if "realtime" in supported_tasks:
279
# Add WebSocket metrics middleware
287
- from vllm.entrypoints.openai.realtime.metrics import (
288
- WebSocketMetricsMiddleware,
280
+ from vllm.entrypoints.speech_to_text.factories import (
281
+ add_websocket_metrics_middleware,
282
)
283
291
- app.add_middleware(WebSocketMetricsMiddleware)
284
+ add_websocket_metrics_middleware(app)
285
286
if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
287
logger.warning(
@@ -421,20 +414,13 @@ async def init_app_state(
414
415
await init_generative_scoring_state(engine_client, state, args, request_logger)
416
424
- if "transcription" in supported_tasks:
425
- from vllm.entrypoints.openai.speech_to_text.api_router import (
426
- init_transcription_state,
427
- )
417
+ if "transcription" in supported_tasks or "realtime" in supported_tasks:
418
+ from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
419
429
- init_transcription_state(
420
+ init_speech_to_text_state(
421
engine_client, state, args, request_logger, supported_tasks
422
)
423
433
- if "realtime" in supported_tasks:
434
- from vllm.entrypoints.openai.realtime.api_router import init_realtime_state
435
-
436
- init_realtime_state(engine_client, state, args, request_logger, supported_tasks)
437
-
424
if any(task in POOLING_TASKS for task in supported_tasks):
425
from vllm.entrypoints.pooling.factories import init_pooling_state
426
vllm/entrypoints/openai/engine/serving.py
+5
-5
@@ -39,11 +39,6 @@ from vllm.entrypoints.openai.engine.protocol import (
39
)
40
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
41
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
42
-from vllm.entrypoints.openai.speech_to_text.protocol import (
43
- TranscriptionRequest,
44
- TranscriptionResponse,
45
- TranslationRequest,
46
-)
42
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
43
from vllm.entrypoints.serve.tokenize.protocol import (
44
DetokenizeRequest,
@@ -51,6 +46,11 @@ from vllm.entrypoints.serve.tokenize.protocol import (
46
TokenizeCompletionRequest,
47
TokenizeResponse,
48
)
49
+from vllm.entrypoints.speech_to_text.transcription.protocol import (
50
+ TranscriptionRequest,
51
+ TranscriptionResponse,
52
+)
53
+from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
54
from vllm.entrypoints.utils import create_error_response
55
from vllm.inputs import EngineInput, PromptType
56
from vllm.logger import init_logger
vllm/entrypoints/openai/run_batch.py
+10
-8
@@ -41,14 +41,6 @@ from vllm.entrypoints.openai.engine.protocol import (
41
ErrorResponse,
42
OpenAIBaseModel,
43
)
44
-from vllm.entrypoints.openai.speech_to_text.protocol import (
45
- TranscriptionRequest,
46
- TranscriptionResponse,
47
- TranscriptionResponseVerbose,
48
- TranslationRequest,
49
- TranslationResponse,
50
- TranslationResponseVerbose,
51
-)
44
from vllm.entrypoints.pooling.embed.protocol import (
45
EmbeddingRequest,
46
EmbeddingResponse,
@@ -59,6 +51,16 @@ from vllm.entrypoints.pooling.scoring.protocol import (
51
ScoreRequest,
52
ScoreResponse,
53
)
54
+from vllm.entrypoints.speech_to_text.transcription.protocol import (
55
+ TranscriptionRequest,
56
+ TranscriptionResponse,
57
+ TranscriptionResponseVerbose,
58
+)
59
+from vllm.entrypoints.speech_to_text.translation.protocol import (
60
+ TranslationRequest,
61
+ TranslationResponse,
62
+ TranslationResponseVerbose,
63
+)
64
from vllm.entrypoints.utils import create_error_response
65
from vllm.exceptions import VLLMValidationError
66
from vllm.logger import init_logger
vllm/entrypoints/openai/speech_to_text/api_router.py
deleted
-148
@@ -1,148 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-
5
-from http import HTTPStatus
6
-from typing import TYPE_CHECKING, Annotated
7
-
8
-from fastapi import APIRouter, FastAPI, Form, Request
9
-from fastapi.responses import JSONResponse, StreamingResponse
10
-
11
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12
-from vllm.entrypoints.openai.speech_to_text.protocol import (
13
- TranscriptionRequest,
14
- TranscriptionResponseVariant,
15
- TranslationRequest,
16
- TranslationResponseVariant,
17
-)
18
-from vllm.entrypoints.openai.speech_to_text.serving import (
19
- OpenAIServingTranscription,
20
- OpenAIServingTranslation,
21
-)
22
-from vllm.entrypoints.utils import (
23
- load_aware_call,
24
- with_cancellation,
25
-)
26
-from vllm.logger import init_logger
27
-
28
-if TYPE_CHECKING:
29
- from argparse import Namespace
30
-
31
- from starlette.datastructures import State
32
-
33
- from vllm.engine.protocol import EngineClient
34
- from vllm.entrypoints.logger import RequestLogger
35
- from vllm.tasks import SupportedTask
36
-else:
37
- RequestLogger = object
38
-
39
-logger = init_logger(__name__)
40
-
41
-router = APIRouter()
42
-
43
-
44
-def transcription(request: Request) -> OpenAIServingTranscription:
45
- return request.app.state.openai_serving_transcription
46
-
47
-
48
-def translation(request: Request) -> OpenAIServingTranslation:
49
- return request.app.state.openai_serving_translation
50
-
51
-
52
-@router.post(
53
- "/v1/audio/transcriptions",
54
- responses={
55
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
56
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
57
- HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
58
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
59
- },
60
-)
61
-@with_cancellation
62
-@load_aware_call
63
-async def create_transcriptions(
64
- raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
65
-):
66
- handler = transcription(raw_request)
67
- if handler is None:
68
- raise NotImplementedError("The model does not support Transcriptions API")
69
-
70
- audio_data = await request.file.read()
71
-
72
- generator = await handler.create_transcription(audio_data, request, raw_request)
73
-
74
- if isinstance(generator, ErrorResponse):
75
- return JSONResponse(
76
- content=generator.model_dump(), status_code=generator.error.code
77
- )
78
-
79
- elif isinstance(generator, TranscriptionResponseVariant):
80
- return JSONResponse(content=generator.model_dump())
81
-
82
- return StreamingResponse(content=generator, media_type="text/event-stream")
83
-
84
-
85
-@router.post(
86
- "/v1/audio/translations",
87
- responses={
88
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
89
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
90
- HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
91
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
92
- },
93
-)
94
-@with_cancellation
95
-@load_aware_call
96
-async def create_translations(
97
- request: Annotated[TranslationRequest, Form()], raw_request: Request
98
-):
99
- handler = translation(raw_request)
100
- if handler is None:
101
- raise NotImplementedError("The model does not support Translations API")
102
-
103
- audio_data = await request.file.read()
104
-
105
- generator = await handler.create_translation(audio_data, request, raw_request)
106
-
107
- if isinstance(generator, ErrorResponse):
108
- return JSONResponse(
109
- content=generator.model_dump(), status_code=generator.error.code
110
- )
111
-
112
- elif isinstance(generator, TranslationResponseVariant):
113
- return JSONResponse(content=generator.model_dump())
114
-
115
- return StreamingResponse(content=generator, media_type="text/event-stream")
116
-
117
-
118
-def attach_router(app: FastAPI):
119
- app.include_router(router)
120
-
121
-
122
-def init_transcription_state(
123
- engine_client: "EngineClient",
124
- state: "State",
125
- args: "Namespace",
126
- request_logger: RequestLogger | None,
127
- supported_tasks: tuple["SupportedTask", ...],
128
-):
129
- state.openai_serving_transcription = (
130
- OpenAIServingTranscription(
131
- engine_client,
132
- state.openai_serving_models,
133
- request_logger=request_logger,
134
- enable_force_include_usage=args.enable_force_include_usage,
135
- )
136
- if "transcription" in supported_tasks
137
- else None
138
- )
139
- state.openai_serving_translation = (
140
- OpenAIServingTranslation(
141
- engine_client,
142
- state.openai_serving_models,
143
- request_logger=request_logger,
144
- enable_force_include_usage=args.enable_force_include_usage,
145
- )
146
- if "transcription" in supported_tasks
147
- else None
148
- )
vllm/entrypoints/speech_to_text/__init__.py
renamed
vllm/entrypoints/speech_to_text/base/__init__.py
vllm/entrypoints/speech_to_text/base/protocol.py
new
+11
@@ -0,0 +1,11 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from typing import Literal, TypeAlias
6
+
7
+import torch
8
+
9
+## Protocols for Audio
10
+AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
11
+_LONG_INFO = torch.iinfo(torch.long)
vllm/entrypoints/speech_to_text/base/serving.py
renamed
+15
-12
@@ -24,18 +24,6 @@ from vllm.entrypoints.openai.engine.protocol import (
24
)
25
from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
26
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
-from vllm.entrypoints.openai.speech_to_text.protocol import (
28
- TranscriptionResponse,
29
- TranscriptionResponseStreamChoice,
30
- TranscriptionResponseVerbose,
31
- TranscriptionSegment,
32
- TranscriptionStreamResponse,
33
- TranslationResponse,
34
- TranslationResponseStreamChoice,
35
- TranslationResponseVerbose,
36
- TranslationSegment,
37
- TranslationStreamResponse,
38
-)
27
from vllm.entrypoints.utils import get_max_tokens
28
from vllm.exceptions import VLLMValidationError
29
from vllm.inputs import EncoderDecoderInput, EngineInput
@@ -51,6 +39,21 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
39
from vllm.tokenizers import get_tokenizer
40
from vllm.utils.async_utils import merge_async_iterators
41
42
+from ..transcription.protocol import (
43
+ TranscriptionResponse,
44
+ TranscriptionResponseStreamChoice,
45
+ TranscriptionResponseVerbose,
46
+ TranscriptionSegment,
47
+ TranscriptionStreamResponse,
48
+)
49
+from ..translation.protocol import (
50
+ TranslationResponse,
51
+ TranslationResponseStreamChoice,
52
+ TranslationResponseVerbose,
53
+ TranslationSegment,
54
+ TranslationStreamResponse,
55
+)
56
+
57
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
58
SpeechToTextResponseVerbose: TypeAlias = (
59
TranscriptionResponseVerbose | TranslationResponseVerbose
vllm/entrypoints/speech_to_text/factories.py
new
+79
@@ -0,0 +1,79 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from typing import TYPE_CHECKING
6
+
7
+from fastapi import FastAPI
8
+
9
+if TYPE_CHECKING:
10
+ from argparse import Namespace
11
+
12
+ from starlette.datastructures import State
13
+
14
+ from vllm.engine.protocol import EngineClient
15
+ from vllm.entrypoints.logger import RequestLogger
16
+ from vllm.tasks import SupportedTask
17
+else:
18
+ RequestLogger = object
19
+
20
+
21
+def register_speech_to_text_api_routers(
22
+ app: FastAPI,
23
+ supported_tasks: tuple["SupportedTask", ...],
24
+):
25
+ if "realtime" in supported_tasks:
26
+ from .realtime.api_router import router as realtime_router
27
+
28
+ app.include_router(realtime_router)
29
+
30
+ if "transcription" in supported_tasks:
31
+ from .transcription.api_router import router as transcription_router
32
+
33
+ app.include_router(transcription_router)
34
+
35
+ from .translation.api_router import router as translation_router
36
+
37
+ app.include_router(translation_router)
38
+
39
+
40
+def add_websocket_metrics_middleware(app: FastAPI):
41
+ from .realtime.metrics import WebSocketMetricsMiddleware
42
+
43
+ app.add_middleware(WebSocketMetricsMiddleware)
44
+
45
+
46
+def init_speech_to_text_state(
47
+ engine_client: "EngineClient",
48
+ state: "State",
49
+ args: "Namespace",
50
+ request_logger: RequestLogger | None,
51
+ supported_tasks: tuple["SupportedTask", ...],
52
+):
53
+ if "transcription" in supported_tasks:
54
+ from .transcription.serving import OpenAIServingTranscription
55
+
56
+ state.openai_serving_transcription = OpenAIServingTranscription(
57
+ engine_client,
58
+ state.openai_serving_models,
59
+ request_logger=request_logger,
60
+ enable_force_include_usage=args.enable_force_include_usage,
61
+ )
62
+
63
+ from .translation.serving import OpenAIServingTranslation
64
+
65
+ state.openai_serving_translation = OpenAIServingTranslation(
66
+ engine_client,
67
+ state.openai_serving_models,
68
+ request_logger=request_logger,
69
+ enable_force_include_usage=args.enable_force_include_usage,
70
+ )
71
+
72
+ if "realtime" in supported_tasks:
73
+ from .realtime.serving import OpenAIServingRealtime
74
+
75
+ state.openai_serving_realtime = OpenAIServingRealtime(
76
+ engine_client,
77
+ state.openai_serving_models,
78
+ request_logger=request_logger,
79
+ )
vllm/entrypoints/speech_to_text/realtime/__init__.py
renamed
vllm/entrypoints/speech_to_text/realtime/api_router.py
renamed
+3
-38
@@ -1,26 +1,15 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from typing import TYPE_CHECKING
4
6
-from fastapi import APIRouter, FastAPI, WebSocket
5
+from fastapi import APIRouter, WebSocket
6
8
-from vllm.entrypoints.openai.realtime.connection import RealtimeConnection
9
-from vllm.entrypoints.openai.realtime.serving import OpenAIServingRealtime
7
from vllm.logger import init_logger
8
12
-logger = init_logger(__name__)
13
-
14
-if TYPE_CHECKING:
15
- from argparse import Namespace
9
+from .connection import RealtimeConnection
10
17
- from starlette.datastructures import State
11
+logger = init_logger(__name__)
12
19
- from vllm.engine.protocol import EngineClient
20
- from vllm.entrypoints.logger import RequestLogger
21
- from vllm.tasks import SupportedTask
22
-else:
23
- RequestLogger = object
13
14
router = APIRouter()
15
@@ -48,27 +37,3 @@ async def realtime_endpoint(websocket: WebSocket):
37
38
connection = RealtimeConnection(websocket, serving)
39
await connection.handle_connection()
51
-
52
-
53
-def attach_router(app: FastAPI):
54
- """Attach the realtime router to the FastAPI app."""
55
- app.include_router(router)
56
- logger.info("Realtime API router attached")
57
-
58
-
59
-def init_realtime_state(
60
- engine_client: "EngineClient",
61
- state: "State",
62
- args: "Namespace",
63
- request_logger: RequestLogger | None,
64
- supported_tasks: tuple["SupportedTask", ...],
65
-):
66
- state.openai_serving_realtime = (
67
- OpenAIServingRealtime(
68
- engine_client,
69
- state.openai_serving_models,
70
- request_logger=request_logger,
71
- )
72
- if "realtime" in supported_tasks
73
- else None
74
- )
vllm/entrypoints/speech_to_text/realtime/connection.py
renamed
+5
-4
@@ -14,7 +14,10 @@ from starlette.websockets import WebSocketDisconnect
14
15
from vllm import envs
16
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
17
-from vllm.entrypoints.openai.realtime.protocol import (
17
+from vllm.exceptions import VLLMValidationError
18
+from vllm.logger import init_logger
19
+
20
+from .protocol import (
21
ErrorEvent,
22
InputAudioBufferAppend,
23
InputAudioBufferCommit,
@@ -22,9 +25,7 @@ from vllm.entrypoints.openai.realtime.protocol import (
25
TranscriptionDelta,
26
TranscriptionDone,
27
)
25
-from vllm.entrypoints.openai.realtime.serving import OpenAIServingRealtime
26
-from vllm.exceptions import VLLMValidationError
27
-from vllm.logger import init_logger
28
+from .serving import OpenAIServingRealtime
29
30
logger = init_logger(__name__)
31
vllm/entrypoints/speech_to_text/realtime/metrics.py
renamed
vllm/entrypoints/speech_to_text/realtime/protocol.py
renamed
vllm/entrypoints/speech_to_text/realtime/serving.py
renamed
vllm/entrypoints/speech_to_text/transcription/__init__.py
vllm/entrypoints/speech_to_text/transcription/api_router.py
new
+60
@@ -0,0 +1,60 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+from typing import Annotated
7
+
8
+from fastapi import APIRouter, Form, Request
9
+from fastapi.responses import JSONResponse, StreamingResponse
10
+
11
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12
+from vllm.entrypoints.utils import (
13
+ load_aware_call,
14
+ with_cancellation,
15
+)
16
+from vllm.logger import init_logger
17
+
18
+from .protocol import TranscriptionRequest, TranscriptionResponseVariant
19
+from .serving import OpenAIServingTranscription
20
+
21
+logger = init_logger(__name__)
22
+
23
+router = APIRouter()
24
+
25
+
26
+def transcription(request: Request) -> OpenAIServingTranscription:
27
+ return request.app.state.openai_serving_transcription
28
+
29
+
30
+@router.post(
31
+ "/v1/audio/transcriptions",
32
+ responses={
33
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
34
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
35
+ HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
36
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
37
+ },
38
+)
39
+@with_cancellation
40
+@load_aware_call
41
+async def create_transcriptions(
42
+ raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
43
+):
44
+ handler = transcription(raw_request)
45
+ if handler is None:
46
+ raise NotImplementedError("The model does not support Transcriptions API")
47
+
48
+ audio_data = await request.file.read()
49
+
50
+ generator = await handler.create_transcription(audio_data, request, raw_request)
51
+
52
+ if isinstance(generator, ErrorResponse):
53
+ return JSONResponse(
54
+ content=generator.model_dump(), status_code=generator.error.code
55
+ )
56
+
57
+ elif isinstance(generator, TranscriptionResponseVariant):
58
+ return JSONResponse(content=generator.model_dump())
59
+
60
+ return StreamingResponse(content=generator, media_type="text/event-stream")
vllm/entrypoints/speech_to_text/transcription/protocol.py
renamed
+2
-279
@@ -6,7 +6,6 @@ import time
6
from http import HTTPStatus
7
from typing import TYPE_CHECKING, Literal, TypeAlias
8
9
-import torch
9
from fastapi import HTTPException, UploadFile
10
from pydantic import (
11
Field,
@@ -28,13 +27,14 @@ from vllm.sampling_params import (
27
)
28
from vllm.utils import random_uuid
29
30
+from ..base.protocol import _LONG_INFO, AudioResponseFormat
31
+
32
if TYPE_CHECKING:
33
import numpy as np
34
35
from vllm.config import ModelConfig, SpeechToTextConfig
36
37
logger = init_logger(__name__)
37
-_LONG_INFO = torch.iinfo(torch.long)
38
39
40
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
@@ -52,10 +52,6 @@ class TranscriptionStreamResponse(OpenAIBaseModel):
52
usage: UsageInfo | None = Field(default=None)
53
54
55
-## Protocols for Audio
56
-AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
57
-
58
-
55
class TranscriptionRequest(OpenAIBaseModel):
56
# Ordered by official OpenAI API documentation
57
# https://platform.openai.com/docs/api-reference/audio/createTranscription
@@ -407,276 +403,3 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):
403
TranscriptionResponseVariant: TypeAlias = (
404
TranscriptionResponse | TranscriptionResponseVerbose
405
)
410
-
411
-
412
-class TranslationResponseStreamChoice(OpenAIBaseModel):
413
- delta: DeltaMessage
414
- finish_reason: str | None = None
415
- stop_reason: int | str | None = None
416
-
417
-
418
-class TranslationStreamResponse(OpenAIBaseModel):
419
- id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
420
- object: Literal["translation.chunk"] = "translation.chunk"
421
- created: int = Field(default_factory=lambda: int(time.time()))
422
- model: str
423
- choices: list[TranslationResponseStreamChoice]
424
- usage: UsageInfo | None = Field(default=None)
425
-
426
-
427
-class TranslationRequest(OpenAIBaseModel):
428
- # Ordered by official OpenAI API documentation
429
- # https://platform.openai.com/docs/api-reference/audio/createTranslation
430
-
431
- file: UploadFile
432
- """
433
- The audio file object (not file name) to translate, in one of these
434
- formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
435
- """
436
-
437
- model: str | None = None
438
- """ID of the model to use.
439
- """
440
-
441
- prompt: str = Field(default="")
442
- """An optional text to guide the model's style or continue a previous audio
443
- segment.
444
-
445
- The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
446
- should match the audio language.
447
- """
448
-
449
- response_format: AudioResponseFormat = Field(default="json")
450
- """
451
- The format of the output, in one of these options: `json`, `text`, `srt`,
452
- `verbose_json`, or `vtt`.
453
- """
454
-
455
- # TODO support additional sampling parameters
456
- # --8<-- [start:translation-sampling-params]
457
- use_beam_search: bool = False
458
- """Whether or not beam search should be used."""
459
-
460
- n: int = 1
461
- """The number of beams to be used in beam search."""
462
-
463
- length_penalty: float = 1.0
464
- """Length penalty to be used for beam search."""
465
-
466
- include_stop_str_in_output: bool = False
467
- """Whether to include the stop strings in output text."""
468
-
469
- seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
470
- """The seed to use for sampling."""
471
-
472
- temperature: float = Field(default=0.0)
473
- """The sampling temperature, between 0 and 1.
474
-
475
- Higher values like 0.8 will make the output more random, while lower values
476
- like 0.2 will make it more focused / deterministic. If set to 0, the model
477
- will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
478
- to automatically increase the temperature until certain thresholds are hit.
479
- """
480
- # --8<-- [end:translation-sampling-params]
481
-
482
- # --8<-- [start:translation-extra-params]
483
- language: str | None = None
484
- """The language of the input audio we translate from.
485
-
486
- Supplying the input language in
487
- [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
488
- will improve accuracy.
489
- """
490
-
491
- hotwords: str | None = None
492
- """
493
- hotwords refers to a list of important words or phrases that the model
494
- should pay extra attention to during transcription.
495
- """
496
-
497
- to_language: str | None = None
498
- """The language of the input audio we translate to.
499
-
500
- Please note that this is not supported by all models, refer to the specific
501
- model documentation for more details.
502
- For instance, Whisper only supports `to_language=en`.
503
- """
504
-
505
- stream: bool | None = False
506
- """Custom field not present in the original OpenAI definition. When set,
507
- it will enable output to be streamed in a similar fashion as the Chat
508
- Completion endpoint.
509
- """
510
- # Flattened stream option to simplify form data.
511
- stream_include_usage: bool | None = False
512
- stream_continuous_usage_stats: bool | None = False
513
-
514
- max_completion_tokens: int | None = None
515
- """The maximum number of tokens to generate."""
516
- # --8<-- [end:translation-extra-params]
517
-
518
- # Default sampling parameters for translation requests.
519
- _DEFAULT_SAMPLING_PARAMS: dict = {
520
- "temperature": 0,
521
- }
522
-
523
- def build_stt_params(
524
- self,
525
- audio: "np.ndarray",
526
- stt_config: "SpeechToTextConfig",
527
- model_config: "ModelConfig",
528
- task_type: str,
529
- ) -> SpeechToTextParams:
530
- return SpeechToTextParams(
531
- audio=audio,
532
- stt_config=stt_config,
533
- model_config=model_config,
534
- language=self.language,
535
- task_type=task_type,
536
- request_prompt=self.prompt,
537
- to_language=self.to_language,
538
- hotwords=self.hotwords,
539
- )
540
-
541
- def to_beam_search_params(
542
- self,
543
- default_max_tokens: int,
544
- default_sampling_params: dict | None = None,
545
- ) -> BeamSearchParams:
546
- if default_sampling_params is None:
547
- default_sampling_params = {}
548
-
549
- max_tokens = default_max_tokens
550
- n = self.n if self.n is not None else 1
551
-
552
- # NOTE: Temp 0 is a different fallback than completions
553
- if (temperature := self.temperature) is None:
554
- temperature = default_sampling_params.get("temperature", 0)
555
-
556
- return BeamSearchParams(
557
- beam_width=n,
558
- max_tokens=max_tokens,
559
- temperature=temperature,
560
- length_penalty=self.length_penalty,
561
- include_stop_str_in_output=self.include_stop_str_in_output,
562
- )
563
-
564
- def to_sampling_params(
565
- self, default_max_tokens: int, default_sampling_params: dict | None = None
566
- ) -> SamplingParams:
567
- max_tokens = default_max_tokens
568
-
569
- if default_sampling_params is None:
570
- default_sampling_params = {}
571
- # Default parameters
572
- if (temperature := self.temperature) is None:
573
- temperature = default_sampling_params.get(
574
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
575
- )
576
-
577
- return SamplingParams.from_optional(
578
- temperature=temperature,
579
- max_tokens=max_tokens,
580
- seed=self.seed,
581
- output_kind=RequestOutputKind.DELTA
582
- if self.stream
583
- else RequestOutputKind.FINAL_ONLY,
584
- skip_clone=True, # Created fresh per request, safe to skip clone
585
- )
586
-
587
- @model_validator(mode="before")
588
- @classmethod
589
- def validate_stream_options(cls, data):
590
- stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
591
- stream = data.get("stream", False)
592
- if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
593
- # Find which specific stream option was set
594
- invalid_param = next(
595
- (so for so in stream_opts if data.get(so, False)),
596
- "stream_include_usage",
597
- )
598
- raise VLLMValidationError(
599
- "Stream options can only be defined when `stream=True`.",
600
- parameter=invalid_param,
601
- )
602
-
603
- return data
604
-
605
-
606
-# Translation response objects
607
-class TranslationResponse(OpenAIBaseModel):
608
- text: str
609
- """The translated text."""
610
-
611
-
612
-class TranslationWord(OpenAIBaseModel):
613
- end: float
614
- """End time of the word in seconds."""
615
-
616
- start: float
617
- """Start time of the word in seconds."""
618
-
619
- word: str
620
- """The text content of the word."""
621
-
622
-
623
-class TranslationSegment(OpenAIBaseModel):
624
- id: int
625
- """Unique identifier of the segment."""
626
-
627
- avg_logprob: float
628
- """Average logprob of the segment.
629
-
630
- If the value is lower than -1, consider the logprobs failed.
631
- """
632
-
633
- compression_ratio: float
634
- """Compression ratio of the segment.
635
-
636
- If the value is greater than 2.4, consider the compression failed.
637
- """
638
-
639
- end: float
640
- """End time of the segment in seconds."""
641
-
642
- no_speech_prob: float | None = None
643
- """Probability of no speech in the segment.
644
-
645
- If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
646
- this segment silent.
647
- """
648
-
649
- seek: int
650
- """Seek offset of the segment."""
651
-
652
- start: float
653
- """Start time of the segment in seconds."""
654
-
655
- temperature: float
656
- """Temperature parameter used for generating the segment."""
657
-
658
- text: str
659
- """Text content of the segment."""
660
-
661
- tokens: list[int]
662
- """Array of token IDs for the text content."""
663
-
664
-
665
-class TranslationResponseVerbose(OpenAIBaseModel):
666
- duration: str
667
- """The duration of the input audio."""
668
-
669
- language: str
670
- """The language of the input audio."""
671
-
672
- text: str
673
- """The translated text."""
674
-
675
- segments: list[TranslationSegment] | None = None
676
- """Segments of the translated text and their corresponding details."""
677
-
678
- words: list[TranslationWord] | None = None
679
- """Extracted words and their corresponding timestamps."""
680
-
681
-
682
-TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
vllm/entrypoints/speech_to_text/transcription/serving.py
renamed
+5
-82
@@ -11,21 +11,17 @@ from vllm.entrypoints.openai.engine.protocol import (
11
RequestResponseMetadata,
12
)
13
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
14
-from vllm.entrypoints.openai.speech_to_text.protocol import (
14
+from vllm.logger import init_logger
15
+from vllm.outputs import RequestOutput
16
+
17
+from ..base.serving import OpenAISpeechToText
18
+from .protocol import (
19
TranscriptionRequest,
20
TranscriptionResponse,
21
TranscriptionResponseStreamChoice,
22
TranscriptionResponseVerbose,
23
TranscriptionStreamResponse,
20
- TranslationRequest,
21
- TranslationResponse,
22
- TranslationResponseStreamChoice,
23
- TranslationResponseVerbose,
24
- TranslationStreamResponse,
24
)
26
-from vllm.entrypoints.openai.speech_to_text.speech_to_text import OpenAISpeechToText
27
-from vllm.logger import init_logger
28
-from vllm.outputs import RequestOutput
25
26
logger = init_logger(__name__)
27
@@ -101,76 +97,3 @@ class OpenAIServingTranscription(OpenAISpeechToText):
97
)
98
async for chunk in generator:
99
yield chunk
104
-
105
-
106
-class OpenAIServingTranslation(OpenAISpeechToText):
107
- """Handles translation requests."""
108
-
109
- def __init__(
110
- self,
111
- engine_client: EngineClient,
112
- models: OpenAIServingModels,
113
- *,
114
- request_logger: RequestLogger | None,
115
- return_tokens_as_token_ids: bool = False,
116
- enable_force_include_usage: bool = False,
117
- ):
118
- super().__init__(
119
- engine_client=engine_client,
120
- models=models,
121
- request_logger=request_logger,
122
- return_tokens_as_token_ids=return_tokens_as_token_ids,
123
- task_type="translate",
124
- enable_force_include_usage=enable_force_include_usage,
125
- )
126
-
127
- async def create_translation(
128
- self,
129
- audio_data: bytes,
130
- request: TranslationRequest,
131
- raw_request: Request | None = None,
132
- ) -> (
133
- TranslationResponse
134
- | TranslationResponseVerbose
135
- | AsyncGenerator[str, None]
136
- | ErrorResponse
137
- ):
138
- """Translation API similar to OpenAI's API.
139
-
140
- See https://platform.openai.com/docs/api-reference/audio/createTranslation
141
- for the API specification. This API mimics the OpenAI translation API.
142
- """
143
- return await self._create_speech_to_text(
144
- audio_data=audio_data,
145
- request=request,
146
- raw_request=raw_request,
147
- response_class=(
148
- TranslationResponseVerbose
149
- if request.response_format == "verbose_json"
150
- else TranslationResponse
151
- ),
152
- stream_generator_method=self.translation_stream_generator,
153
- )
154
-
155
- async def translation_stream_generator(
156
- self,
157
- request: TranslationRequest,
158
- result_generator: list[AsyncGenerator[RequestOutput, None]],
159
- request_id: str,
160
- request_metadata: RequestResponseMetadata,
161
- audio_duration_s: float,
162
- separator: str,
163
- ) -> AsyncGenerator[str, None]:
164
- generator = self._speech_to_text_stream_generator(
165
- request=request,
166
- list_result_generator=result_generator,
167
- request_id=request_id,
168
- request_metadata=request_metadata,
169
- audio_duration_s=audio_duration_s,
170
- chunk_object_type="translation.chunk",
171
- response_stream_choice_class=TranslationResponseStreamChoice,
172
- stream_response_class=TranslationStreamResponse,
173
- separator=separator,
174
- )
175
- async for chunk in generator:
176
- yield chunk
vllm/entrypoints/speech_to_text/translation/__init__.py
vllm/entrypoints/speech_to_text/translation/api_router.py
new
+60
@@ -0,0 +1,60 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+from typing import Annotated
7
+
8
+from fastapi import APIRouter, Form, Request
9
+from fastapi.responses import JSONResponse, StreamingResponse
10
+
11
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12
+from vllm.entrypoints.utils import (
13
+ load_aware_call,
14
+ with_cancellation,
15
+)
16
+from vllm.logger import init_logger
17
+
18
+from .protocol import TranslationRequest, TranslationResponseVariant
19
+from .serving import OpenAIServingTranslation
20
+
21
+logger = init_logger(__name__)
22
+
23
+router = APIRouter()
24
+
25
+
26
+def translation(request: Request) -> OpenAIServingTranslation:
27
+ return request.app.state.openai_serving_translation
28
+
29
+
30
+@router.post(
31
+ "/v1/audio/translations",
32
+ responses={
33
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
34
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
35
+ HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
36
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
37
+ },
38
+)
39
+@with_cancellation
40
+@load_aware_call
41
+async def create_translations(
42
+ request: Annotated[TranslationRequest, Form()], raw_request: Request
43
+):
44
+ handler = translation(raw_request)
45
+ if handler is None:
46
+ raise NotImplementedError("The model does not support Translations API")
47
+
48
+ audio_data = await request.file.read()
49
+
50
+ generator = await handler.create_translation(audio_data, request, raw_request)
51
+
52
+ if isinstance(generator, ErrorResponse):
53
+ return JSONResponse(
54
+ content=generator.model_dump(), status_code=generator.error.code
55
+ )
56
+
57
+ elif isinstance(generator, TranslationResponseVariant):
58
+ return JSONResponse(content=generator.model_dump())
59
+
60
+ return StreamingResponse(content=generator, media_type="text/event-stream")
vllm/entrypoints/speech_to_text/translation/protocol.py
new
+308
@@ -0,0 +1,308 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import time
5
+from typing import TYPE_CHECKING, Literal, TypeAlias
6
+
7
+from fastapi import UploadFile
8
+from pydantic import (
9
+ Field,
10
+ model_validator,
11
+)
12
+
13
+from vllm.config.speech_to_text import SpeechToTextParams
14
+from vllm.entrypoints.openai.engine.protocol import (
15
+ DeltaMessage,
16
+ OpenAIBaseModel,
17
+ UsageInfo,
18
+)
19
+from vllm.exceptions import VLLMValidationError
20
+from vllm.logger import init_logger
21
+from vllm.sampling_params import (
22
+ BeamSearchParams,
23
+ RequestOutputKind,
24
+ SamplingParams,
25
+)
26
+from vllm.utils import random_uuid
27
+
28
+from ..base.protocol import _LONG_INFO, AudioResponseFormat
29
+
30
+if TYPE_CHECKING:
31
+ import numpy as np
32
+
33
+ from vllm.config import ModelConfig, SpeechToTextConfig
34
+
35
+logger = init_logger(__name__)
36
+
37
+
38
+class TranslationResponseStreamChoice(OpenAIBaseModel):
39
+ delta: DeltaMessage
40
+ finish_reason: str | None = None
41
+ stop_reason: int | str | None = None
42
+
43
+
44
+class TranslationStreamResponse(OpenAIBaseModel):
45
+ id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
46
+ object: Literal["translation.chunk"] = "translation.chunk"
47
+ created: int = Field(default_factory=lambda: int(time.time()))
48
+ model: str
49
+ choices: list[TranslationResponseStreamChoice]
50
+ usage: UsageInfo | None = Field(default=None)
51
+
52
+
53
+class TranslationRequest(OpenAIBaseModel):
54
+ # Ordered by official OpenAI API documentation
55
+ # https://platform.openai.com/docs/api-reference/audio/createTranslation
56
+
57
+ file: UploadFile
58
+ """
59
+ The audio file object (not file name) to translate, in one of these
60
+ formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
61
+ """
62
+
63
+ model: str | None = None
64
+ """ID of the model to use.
65
+ """
66
+
67
+ prompt: str = Field(default="")
68
+ """An optional text to guide the model's style or continue a previous audio
69
+ segment.
70
+
71
+ The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
72
+ should match the audio language.
73
+ """
74
+
75
+ response_format: AudioResponseFormat = Field(default="json")
76
+ """
77
+ The format of the output, in one of these options: `json`, `text`, `srt`,
78
+ `verbose_json`, or `vtt`.
79
+ """
80
+
81
+ # TODO support additional sampling parameters
82
+ # --8<-- [start:translation-sampling-params]
83
+ use_beam_search: bool = False
84
+ """Whether or not beam search should be used."""
85
+
86
+ n: int = 1
87
+ """The number of beams to be used in beam search."""
88
+
89
+ length_penalty: float = 1.0
90
+ """Length penalty to be used for beam search."""
91
+
92
+ include_stop_str_in_output: bool = False
93
+ """Whether to include the stop strings in output text."""
94
+
95
+ seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
96
+ """The seed to use for sampling."""
97
+
98
+ temperature: float = Field(default=0.0)
99
+ """The sampling temperature, between 0 and 1.
100
+
101
+ Higher values like 0.8 will make the output more random, while lower values
102
+ like 0.2 will make it more focused / deterministic. If set to 0, the model
103
+ will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
104
+ to automatically increase the temperature until certain thresholds are hit.
105
+ """
106
+ # --8<-- [end:translation-sampling-params]
107
+
108
+ # --8<-- [start:translation-extra-params]
109
+ language: str | None = None
110
+ """The language of the input audio we translate from.
111
+
112
+ Supplying the input language in
113
+ [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
114
+ will improve accuracy.
115
+ """
116
+
117
+ hotwords: str | None = None
118
+ """
119
+ hotwords refers to a list of important words or phrases that the model
120
+ should pay extra attention to during transcription.
121
+ """
122
+
123
+ to_language: str | None = None
124
+ """The language of the input audio we translate to.
125
+
126
+ Please note that this is not supported by all models, refer to the specific
127
+ model documentation for more details.
128
+ For instance, Whisper only supports `to_language=en`.
129
+ """
130
+
131
+ stream: bool | None = False
132
+ """Custom field not present in the original OpenAI definition. When set,
133
+ it will enable output to be streamed in a similar fashion as the Chat
134
+ Completion endpoint.
135
+ """
136
+ # Flattened stream option to simplify form data.
137
+ stream_include_usage: bool | None = False
138
+ stream_continuous_usage_stats: bool | None = False
139
+
140
+ max_completion_tokens: int | None = None
141
+ """The maximum number of tokens to generate."""
142
+ # --8<-- [end:translation-extra-params]
143
+
144
+ # Default sampling parameters for translation requests.
145
+ _DEFAULT_SAMPLING_PARAMS: dict = {
146
+ "temperature": 0,
147
+ }
148
+
149
+ def build_stt_params(
150
+ self,
151
+ audio: "np.ndarray",
152
+ stt_config: "SpeechToTextConfig",
153
+ model_config: "ModelConfig",
154
+ task_type: str,
155
+ ) -> SpeechToTextParams:
156
+ return SpeechToTextParams(
157
+ audio=audio,
158
+ stt_config=stt_config,
159
+ model_config=model_config,
160
+ language=self.language,
161
+ task_type=task_type,
162
+ request_prompt=self.prompt,
163
+ to_language=self.to_language,
164
+ hotwords=self.hotwords,
165
+ )
166
+
167
+ def to_beam_search_params(
168
+ self,
169
+ default_max_tokens: int,
170
+ default_sampling_params: dict | None = None,
171
+ ) -> BeamSearchParams:
172
+ if default_sampling_params is None:
173
+ default_sampling_params = {}
174
+
175
+ max_tokens = default_max_tokens
176
+ n = self.n if self.n is not None else 1
177
+
178
+ # NOTE: Temp 0 is a different fallback than completions
179
+ if (temperature := self.temperature) is None:
180
+ temperature = default_sampling_params.get("temperature", 0)
181
+
182
+ return BeamSearchParams(
183
+ beam_width=n,
184
+ max_tokens=max_tokens,
185
+ temperature=temperature,
186
+ length_penalty=self.length_penalty,
187
+ include_stop_str_in_output=self.include_stop_str_in_output,
188
+ )
189
+
190
+ def to_sampling_params(
191
+ self, default_max_tokens: int, default_sampling_params: dict | None = None
192
+ ) -> SamplingParams:
193
+ max_tokens = default_max_tokens
194
+
195
+ if default_sampling_params is None:
196
+ default_sampling_params = {}
197
+ # Default parameters
198
+ if (temperature := self.temperature) is None:
199
+ temperature = default_sampling_params.get(
200
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
201
+ )
202
+
203
+ return SamplingParams.from_optional(
204
+ temperature=temperature,
205
+ max_tokens=max_tokens,
206
+ seed=self.seed,
207
+ output_kind=RequestOutputKind.DELTA
208
+ if self.stream
209
+ else RequestOutputKind.FINAL_ONLY,
210
+ skip_clone=True, # Created fresh per request, safe to skip clone
211
+ )
212
+
213
+ @model_validator(mode="before")
214
+ @classmethod
215
+ def validate_stream_options(cls, data):
216
+ stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
217
+ stream = data.get("stream", False)
218
+ if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
219
+ # Find which specific stream option was set
220
+ invalid_param = next(
221
+ (so for so in stream_opts if data.get(so, False)),
222
+ "stream_include_usage",
223
+ )
224
+ raise VLLMValidationError(
225
+ "Stream options can only be defined when `stream=True`.",
226
+ parameter=invalid_param,
227
+ )
228
+
229
+ return data
230
+
231
+
232
+# Translation response objects
233
+class TranslationResponse(OpenAIBaseModel):
234
+ text: str
235
+ """The translated text."""
236
+
237
+
238
+class TranslationWord(OpenAIBaseModel):
239
+ end: float
240
+ """End time of the word in seconds."""
241
+
242
+ start: float
243
+ """Start time of the word in seconds."""
244
+
245
+ word: str
246
+ """The text content of the word."""
247
+
248
+
249
+class TranslationSegment(OpenAIBaseModel):
250
+ id: int
251
+ """Unique identifier of the segment."""
252
+
253
+ avg_logprob: float
254
+ """Average logprob of the segment.
255
+
256
+ If the value is lower than -1, consider the logprobs failed.
257
+ """
258
+
259
+ compression_ratio: float
260
+ """Compression ratio of the segment.
261
+
262
+ If the value is greater than 2.4, consider the compression failed.
263
+ """
264
+
265
+ end: float
266
+ """End time of the segment in seconds."""
267
+
268
+ no_speech_prob: float | None = None
269
+ """Probability of no speech in the segment.
270
+
271
+ If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
272
+ this segment silent.
273
+ """
274
+
275
+ seek: int
276
+ """Seek offset of the segment."""
277
+
278
+ start: float
279
+ """Start time of the segment in seconds."""
280
+
281
+ temperature: float
282
+ """Temperature parameter used for generating the segment."""
283
+
284
+ text: str
285
+ """Text content of the segment."""
286
+
287
+ tokens: list[int]
288
+ """Array of token IDs for the text content."""
289
+
290
+
291
+class TranslationResponseVerbose(OpenAIBaseModel):
292
+ duration: str
293
+ """The duration of the input audio."""
294
+
295
+ language: str
296
+ """The language of the input audio."""
297
+
298
+ text: str
299
+ """The translated text."""
300
+
301
+ segments: list[TranslationSegment] | None = None
302
+ """Segments of the translated text and their corresponding details."""
303
+
304
+ words: list[TranslationWord] | None = None
305
+ """Extracted words and their corresponding timestamps."""
306
+
307
+
308
+TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
vllm/entrypoints/speech_to_text/translation/serving.py
new
+99
@@ -0,0 +1,99 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from collections.abc import AsyncGenerator
4
+
5
+from fastapi import Request
6
+
7
+from vllm.engine.protocol import EngineClient
8
+from vllm.entrypoints.logger import RequestLogger
9
+from vllm.entrypoints.openai.engine.protocol import (
10
+ ErrorResponse,
11
+ RequestResponseMetadata,
12
+)
13
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
14
+from vllm.logger import init_logger
15
+from vllm.outputs import RequestOutput
16
+
17
+from ..base.serving import OpenAISpeechToText
18
+from .protocol import (
19
+ TranslationRequest,
20
+ TranslationResponse,
21
+ TranslationResponseStreamChoice,
22
+ TranslationResponseVerbose,
23
+ TranslationStreamResponse,
24
+)
25
+
26
+logger = init_logger(__name__)
27
+
28
+
29
+class OpenAIServingTranslation(OpenAISpeechToText):
30
+ """Handles translation requests."""
31
+
32
+ def __init__(
33
+ self,
34
+ engine_client: EngineClient,
35
+ models: OpenAIServingModels,
36
+ *,
37
+ request_logger: RequestLogger | None,
38
+ return_tokens_as_token_ids: bool = False,
39
+ enable_force_include_usage: bool = False,
40
+ ):
41
+ super().__init__(
42
+ engine_client=engine_client,
43
+ models=models,
44
+ request_logger=request_logger,
45
+ return_tokens_as_token_ids=return_tokens_as_token_ids,
46
+ task_type="translate",
47
+ enable_force_include_usage=enable_force_include_usage,
48
+ )
49
+
50
+ async def create_translation(
51
+ self,
52
+ audio_data: bytes,
53
+ request: TranslationRequest,
54
+ raw_request: Request | None = None,
55
+ ) -> (
56
+ TranslationResponse
57
+ | TranslationResponseVerbose
58
+ | AsyncGenerator[str, None]
59
+ | ErrorResponse
60
+ ):
61
+ """Translation API similar to OpenAI's API.
62
+
63
+ See https://platform.openai.com/docs/api-reference/audio/createTranslation
64
+ for the API specification. This API mimics the OpenAI translation API.
65
+ """
66
+ return await self._create_speech_to_text(
67
+ audio_data=audio_data,
68
+ request=request,
69
+ raw_request=raw_request,
70
+ response_class=(
71
+ TranslationResponseVerbose
72
+ if request.response_format == "verbose_json"
73
+ else TranslationResponse
74
+ ),
75
+ stream_generator_method=self.translation_stream_generator,
76
+ )
77
+
78
+ async def translation_stream_generator(
79
+ self,
80
+ request: TranslationRequest,
81
+ result_generator: list[AsyncGenerator[RequestOutput, None]],
82
+ request_id: str,
83
+ request_metadata: RequestResponseMetadata,
84
+ audio_duration_s: float,
85
+ separator: str,
86
+ ) -> AsyncGenerator[str, None]:
87
+ generator = self._speech_to_text_stream_generator(
88
+ request=request,
89
+ list_result_generator=result_generator,
90
+ request_id=request_id,
91
+ request_metadata=request_metadata,
92
+ audio_duration_s=audio_duration_s,
93
+ chunk_object_type="translation.chunk",
94
+ response_stream_choice_class=TranslationResponseStreamChoice,
95
+ stream_response_class=TranslationStreamResponse,
96
+ separator=separator,
97
+ )
98
+ async for chunk in generator:
99
+ yield chunk