A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Refactor] [9/N] to simplify the vLLM openai translations serving ar chitecture (#32313)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jan 14, 2026 at 18:20 UTC
769d0629e1c55d0a92dbf7244e7a6de4103c6ba4
8 files changed
+692
-628
vllm/entrypoints/openai/api_server.py
+8
-101
@@ -17,12 +17,12 @@ from argparse import Namespace
17
from collections.abc import AsyncIterator, Awaitable
18
from contextlib import asynccontextmanager
19
from http import HTTPStatus
20
-from typing import Annotated, Any
20
+from typing import Any
21
22
import model_hosting_container_standards.sagemaker as sagemaker_standards
23
import pydantic
24
import uvloop
25
-from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
25
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
26
from fastapi.exceptions import RequestValidationError
27
from fastapi.middleware.cors import CORSMiddleware
28
from fastapi.responses import JSONResponse, StreamingResponse
@@ -49,10 +49,6 @@ from vllm.entrypoints.openai.engine.protocol import (
49
CompletionResponse,
50
ErrorInfo,
51
ErrorResponse,
52
- TranscriptionRequest,
53
- TranscriptionResponseVariant,
54
- TranslationRequest,
55
- TranslationResponseVariant,
52
)
53
from vllm.entrypoints.openai.engine.serving import OpenAIServing
54
from vllm.entrypoints.openai.orca_metrics import metrics_header
@@ -62,7 +58,7 @@ from vllm.entrypoints.openai.serving_models import (
58
BaseModelPath,
59
OpenAIServingModels,
60
)
65
-from vllm.entrypoints.openai.serving_transcription import (
61
+from vllm.entrypoints.openai.translations.serving import (
62
OpenAIServingTranscription,
63
OpenAIServingTranslation,
64
)
@@ -239,10 +235,6 @@ def models(request: Request) -> OpenAIServingModels:
235
return request.app.state.openai_serving_models
236
237
242
-def responses(request: Request) -> OpenAIServingResponses | None:
243
- return request.app.state.openai_serving_responses
244
-
245
-
238
def messages(request: Request) -> AnthropicServingMessages:
239
return request.app.state.anthropic_serving_messages
240
@@ -259,22 +251,10 @@ def tokenization(request: Request) -> OpenAIServingTokenization:
251
return request.app.state.openai_serving_tokenization
252
253
262
-def transcription(request: Request) -> OpenAIServingTranscription:
263
- return request.app.state.openai_serving_transcription
264
-
265
-
266
-def translation(request: Request) -> OpenAIServingTranslation:
267
- return request.app.state.openai_serving_translation
268
-
269
-
254
def engine_client(request: Request) -> EngineClient:
255
return request.app.state.engine_client
256
257
274
-def generate_tokens(request: Request) -> ServingTokens | None:
275
- return request.app.state.serving_tokens
276
-
277
-
258
@router.get("/load")
259
async def get_server_load_metrics(request: Request):
260
# This endpoint returns the current server load metrics.
@@ -410,84 +390,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
390
return StreamingResponse(content=generator, media_type="text/event-stream")
391
392
413
-@router.post(
414
- "/v1/audio/transcriptions",
415
- responses={
416
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
417
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
418
- HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
419
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
420
- },
421
-)
422
-@with_cancellation
423
-@load_aware_call
424
-async def create_transcriptions(
425
- raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
426
-):
427
- handler = transcription(raw_request)
428
- if handler is None:
429
- return base(raw_request).create_error_response(
430
- message="The model does not support Transcriptions API"
431
- )
432
-
433
- audio_data = await request.file.read()
434
- try:
435
- generator = await handler.create_transcription(audio_data, request, raw_request)
436
- except Exception as e:
437
- raise HTTPException(
438
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
439
- ) from e
440
-
441
- if isinstance(generator, ErrorResponse):
442
- return JSONResponse(
443
- content=generator.model_dump(), status_code=generator.error.code
444
- )
445
-
446
- elif isinstance(generator, TranscriptionResponseVariant):
447
- return JSONResponse(content=generator.model_dump())
448
-
449
- return StreamingResponse(content=generator, media_type="text/event-stream")
450
-
451
-
452
-@router.post(
453
- "/v1/audio/translations",
454
- responses={
455
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
456
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
457
- HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
458
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
459
- },
460
-)
461
-@with_cancellation
462
-@load_aware_call
463
-async def create_translations(
464
- request: Annotated[TranslationRequest, Form()], raw_request: Request
465
-):
466
- handler = translation(raw_request)
467
- if handler is None:
468
- return base(raw_request).create_error_response(
469
- message="The model does not support Translations API"
470
- )
471
-
472
- audio_data = await request.file.read()
473
- try:
474
- generator = await handler.create_translation(audio_data, request, raw_request)
475
- except Exception as e:
476
- raise HTTPException(
477
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
478
- ) from e
479
-
480
- if isinstance(generator, ErrorResponse):
481
- return JSONResponse(
482
- content=generator.model_dump(), status_code=generator.error.code
483
- )
484
-
485
- elif isinstance(generator, TranslationResponseVariant):
486
- return JSONResponse(content=generator.model_dump())
487
-
488
- return StreamingResponse(content=generator, media_type="text/event-stream")
489
-
490
-
393
def load_log_config(log_config_file: str | None) -> dict | None:
394
if not log_config_file:
395
return None
@@ -741,6 +643,11 @@ def build_app(args: Namespace) -> FastAPI:
643
)
644
645
register_responses_api_router(app)
646
+ from vllm.entrypoints.openai.translations.api_router import (
647
+ attach_router as register_translations_api_router,
648
+ )
649
+
650
+ register_translations_api_router(app)
651
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
652
653
register_sagemaker_routes(router)
vllm/entrypoints/openai/engine/protocol.py
-518
@@ -5,12 +5,10 @@
5
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
7
import time
8
-from http import HTTPStatus
8
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
9
10
import regex as re
11
import torch
13
-from fastapi import HTTPException, UploadFile
12
from pydantic import (
13
BaseModel,
14
ConfigDict,
@@ -702,522 +700,6 @@ class DeltaMessage(OpenAIBaseModel):
700
return self
701
702
705
-class TranscriptionResponseStreamChoice(OpenAIBaseModel):
706
- delta: DeltaMessage
707
- finish_reason: str | None = None
708
- stop_reason: int | str | None = None
709
-
710
-
711
-class TranscriptionStreamResponse(OpenAIBaseModel):
712
- id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
713
- object: Literal["transcription.chunk"] = "transcription.chunk"
714
- created: int = Field(default_factory=lambda: int(time.time()))
715
- model: str
716
- choices: list[TranscriptionResponseStreamChoice]
717
- usage: UsageInfo | None = Field(default=None)
718
-
719
-
720
-## Protocols for Audio
721
-AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
722
-
723
-
724
-class TranscriptionRequest(OpenAIBaseModel):
725
- # Ordered by official OpenAI API documentation
726
- # https://platform.openai.com/docs/api-reference/audio/createTranscription
727
-
728
- file: UploadFile
729
- """
730
- The audio file object (not file name) to transcribe, in one of these
731
- formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
732
- """
733
-
734
- model: str | None = None
735
- """ID of the model to use.
736
- """
737
-
738
- language: str | None = None
739
- """The language of the input audio.
740
-
741
- Supplying the input language in
742
- [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
743
- will improve accuracy and latency.
744
- """
745
-
746
- prompt: str = Field(default="")
747
- """An optional text to guide the model's style or continue a previous audio
748
- segment.
749
-
750
- The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
751
- should match the audio language.
752
- """
753
-
754
- response_format: AudioResponseFormat = Field(default="json")
755
- """
756
- The format of the output, in one of these options: `json`, `text`, `srt`,
757
- `verbose_json`, or `vtt`.
758
- """
759
-
760
- ## TODO (varun) : Support if set to 0, certain thresholds are met !!
761
-
762
- timestamp_granularities: list[Literal["word", "segment"]] = Field(
763
- alias="timestamp_granularities[]", default=[]
764
- )
765
- """The timestamp granularities to populate for this transcription.
766
-
767
- `response_format` must be set `verbose_json` to use timestamp granularities.
768
- Either or both of these options are supported: `word`, or `segment`. Note:
769
- There is no additional latency for segment timestamps, but generating word
770
- timestamps incurs additional latency.
771
- """
772
-
773
- stream: bool | None = False
774
- """When set, it will enable output to be streamed in a similar fashion
775
- as the Chat Completion endpoint.
776
- """
777
- # --8<-- [start:transcription-extra-params]
778
- # Flattened stream option to simplify form data.
779
- stream_include_usage: bool | None = False
780
- stream_continuous_usage_stats: bool | None = False
781
-
782
- vllm_xargs: dict[str, str | int | float] | None = Field(
783
- default=None,
784
- description=(
785
- "Additional request parameters with string or "
786
- "numeric values, used by custom extensions."
787
- ),
788
- )
789
- # --8<-- [end:transcription-extra-params]
790
-
791
- to_language: str | None = None
792
- """The language of the output audio we transcribe to.
793
-
794
- Please note that this is not currently used by supported models at this
795
- time, but it is a placeholder for future use, matching translation api.
796
- """
797
-
798
- # --8<-- [start:transcription-sampling-params]
799
- temperature: float = Field(default=0.0)
800
- """The sampling temperature, between 0 and 1.
801
-
802
- Higher values like 0.8 will make the output more random, while lower values
803
- like 0.2 will make it more focused / deterministic. If set to 0, the model
804
- will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
805
- to automatically increase the temperature until certain thresholds are hit.
806
- """
807
-
808
- top_p: float | None = None
809
- """Enables nucleus (top-p) sampling, where tokens are selected from the
810
- smallest possible set whose cumulative probability exceeds `p`.
811
- """
812
-
813
- top_k: int | None = None
814
- """Limits sampling to the `k` most probable tokens at each step."""
815
-
816
- min_p: float | None = None
817
- """Filters out tokens with a probability lower than `min_p`, ensuring a
818
- minimum likelihood threshold during sampling.
819
- """
820
-
821
- seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
822
- """The seed to use for sampling."""
823
-
824
- frequency_penalty: float | None = 0.0
825
- """The frequency penalty to use for sampling."""
826
-
827
- repetition_penalty: float | None = None
828
- """The repetition penalty to use for sampling."""
829
-
830
- presence_penalty: float | None = 0.0
831
- """The presence penalty to use for sampling."""
832
-
833
- max_completion_tokens: int | None = None
834
- """The maximum number of tokens to generate."""
835
- # --8<-- [end:transcription-sampling-params]
836
-
837
- # Default sampling parameters for transcription requests.
838
- _DEFAULT_SAMPLING_PARAMS: dict = {
839
- "repetition_penalty": 1.0,
840
- "temperature": 1.0,
841
- "top_p": 1.0,
842
- "top_k": 0,
843
- "min_p": 0.0,
844
- }
845
-
846
- def to_sampling_params(
847
- self, default_max_tokens: int, default_sampling_params: dict | None = None
848
- ) -> SamplingParams:
849
- max_tokens = default_max_tokens
850
-
851
- if default_sampling_params is None:
852
- default_sampling_params = {}
853
-
854
- # Default parameters
855
- if (temperature := self.temperature) is None:
856
- temperature = default_sampling_params.get(
857
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
858
- )
859
- if (top_p := self.top_p) is None:
860
- top_p = default_sampling_params.get(
861
- "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
862
- )
863
- if (top_k := self.top_k) is None:
864
- top_k = default_sampling_params.get(
865
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
866
- )
867
- if (min_p := self.min_p) is None:
868
- min_p = default_sampling_params.get(
869
- "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
870
- )
871
-
872
- if (repetition_penalty := self.repetition_penalty) is None:
873
- repetition_penalty = default_sampling_params.get(
874
- "repetition_penalty",
875
- self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
876
- )
877
-
878
- return SamplingParams.from_optional(
879
- temperature=temperature,
880
- max_tokens=max_tokens,
881
- seed=self.seed,
882
- top_p=top_p,
883
- top_k=top_k,
884
- min_p=min_p,
885
- frequency_penalty=self.frequency_penalty,
886
- repetition_penalty=repetition_penalty,
887
- presence_penalty=self.presence_penalty,
888
- output_kind=RequestOutputKind.DELTA
889
- if self.stream
890
- else RequestOutputKind.FINAL_ONLY,
891
- extra_args=self.vllm_xargs,
892
- skip_clone=True, # Created fresh per request, safe to skip clone
893
- )
894
-
895
- @model_validator(mode="before")
896
- @classmethod
897
- def validate_transcription_request(cls, data):
898
- if isinstance(data.get("file"), str):
899
- raise HTTPException(
900
- status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
901
- detail="Expected 'file' to be a file-like object, not 'str'.",
902
- )
903
-
904
- stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
905
- stream = data.get("stream", False)
906
- if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
907
- # Find which specific stream option was set
908
- invalid_param = next(
909
- (so for so in stream_opts if data.get(so, False)),
910
- "stream_include_usage",
911
- )
912
- raise VLLMValidationError(
913
- "Stream options can only be defined when `stream=True`.",
914
- parameter=invalid_param,
915
- )
916
-
917
- return data
918
-
919
-
920
-# Transcription response objects
921
-class TranscriptionUsageAudio(OpenAIBaseModel):
922
- type: Literal["duration"] = "duration"
923
- seconds: int
924
-
925
-
926
-class TranscriptionResponse(OpenAIBaseModel):
927
- text: str
928
- """The transcribed text."""
929
- usage: TranscriptionUsageAudio
930
-
931
-
932
-class TranscriptionWord(OpenAIBaseModel):
933
- end: float
934
- """End time of the word in seconds."""
935
-
936
- start: float
937
- """Start time of the word in seconds."""
938
-
939
- word: str
940
- """The text content of the word."""
941
-
942
-
943
-class TranscriptionSegment(OpenAIBaseModel):
944
- id: int
945
- """Unique identifier of the segment."""
946
-
947
- avg_logprob: float | None = None
948
- """Average logprob of the segment.
949
-
950
- If the value is lower than -1, consider the logprobs failed.
951
- """
952
-
953
- compression_ratio: float | None = None
954
- """Compression ratio of the segment.
955
-
956
- If the value is greater than 2.4, consider the compression failed.
957
- """
958
-
959
- end: float
960
- """End time of the segment in seconds."""
961
-
962
- no_speech_prob: float | None = None
963
- """Probability of no speech in the segment.
964
-
965
- If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
966
- this segment silent.
967
- """
968
-
969
- seek: int
970
- """Seek offset of the segment."""
971
-
972
- start: float
973
- """Start time of the segment in seconds."""
974
-
975
- temperature: float
976
- """Temperature parameter used for generating the segment."""
977
-
978
- text: str
979
- """Text content of the segment."""
980
-
981
- tokens: list[int]
982
- """Array of token IDs for the text content."""
983
-
984
-
985
-class TranscriptionResponseVerbose(OpenAIBaseModel):
986
- duration: str
987
- """The duration of the input audio."""
988
-
989
- language: str
990
- """The language of the input audio."""
991
-
992
- text: str
993
- """The transcribed text."""
994
-
995
- segments: list[TranscriptionSegment] | None = None
996
- """Segments of the transcribed text and their corresponding details."""
997
-
998
- words: list[TranscriptionWord] | None = None
999
- """Extracted words and their corresponding timestamps."""
1000
-
1001
-
1002
-TranscriptionResponseVariant: TypeAlias = (
1003
- TranscriptionResponse | TranscriptionResponseVerbose
1004
-)
1005
-
1006
-
1007
-class TranslationResponseStreamChoice(OpenAIBaseModel):
1008
- delta: DeltaMessage
1009
- finish_reason: str | None = None
1010
- stop_reason: int | str | None = None
1011
-
1012
-
1013
-class TranslationStreamResponse(OpenAIBaseModel):
1014
- id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
1015
- object: Literal["translation.chunk"] = "translation.chunk"
1016
- created: int = Field(default_factory=lambda: int(time.time()))
1017
- model: str
1018
- choices: list[TranslationResponseStreamChoice]
1019
- usage: UsageInfo | None = Field(default=None)
1020
-
1021
-
1022
-class TranslationRequest(OpenAIBaseModel):
1023
- # Ordered by official OpenAI API documentation
1024
- # https://platform.openai.com/docs/api-reference/audio/createTranslation
1025
-
1026
- file: UploadFile
1027
- """
1028
- The audio file object (not file name) to translate, in one of these
1029
- formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
1030
- """
1031
-
1032
- model: str | None = None
1033
- """ID of the model to use.
1034
- """
1035
-
1036
- prompt: str = Field(default="")
1037
- """An optional text to guide the model's style or continue a previous audio
1038
- segment.
1039
-
1040
- The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
1041
- should match the audio language.
1042
- """
1043
-
1044
- response_format: AudioResponseFormat = Field(default="json")
1045
- """
1046
- The format of the output, in one of these options: `json`, `text`, `srt`,
1047
- `verbose_json`, or `vtt`.
1048
- """
1049
-
1050
- # TODO support additional sampling parameters
1051
- # --8<-- [start:translation-sampling-params]
1052
- seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
1053
- """The seed to use for sampling."""
1054
-
1055
- temperature: float = Field(default=0.0)
1056
- """The sampling temperature, between 0 and 1.
1057
-
1058
- Higher values like 0.8 will make the output more random, while lower values
1059
- like 0.2 will make it more focused / deterministic. If set to 0, the model
1060
- will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
1061
- to automatically increase the temperature until certain thresholds are hit.
1062
- """
1063
- # --8<-- [end:translation-sampling-params]
1064
-
1065
- # --8<-- [start:translation-extra-params]
1066
- language: str | None = None
1067
- """The language of the input audio we translate from.
1068
-
1069
- Supplying the input language in
1070
- [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
1071
- will improve accuracy.
1072
- """
1073
-
1074
- to_language: str | None = None
1075
- """The language of the input audio we translate to.
1076
-
1077
- Please note that this is not supported by all models, refer to the specific
1078
- model documentation for more details.
1079
- For instance, Whisper only supports `to_language=en`.
1080
- """
1081
-
1082
- stream: bool | None = False
1083
- """Custom field not present in the original OpenAI definition. When set,
1084
- it will enable output to be streamed in a similar fashion as the Chat
1085
- Completion endpoint.
1086
- """
1087
- # Flattened stream option to simplify form data.
1088
- stream_include_usage: bool | None = False
1089
- stream_continuous_usage_stats: bool | None = False
1090
-
1091
- max_completion_tokens: int | None = None
1092
- """The maximum number of tokens to generate."""
1093
- # --8<-- [end:translation-extra-params]
1094
-
1095
- # Default sampling parameters for translation requests.
1096
- _DEFAULT_SAMPLING_PARAMS: dict = {
1097
- "temperature": 0,
1098
- }
1099
-
1100
- def to_sampling_params(
1101
- self, default_max_tokens: int, default_sampling_params: dict | None = None
1102
- ) -> SamplingParams:
1103
- max_tokens = default_max_tokens
1104
-
1105
- if default_sampling_params is None:
1106
- default_sampling_params = {}
1107
- # Default parameters
1108
- if (temperature := self.temperature) is None:
1109
- temperature = default_sampling_params.get(
1110
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
1111
- )
1112
-
1113
- return SamplingParams.from_optional(
1114
- temperature=temperature,
1115
- max_tokens=max_tokens,
1116
- seed=self.seed,
1117
- output_kind=RequestOutputKind.DELTA
1118
- if self.stream
1119
- else RequestOutputKind.FINAL_ONLY,
1120
- skip_clone=True, # Created fresh per request, safe to skip clone
1121
- )
1122
-
1123
- @model_validator(mode="before")
1124
- @classmethod
1125
- def validate_stream_options(cls, data):
1126
- stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
1127
- stream = data.get("stream", False)
1128
- if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
1129
- # Find which specific stream option was set
1130
- invalid_param = next(
1131
- (so for so in stream_opts if data.get(so, False)),
1132
- "stream_include_usage",
1133
- )
1134
- raise VLLMValidationError(
1135
- "Stream options can only be defined when `stream=True`.",
1136
- parameter=invalid_param,
1137
- )
1138
-
1139
- return data
1140
-
1141
-
1142
-# Translation response objects
1143
-class TranslationResponse(OpenAIBaseModel):
1144
- text: str
1145
- """The translated text."""
1146
-
1147
-
1148
-class TranslationWord(OpenAIBaseModel):
1149
- end: float
1150
- """End time of the word in seconds."""
1151
-
1152
- start: float
1153
- """Start time of the word in seconds."""
1154
-
1155
- word: str
1156
- """The text content of the word."""
1157
-
1158
-
1159
-class TranslationSegment(OpenAIBaseModel):
1160
- id: int
1161
- """Unique identifier of the segment."""
1162
-
1163
- avg_logprob: float | None = None
1164
- """Average logprob of the segment.
1165
-
1166
- If the value is lower than -1, consider the logprobs failed.
1167
- """
1168
-
1169
- compression_ratio: float | None = None
1170
- """Compression ratio of the segment.
1171
-
1172
- If the value is greater than 2.4, consider the compression failed.
1173
- """
1174
-
1175
- end: float
1176
- """End time of the segment in seconds."""
1177
-
1178
- no_speech_prob: float | None = None
1179
- """Probability of no speech in the segment.
1180
-
1181
- If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
1182
- this segment silent.
1183
- """
1184
-
1185
- seek: int
1186
- """Seek offset of the segment."""
1187
-
1188
- start: float
1189
- """Start time of the segment in seconds."""
1190
-
1191
- temperature: float
1192
- """Temperature parameter used for generating the segment."""
1193
-
1194
- text: str
1195
- """Text content of the segment."""
1196
-
1197
- tokens: list[int]
1198
- """Array of token IDs for the text content."""
1199
-
1200
-
1201
-class TranslationResponseVerbose(OpenAIBaseModel):
1202
- duration: str
1203
- """The duration of the input audio."""
1204
-
1205
- language: str
1206
- """The language of the input audio."""
1207
-
1208
- text: str
1209
- """The translated text."""
1210
-
1211
- segments: list[TranslationSegment] | None = None
1212
- """Segments of the translated text and their corresponding details."""
1213
-
1214
- words: list[TranslationWord] | None = None
1215
- """Extracted words and their corresponding timestamps."""
1216
-
1217
-
1218
-TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
1219
-
1220
-
703
####### Tokens IN <> Tokens OUT #######
704
class GenerateRequest(BaseModel):
705
request_id: str = Field(
vllm/entrypoints/openai/engine/serving.py
+5
-3
@@ -50,9 +50,6 @@ from vllm.entrypoints.openai.engine.protocol import (
50
ErrorResponse,
51
FunctionCall,
52
FunctionDefinition,
53
- TranscriptionRequest,
54
- TranscriptionResponse,
55
- TranslationRequest,
53
VLLMValidationError,
54
)
55
from vllm.entrypoints.openai.responses.protocol import (
@@ -60,6 +57,11 @@ from vllm.entrypoints.openai.responses.protocol import (
57
ResponsesRequest,
58
)
59
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
60
+from vllm.entrypoints.openai.translations.protocol import (
61
+ TranscriptionRequest,
62
+ TranscriptionResponse,
63
+ TranslationRequest,
64
+)
65
from vllm.entrypoints.pooling.classify.protocol import (
66
ClassificationChatRequest,
67
ClassificationCompletionRequest,
vllm/entrypoints/openai/translations/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/openai/translations/api_router.py
new
+122
@@ -0,0 +1,122 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+from typing import Annotated
7
+
8
+from fastapi import APIRouter, FastAPI, Form, HTTPException, Request
9
+from fastapi.responses import JSONResponse, StreamingResponse
10
+
11
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12
+from vllm.entrypoints.openai.translations.protocol import (
13
+ TranscriptionRequest,
14
+ TranscriptionResponseVariant,
15
+ TranslationRequest,
16
+ TranslationResponseVariant,
17
+)
18
+from vllm.entrypoints.openai.translations.serving import (
19
+ OpenAIServingTranscription,
20
+ OpenAIServingTranslation,
21
+)
22
+from vllm.entrypoints.utils import (
23
+ load_aware_call,
24
+ with_cancellation,
25
+)
26
+from vllm.logger import init_logger
27
+
28
+logger = init_logger(__name__)
29
+
30
+router = APIRouter()
31
+
32
+
33
+def transcription(request: Request) -> OpenAIServingTranscription:
34
+ return request.app.state.openai_serving_transcription
35
+
36
+
37
+def translation(request: Request) -> OpenAIServingTranslation:
38
+ return request.app.state.openai_serving_translation
39
+
40
+
41
+@router.post(
42
+ "/v1/audio/transcriptions",
43
+ responses={
44
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
45
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
46
+ HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
47
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
48
+ },
49
+)
50
+@with_cancellation
51
+@load_aware_call
52
+async def create_transcriptions(
53
+ raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
54
+):
55
+ handler = transcription(raw_request)
56
+ if handler is None:
57
+ base_server = raw_request.app.state.openai_serving_tokenization
58
+ return base_server.create_error_response(
59
+ message="The model does not support Transcriptions API"
60
+ )
61
+
62
+ audio_data = await request.file.read()
63
+ try:
64
+ generator = await handler.create_transcription(audio_data, request, raw_request)
65
+ except Exception as e:
66
+ raise HTTPException(
67
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
68
+ ) from e
69
+
70
+ if isinstance(generator, ErrorResponse):
71
+ return JSONResponse(
72
+ content=generator.model_dump(), status_code=generator.error.code
73
+ )
74
+
75
+ elif isinstance(generator, TranscriptionResponseVariant):
76
+ return JSONResponse(content=generator.model_dump())
77
+
78
+ return StreamingResponse(content=generator, media_type="text/event-stream")
79
+
80
+
81
+@router.post(
82
+ "/v1/audio/translations",
83
+ responses={
84
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
85
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
86
+ HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
87
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
88
+ },
89
+)
90
+@with_cancellation
91
+@load_aware_call
92
+async def create_translations(
93
+ request: Annotated[TranslationRequest, Form()], raw_request: Request
94
+):
95
+ handler = translation(raw_request)
96
+ if handler is None:
97
+ base_server = raw_request.app.state.openai_serving_tokenization
98
+ return base_server.create_error_response(
99
+ message="The model does not support Translations API"
100
+ )
101
+
102
+ audio_data = await request.file.read()
103
+ try:
104
+ generator = await handler.create_translation(audio_data, request, raw_request)
105
+ except Exception as e:
106
+ raise HTTPException(
107
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
108
+ ) from e
109
+
110
+ if isinstance(generator, ErrorResponse):
111
+ return JSONResponse(
112
+ content=generator.model_dump(), status_code=generator.error.code
113
+ )
114
+
115
+ elif isinstance(generator, TranslationResponseVariant):
116
+ return JSONResponse(content=generator.model_dump())
117
+
118
+ return StreamingResponse(content=generator, media_type="text/event-stream")
119
+
120
+
121
+def attach_router(app: FastAPI):
122
+ app.include_router(router)
vllm/entrypoints/openai/translations/protocol.py
new
+545
@@ -0,0 +1,545 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import time
5
+from http import HTTPStatus
6
+from typing import Literal, TypeAlias
7
+
8
+import torch
9
+from fastapi import HTTPException, UploadFile
10
+from pydantic import (
11
+ Field,
12
+ model_validator,
13
+)
14
+
15
+from vllm.entrypoints.openai.engine.protocol import (
16
+ DeltaMessage,
17
+ OpenAIBaseModel,
18
+ UsageInfo,
19
+)
20
+from vllm.exceptions import VLLMValidationError
21
+from vllm.logger import init_logger
22
+from vllm.sampling_params import (
23
+ RequestOutputKind,
24
+ SamplingParams,
25
+)
26
+from vllm.utils import random_uuid
27
+
28
+logger = init_logger(__name__)
29
+_LONG_INFO = torch.iinfo(torch.long)
30
+
31
+
32
+class TranscriptionResponseStreamChoice(OpenAIBaseModel):
33
+ delta: DeltaMessage
34
+ finish_reason: str | None = None
35
+ stop_reason: int | str | None = None
36
+
37
+
38
+class TranscriptionStreamResponse(OpenAIBaseModel):
39
+ id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
40
+ object: Literal["transcription.chunk"] = "transcription.chunk"
41
+ created: int = Field(default_factory=lambda: int(time.time()))
42
+ model: str
43
+ choices: list[TranscriptionResponseStreamChoice]
44
+ usage: UsageInfo | None = Field(default=None)
45
+
46
+
47
+## Protocols for Audio
48
+AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
49
+
50
+
51
+class TranscriptionRequest(OpenAIBaseModel):
52
+ # Ordered by official OpenAI API documentation
53
+ # https://platform.openai.com/docs/api-reference/audio/createTranscription
54
+
55
+ file: UploadFile
56
+ """
57
+ The audio file object (not file name) to transcribe, in one of these
58
+ formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
59
+ """
60
+
61
+ model: str | None = None
62
+ """ID of the model to use.
63
+ """
64
+
65
+ language: str | None = None
66
+ """The language of the input audio.
67
+
68
+ Supplying the input language in
69
+ [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
70
+ will improve accuracy and latency.
71
+ """
72
+
73
+ prompt: str = Field(default="")
74
+ """An optional text to guide the model's style or continue a previous audio
75
+ segment.
76
+
77
+ The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
78
+ should match the audio language.
79
+ """
80
+
81
+ response_format: AudioResponseFormat = Field(default="json")
82
+ """
83
+ The format of the output, in one of these options: `json`, `text`, `srt`,
84
+ `verbose_json`, or `vtt`.
85
+ """
86
+
87
+ ## TODO (varun) : Support if set to 0, certain thresholds are met !!
88
+
89
+ timestamp_granularities: list[Literal["word", "segment"]] = Field(
90
+ alias="timestamp_granularities[]", default=[]
91
+ )
92
+ """The timestamp granularities to populate for this transcription.
93
+
94
+ `response_format` must be set `verbose_json` to use timestamp granularities.
95
+ Either or both of these options are supported: `word`, or `segment`. Note:
96
+ There is no additional latency for segment timestamps, but generating word
97
+ timestamps incurs additional latency.
98
+ """
99
+
100
+ stream: bool | None = False
101
+ """When set, it will enable output to be streamed in a similar fashion
102
+ as the Chat Completion endpoint.
103
+ """
104
+ # --8<-- [start:transcription-extra-params]
105
+ # Flattened stream option to simplify form data.
106
+ stream_include_usage: bool | None = False
107
+ stream_continuous_usage_stats: bool | None = False
108
+
109
+ vllm_xargs: dict[str, str | int | float] | None = Field(
110
+ default=None,
111
+ description=(
112
+ "Additional request parameters with string or "
113
+ "numeric values, used by custom extensions."
114
+ ),
115
+ )
116
+ # --8<-- [end:transcription-extra-params]
117
+
118
+ to_language: str | None = None
119
+ """The language of the output audio we transcribe to.
120
+
121
+ Please note that this is not currently used by supported models at this
122
+ time, but it is a placeholder for future use, matching translation api.
123
+ """
124
+
125
+ # --8<-- [start:transcription-sampling-params]
126
+ temperature: float = Field(default=0.0)
127
+ """The sampling temperature, between 0 and 1.
128
+
129
+ Higher values like 0.8 will make the output more random, while lower values
130
+ like 0.2 will make it more focused / deterministic. If set to 0, the model
131
+ will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
132
+ to automatically increase the temperature until certain thresholds are hit.
133
+ """
134
+
135
+ top_p: float | None = None
136
+ """Enables nucleus (top-p) sampling, where tokens are selected from the
137
+ smallest possible set whose cumulative probability exceeds `p`.
138
+ """
139
+
140
+ top_k: int | None = None
141
+ """Limits sampling to the `k` most probable tokens at each step."""
142
+
143
+ min_p: float | None = None
144
+ """Filters out tokens with a probability lower than `min_p`, ensuring a
145
+ minimum likelihood threshold during sampling.
146
+ """
147
+
148
+ seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
149
+ """The seed to use for sampling."""
150
+
151
+ frequency_penalty: float | None = 0.0
152
+ """The frequency penalty to use for sampling."""
153
+
154
+ repetition_penalty: float | None = None
155
+ """The repetition penalty to use for sampling."""
156
+
157
+ presence_penalty: float | None = 0.0
158
+ """The presence penalty to use for sampling."""
159
+
160
+ max_completion_tokens: int | None = None
161
+ """The maximum number of tokens to generate."""
162
+ # --8<-- [end:transcription-sampling-params]
163
+
164
+ # Default sampling parameters for transcription requests.
165
+ _DEFAULT_SAMPLING_PARAMS: dict = {
166
+ "repetition_penalty": 1.0,
167
+ "temperature": 1.0,
168
+ "top_p": 1.0,
169
+ "top_k": 0,
170
+ "min_p": 0.0,
171
+ }
172
+
173
+ def to_sampling_params(
174
+ self, default_max_tokens: int, default_sampling_params: dict | None = None
175
+ ) -> SamplingParams:
176
+ max_tokens = default_max_tokens
177
+
178
+ if default_sampling_params is None:
179
+ default_sampling_params = {}
180
+
181
+ # Default parameters
182
+ if (temperature := self.temperature) is None:
183
+ temperature = default_sampling_params.get(
184
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
185
+ )
186
+ if (top_p := self.top_p) is None:
187
+ top_p = default_sampling_params.get(
188
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
189
+ )
190
+ if (top_k := self.top_k) is None:
191
+ top_k = default_sampling_params.get(
192
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
193
+ )
194
+ if (min_p := self.min_p) is None:
195
+ min_p = default_sampling_params.get(
196
+ "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
197
+ )
198
+
199
+ if (repetition_penalty := self.repetition_penalty) is None:
200
+ repetition_penalty = default_sampling_params.get(
201
+ "repetition_penalty",
202
+ self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
203
+ )
204
+
205
+ return SamplingParams.from_optional(
206
+ temperature=temperature,
207
+ max_tokens=max_tokens,
208
+ seed=self.seed,
209
+ top_p=top_p,
210
+ top_k=top_k,
211
+ min_p=min_p,
212
+ frequency_penalty=self.frequency_penalty,
213
+ repetition_penalty=repetition_penalty,
214
+ presence_penalty=self.presence_penalty,
215
+ output_kind=RequestOutputKind.DELTA
216
+ if self.stream
217
+ else RequestOutputKind.FINAL_ONLY,
218
+ extra_args=self.vllm_xargs,
219
+ skip_clone=True, # Created fresh per request, safe to skip clone
220
+ )
221
+
222
+ @model_validator(mode="before")
223
+ @classmethod
224
+ def validate_transcription_request(cls, data):
225
+ if isinstance(data.get("file"), str):
226
+ raise HTTPException(
227
+ status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
228
+ detail="Expected 'file' to be a file-like object, not 'str'.",
229
+ )
230
+
231
+ stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
232
+ stream = data.get("stream", False)
233
+ if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
234
+ # Find which specific stream option was set
235
+ invalid_param = next(
236
+ (so for so in stream_opts if data.get(so, False)),
237
+ "stream_include_usage",
238
+ )
239
+ raise VLLMValidationError(
240
+ "Stream options can only be defined when `stream=True`.",
241
+ parameter=invalid_param,
242
+ )
243
+
244
+ return data
245
+
246
+
247
+# Transcription response objects
248
+class TranscriptionUsageAudio(OpenAIBaseModel):
249
+ type: Literal["duration"] = "duration"
250
+ seconds: int
251
+
252
+
253
+class TranscriptionResponse(OpenAIBaseModel):
254
+ text: str
255
+ """The transcribed text."""
256
+ usage: TranscriptionUsageAudio
257
+
258
+
259
+class TranscriptionWord(OpenAIBaseModel):
260
+ end: float
261
+ """End time of the word in seconds."""
262
+
263
+ start: float
264
+ """Start time of the word in seconds."""
265
+
266
+ word: str
267
+ """The text content of the word."""
268
+
269
+
270
+class TranscriptionSegment(OpenAIBaseModel):
271
+ id: int
272
+ """Unique identifier of the segment."""
273
+
274
+ avg_logprob: float | None = None
275
+ """Average logprob of the segment.
276
+
277
+ If the value is lower than -1, consider the logprobs failed.
278
+ """
279
+
280
+ compression_ratio: float | None = None
281
+ """Compression ratio of the segment.
282
+
283
+ If the value is greater than 2.4, consider the compression failed.
284
+ """
285
+
286
+ end: float
287
+ """End time of the segment in seconds."""
288
+
289
+ no_speech_prob: float | None = None
290
+ """Probability of no speech in the segment.
291
+
292
+ If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
293
+ this segment silent.
294
+ """
295
+
296
+ seek: int
297
+ """Seek offset of the segment."""
298
+
299
+ start: float
300
+ """Start time of the segment in seconds."""
301
+
302
+ temperature: float
303
+ """Temperature parameter used for generating the segment."""
304
+
305
+ text: str
306
+ """Text content of the segment."""
307
+
308
+ tokens: list[int]
309
+ """Array of token IDs for the text content."""
310
+
311
+
312
+class TranscriptionResponseVerbose(OpenAIBaseModel):
313
+ duration: str
314
+ """The duration of the input audio."""
315
+
316
+ language: str
317
+ """The language of the input audio."""
318
+
319
+ text: str
320
+ """The transcribed text."""
321
+
322
+ segments: list[TranscriptionSegment] | None = None
323
+ """Segments of the transcribed text and their corresponding details."""
324
+
325
+ words: list[TranscriptionWord] | None = None
326
+ """Extracted words and their corresponding timestamps."""
327
+
328
+
329
+TranscriptionResponseVariant: TypeAlias = (
330
+ TranscriptionResponse | TranscriptionResponseVerbose
331
+)
332
+
333
+
334
+class TranslationResponseStreamChoice(OpenAIBaseModel):
335
+ delta: DeltaMessage
336
+ finish_reason: str | None = None
337
+ stop_reason: int | str | None = None
338
+
339
+
340
+class TranslationStreamResponse(OpenAIBaseModel):
341
+ id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
342
+ object: Literal["translation.chunk"] = "translation.chunk"
343
+ created: int = Field(default_factory=lambda: int(time.time()))
344
+ model: str
345
+ choices: list[TranslationResponseStreamChoice]
346
+ usage: UsageInfo | None = Field(default=None)
347
+
348
+
349
+class TranslationRequest(OpenAIBaseModel):
350
+ # Ordered by official OpenAI API documentation
351
+ # https://platform.openai.com/docs/api-reference/audio/createTranslation
352
+
353
+ file: UploadFile
354
+ """
355
+ The audio file object (not file name) to translate, in one of these
356
+ formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
357
+ """
358
+
359
+ model: str | None = None
360
+ """ID of the model to use.
361
+ """
362
+
363
+ prompt: str = Field(default="")
364
+ """An optional text to guide the model's style or continue a previous audio
365
+ segment.
366
+
367
+ The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
368
+ should match the audio language.
369
+ """
370
+
371
+ response_format: AudioResponseFormat = Field(default="json")
372
+ """
373
+ The format of the output, in one of these options: `json`, `text`, `srt`,
374
+ `verbose_json`, or `vtt`.
375
+ """
376
+
377
+ # TODO support additional sampling parameters
378
+ # --8<-- [start:translation-sampling-params]
379
+ seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
380
+ """The seed to use for sampling."""
381
+
382
+ temperature: float = Field(default=0.0)
383
+ """The sampling temperature, between 0 and 1.
384
+
385
+ Higher values like 0.8 will make the output more random, while lower values
386
+ like 0.2 will make it more focused / deterministic. If set to 0, the model
387
+ will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
388
+ to automatically increase the temperature until certain thresholds are hit.
389
+ """
390
+ # --8<-- [end:translation-sampling-params]
391
+
392
+ # --8<-- [start:translation-extra-params]
393
+ language: str | None = None
394
+ """The language of the input audio we translate from.
395
+
396
+ Supplying the input language in
397
+ [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
398
+ will improve accuracy.
399
+ """
400
+
401
+ to_language: str | None = None
402
+ """The language of the input audio we translate to.
403
+
404
+ Please note that this is not supported by all models, refer to the specific
405
+ model documentation for more details.
406
+ For instance, Whisper only supports `to_language=en`.
407
+ """
408
+
409
+ stream: bool | None = False
410
+ """Custom field not present in the original OpenAI definition. When set,
411
+ it will enable output to be streamed in a similar fashion as the Chat
412
+ Completion endpoint.
413
+ """
414
+ # Flattened stream option to simplify form data.
415
+ stream_include_usage: bool | None = False
416
+ stream_continuous_usage_stats: bool | None = False
417
+
418
+ max_completion_tokens: int | None = None
419
+ """The maximum number of tokens to generate."""
420
+ # --8<-- [end:translation-extra-params]
421
+
422
+ # Default sampling parameters for translation requests.
423
+ _DEFAULT_SAMPLING_PARAMS: dict = {
424
+ "temperature": 0,
425
+ }
426
+
427
+ def to_sampling_params(
428
+ self, default_max_tokens: int, default_sampling_params: dict | None = None
429
+ ) -> SamplingParams:
430
+ max_tokens = default_max_tokens
431
+
432
+ if default_sampling_params is None:
433
+ default_sampling_params = {}
434
+ # Default parameters
435
+ if (temperature := self.temperature) is None:
436
+ temperature = default_sampling_params.get(
437
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
438
+ )
439
+
440
+ return SamplingParams.from_optional(
441
+ temperature=temperature,
442
+ max_tokens=max_tokens,
443
+ seed=self.seed,
444
+ output_kind=RequestOutputKind.DELTA
445
+ if self.stream
446
+ else RequestOutputKind.FINAL_ONLY,
447
+ skip_clone=True, # Created fresh per request, safe to skip clone
448
+ )
449
+
450
+ @model_validator(mode="before")
451
+ @classmethod
452
+ def validate_stream_options(cls, data):
453
+ stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
454
+ stream = data.get("stream", False)
455
+ if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
456
+ # Find which specific stream option was set
457
+ invalid_param = next(
458
+ (so for so in stream_opts if data.get(so, False)),
459
+ "stream_include_usage",
460
+ )
461
+ raise VLLMValidationError(
462
+ "Stream options can only be defined when `stream=True`.",
463
+ parameter=invalid_param,
464
+ )
465
+
466
+ return data
467
+
468
+
469
+# Translation response objects
470
+class TranslationResponse(OpenAIBaseModel):
471
+ text: str
472
+ """The translated text."""
473
+
474
+
475
+class TranslationWord(OpenAIBaseModel):
476
+ end: float
477
+ """End time of the word in seconds."""
478
+
479
+ start: float
480
+ """Start time of the word in seconds."""
481
+
482
+ word: str
483
+ """The text content of the word."""
484
+
485
+
486
+class TranslationSegment(OpenAIBaseModel):
487
+ id: int
488
+ """Unique identifier of the segment."""
489
+
490
+ avg_logprob: float | None = None
491
+ """Average logprob of the segment.
492
+
493
+ If the value is lower than -1, consider the logprobs failed.
494
+ """
495
+
496
+ compression_ratio: float | None = None
497
+ """Compression ratio of the segment.
498
+
499
+ If the value is greater than 2.4, consider the compression failed.
500
+ """
501
+
502
+ end: float
503
+ """End time of the segment in seconds."""
504
+
505
+ no_speech_prob: float | None = None
506
+ """Probability of no speech in the segment.
507
+
508
+ If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
509
+ this segment silent.
510
+ """
511
+
512
+ seek: int
513
+ """Seek offset of the segment."""
514
+
515
+ start: float
516
+ """Start time of the segment in seconds."""
517
+
518
+ temperature: float
519
+ """Temperature parameter used for generating the segment."""
520
+
521
+ text: str
522
+ """Text content of the segment."""
523
+
524
+ tokens: list[int]
525
+ """Array of token IDs for the text content."""
526
+
527
+
528
+class TranslationResponseVerbose(OpenAIBaseModel):
529
+ duration: str
530
+ """The duration of the input audio."""
531
+
532
+ language: str
533
+ """The language of the input audio."""
534
+
535
+ text: str
536
+ """The translated text."""
537
+
538
+ segments: list[TranslationSegment] | None = None
539
+ """Segments of the translated text and their corresponding details."""
540
+
541
+ words: list[TranslationWord] | None = None
542
+ """Extracted words and their corresponding timestamps."""
543
+
544
+
545
+TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
vllm/entrypoints/openai/translations/serving.py
renamed
+4
-2
@@ -9,6 +9,9 @@ from vllm.entrypoints.logger import RequestLogger
9
from vllm.entrypoints.openai.engine.protocol import (
10
ErrorResponse,
11
RequestResponseMetadata,
12
+)
13
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
14
+from vllm.entrypoints.openai.translations.protocol import (
15
TranscriptionRequest,
16
TranscriptionResponse,
17
TranscriptionResponseStreamChoice,
@@ -20,8 +23,7 @@ from vllm.entrypoints.openai.engine.protocol import (
23
TranslationResponseVerbose,
24
TranslationStreamResponse,
25
)
23
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
24
-from vllm.entrypoints.openai.speech_to_text import OpenAISpeechToText
26
+from vllm.entrypoints.openai.translations.speech_to_text import OpenAISpeechToText
27
from vllm.logger import init_logger
28
from vllm.outputs import RequestOutput
29
vllm/entrypoints/openai/translations/speech_to_text.py
renamed
+6
-4
@@ -19,6 +19,11 @@ from vllm.entrypoints.openai.engine.protocol import (
19
DeltaMessage,
20
ErrorResponse,
21
RequestResponseMetadata,
22
+ UsageInfo,
23
+)
24
+from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
25
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26
+from vllm.entrypoints.openai.translations.protocol import (
27
TranscriptionResponse,
28
TranscriptionResponseStreamChoice,
29
TranscriptionResponseVerbose,
@@ -29,11 +34,8 @@ from vllm.entrypoints.openai.engine.protocol import (
34
TranslationResponseVerbose,
35
TranslationSegment,
36
TranslationStreamResponse,
32
- UsageInfo,
33
- VLLMValidationError,
37
)
35
-from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
36
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
38
+from vllm.exceptions import VLLMValidationError
39
from vllm.inputs.data import PromptType
40
from vllm.logger import init_logger
41
from vllm.model_executor.models import SupportsTranscription, supports_transcription