@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 092bb73b8a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 06:00 UTC

[Frontend] add 'verbose_json' and 'timestamp' feature on Whisper Transcription/Translation (#24209)

Signed-off-by: sangbumlikeagod <oironese@naver.com> Signed-off-by: sangbumlikeagod <98077576+sangbumlikeagod@users.noreply.github.com>

sangbumlikeagod committed Dec 2, 2025 at 02:19 UTC 092bb73b8a36ccdb6d6bbac897ba3aa79f660e36

8 files changed +224 -23

docs/serving/openai_compatible_server.md

@@ -456,6 +456,7 @@ For `verbose_json` response format:

456 ]

457 }

458 ```

459 +Currently “verbose_json” response format doesn’t support avg_logprob, compression_ratio, no_speech_prob.

460

461 #### Extra Parameters

462

tests/entrypoints/openai/test_transcription_validation_whisper.py

+13

@@ -235,3 +235,16 @@ async def test_audio_prompt(mary_had_lamb, whisper_client):

235 )

236 out_prompt = json.loads(transcription_wprompt)["text"]

237 assert prefix in out_prompt

238 +

239 +

240 +@pytest.mark.asyncio

241 +async def test_audio_with_timestamp(mary_had_lamb, whisper_client):

242 + transcription = await whisper_client.audio.transcriptions.create(

243 + model=MODEL_NAME,

244 + file=mary_had_lamb,

245 + language="en",

246 + response_format="verbose_json",

247 + temperature=0.0,

248 + )

249 + assert transcription.segments is not None

250 + assert len(transcription.segments) > 0

vllm/entrypoints/openai/api_server.py

+4 -4

@@ -68,9 +68,9 @@ from vllm.entrypoints.openai.protocol import (

68 TokenizeRequest,

69 TokenizeResponse,

70 TranscriptionRequest,

71 - TranscriptionResponse,

71 + TranscriptionResponseVariant,

72 TranslationRequest,

73 - TranslationResponse,

73 + TranslationResponseVariant,

74 )

75 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat

76 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion

@@ -809,7 +809,7 @@ async def create_transcriptions(

809 content=generator.model_dump(), status_code=generator.error.code

810 )

811

812 - elif isinstance(generator, TranscriptionResponse):

812 + elif isinstance(generator, TranscriptionResponseVariant):

813 return JSONResponse(content=generator.model_dump())

814

815 return StreamingResponse(content=generator, media_type="text/event-stream")

@@ -848,7 +848,7 @@ async def create_translations(

848 content=generator.model_dump(), status_code=generator.error.code

849 )

850

851 - elif isinstance(generator, TranslationResponse):

851 + elif isinstance(generator, TranslationResponseVariant):

852 return JSONResponse(content=generator.model_dump())

853

854 return StreamingResponse(content=generator, media_type="text/event-stream")

vllm/entrypoints/openai/protocol.py

+14 -6

@@ -2126,13 +2126,13 @@ class TranscriptionSegment(OpenAIBaseModel):

2126 id: int

2127 """Unique identifier of the segment."""

2128

2129 - avg_logprob: float

2129 + avg_logprob: float | None = None

2130 """Average logprob of the segment.

2131

2132 If the value is lower than -1, consider the logprobs failed.

2133 """

2134

2135 - compression_ratio: float

2135 + compression_ratio: float | None = None

2136 """Compression ratio of the segment.

2137

2138 If the value is greater than 2.4, consider the compression failed.

@@ -2141,7 +2141,7 @@ class TranscriptionSegment(OpenAIBaseModel):

2141 end: float

2142 """End time of the segment in seconds."""

2143

2144 - no_speech_prob: float

2144 + no_speech_prob: float | None = None

2145 """Probability of no speech in the segment.

2146

2147 If the value is higher than 1.0 and the `avg_logprob` is below -1, consider

@@ -2181,6 +2181,11 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):

2181 """Extracted words and their corresponding timestamps."""

2182

2183

2184 +TranscriptionResponseVariant: TypeAlias = (

2185 + TranscriptionResponse | TranscriptionResponseVerbose

2186 +)

2187 +

2188 +

2189 class TranslationResponseStreamChoice(OpenAIBaseModel):

2190 delta: DeltaMessage

2191 finish_reason: str | None = None

@@ -2325,13 +2330,13 @@ class TranslationSegment(OpenAIBaseModel):

2330 id: int

2331 """Unique identifier of the segment."""

2332

2328 - avg_logprob: float

2333 + avg_logprob: float | None = None

2334 """Average logprob of the segment.

2335

2336 If the value is lower than -1, consider the logprobs failed.

2337 """

2338

2334 - compression_ratio: float

2339 + compression_ratio: float | None = None

2340 """Compression ratio of the segment.

2341

2342 If the value is greater than 2.4, consider the compression failed.

@@ -2340,7 +2345,7 @@ class TranslationSegment(OpenAIBaseModel):

2345 end: float

2346 """End time of the segment in seconds."""

2347

2343 - no_speech_prob: float

2348 + no_speech_prob: float | None = None

2349 """Probability of no speech in the segment.

2350

2351 If the value is higher than 1.0 and the `avg_logprob` is below -1, consider

@@ -2380,6 +2385,9 @@ class TranslationResponseVerbose(OpenAIBaseModel):

2385 """Extracted words and their corresponding timestamps."""

2386

2387

2388 +TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose

2389 +

2390 +

2391 ####### Tokens IN <> Tokens OUT #######

2392 class GenerateRequest(BaseModel):

2393 request_id: str = Field(

vllm/entrypoints/openai/serving_transcription.py

+24 -4

@@ -12,10 +12,12 @@ from vllm.entrypoints.openai.protocol import (

12 TranscriptionRequest,

13 TranscriptionResponse,

14 TranscriptionResponseStreamChoice,

15 + TranscriptionResponseVerbose,

16 TranscriptionStreamResponse,

17 TranslationRequest,

18 TranslationResponse,

19 TranslationResponseStreamChoice,

20 + TranslationResponseVerbose,

21 TranslationStreamResponse,

22 )

23 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

@@ -51,7 +53,12 @@ class OpenAIServingTranscription(OpenAISpeechToText):

54 async def create_transcription(

55 self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request

54 - ) -> TranscriptionResponse | AsyncGenerator[str, None] | ErrorResponse:

56 + ) -> (

57 + TranscriptionResponse

58 + | TranscriptionResponseVerbose

59 + | AsyncGenerator[str, None]

60 + | ErrorResponse

61 + ):

62 """Transcription API similar to OpenAI's API.

64 See https://platform.openai.com/docs/api-reference/audio/createTranscription

@@ -61,7 +68,11 @@ class OpenAIServingTranscription(OpenAISpeechToText):

68 audio_data=audio_data,

69 request=request,

70 raw_request=raw_request,

64 - response_class=TranscriptionResponse,

71 + response_class=(

72 + TranscriptionResponseVerbose

73 + if request.response_format == "verbose_json"

74 + else TranscriptionResponse

75 + ),

76 stream_generator_method=self.transcription_stream_generator,

77 )

@@ -112,7 +123,12 @@ class OpenAIServingTranslation(OpenAISpeechToText):

123

124 async def create_translation(

125 self, audio_data: bytes, request: TranslationRequest, raw_request: Request

115 - ) -> TranslationResponse | AsyncGenerator[str, None] | ErrorResponse:

126 + ) -> (

127 + TranslationResponse

128 + | TranslationResponseVerbose

129 + | AsyncGenerator[str, None]

130 + | ErrorResponse

131 + ):

132 """Translation API similar to OpenAI's API.

133

134 See https://platform.openai.com/docs/api-reference/audio/createTranslation

@@ -122,7 +138,11 @@ class OpenAIServingTranslation(OpenAISpeechToText):

138 audio_data=audio_data,

139 request=request,

140 raw_request=raw_request,

125 - response_class=TranslationResponse,

141 + response_class=(

142 + TranslationResponseVerbose

143 + if request.response_format == "verbose_json"

144 + else TranslationResponse

145 + ),

146 stream_generator_method=self.translation_stream_generator,

147 )

148

vllm/entrypoints/openai/speech_to_text.py

+163 -9

@@ -10,6 +10,7 @@ from typing import Literal, TypeAlias, TypeVar, cast

11 import numpy as np

12 from fastapi import Request

13 +from transformers import PreTrainedTokenizerBase

15 import vllm.envs as envs

16 from vllm.engine.protocol import EngineClient

@@ -20,9 +21,13 @@ from vllm.entrypoints.openai.protocol import (

21 RequestResponseMetadata,

22 TranscriptionResponse,

23 TranscriptionResponseStreamChoice,

24 + TranscriptionResponseVerbose,

25 + TranscriptionSegment,

26 TranscriptionStreamResponse,

27 TranslationResponse,

28 TranslationResponseStreamChoice,

29 + TranslationResponseVerbose,

30 + TranslationSegment,

31 TranslationStreamResponse,

32 UsageInfo,

33 )

@@ -32,6 +37,7 @@ from vllm.inputs.data import PromptType

37 from vllm.logger import init_logger

38 from vllm.model_executor.models import SupportsTranscription

39 from vllm.outputs import RequestOutput

40 +from vllm.transformers_utils.tokenizer import get_tokenizer

41 from vllm.utils.import_utils import PlaceholderModule

43 try:

@@ -40,7 +46,20 @@ except ImportError:

46 librosa = PlaceholderModule("librosa") # type: ignore[assignment]

48 SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse

49 +SpeechToTextResponseVerbose: TypeAlias = (

50 + TranscriptionResponseVerbose | TranslationResponseVerbose

51 +)

52 +SpeechToTextSegment: TypeAlias = TranscriptionSegment | TranslationSegment

53 T = TypeVar("T", bound=SpeechToTextResponse)

54 +V = TypeVar("V", bound=SpeechToTextResponseVerbose)

55 +S = TypeVar("S", bound=SpeechToTextSegment)

56 +

57 +ResponseType: TypeAlias = (

58 + TranscriptionResponse

59 + | TranslationResponse

60 + | TranscriptionResponseVerbose

61 + | TranslationResponseVerbose

62 +)

64 logger = init_logger(__name__)

@@ -78,6 +97,14 @@ class OpenAISpeechToText(OpenAIServing):

97 self.enable_force_include_usage = enable_force_include_usage

99 self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB

100 + if self.model_cls.supports_segment_timestamp:

101 + self.tokenizer = cast(

102 + PreTrainedTokenizerBase,

103 + get_tokenizer(

104 + tokenizer_name=self.model_config.tokenizer,

105 + tokenizer_mode=self.model_config.tokenizer_mode,

106 + ),

107 + )

108

109 if self.default_sampling_params:

110 logger.info(

@@ -133,17 +160,87 @@ class OpenAISpeechToText(OpenAIServing):

160 request_prompt=request.prompt,

161 to_language=to_language,

162 )

163 + if request.response_format == "verbose_json":

164 + if not isinstance(prompt, dict):

165 + raise ValueError(f"Expected prompt to be a dict,got {type(prompt)}")

166 + prompt_dict = cast(dict, prompt)

167 + decoder_prompt = prompt.get("decoder_prompt")

168 + if not isinstance(decoder_prompt, str):

169 + raise ValueError(

170 + f"Expected decoder_prompt to bestr, got {type(decoder_prompt)}"

171 + )

172 + prompt_dict["decoder_prompt"] = decoder_prompt.replace(

173 + "<|notimestamps|>", "<|0.00|>"

174 + )

175 prompts.append(prompt)

176 return prompts, duration

177

178 + def _get_verbose_segments(

179 + self,

180 + tokens: tuple,

181 + request: SpeechToTextRequest,

182 + segment_class: type[SpeechToTextSegment],

183 + start_time: float = 0,

184 + ) -> list[SpeechToTextSegment]:

185 + """

186 + Convert tokens to verbose segments.

187 +

188 + This method expects the model to produce

189 + timestamps as tokens (similar to Whisper).

190 + If the tokens do not include timestamp information,

191 + the segments may not be generated correctly.

192 +

193 + Note: Fields like avg_logprob, compression_ratio,

194 + and no_speech_prob are not supported

195 + in this implementation and will be None. See docs for details.

196 + """

197 + BASE_OFFSET = 0.02

198 + init_token = self.tokenizer.encode("<|0.00|>", add_special_tokens=False)[0]

199 + if tokens[-1] == self.tokenizer.eos_token_id:

200 + tokens = tokens[:-1]

201 +

202 + tokens_with_start = (init_token,) + tokens

203 + segments: list[SpeechToTextSegment] = []

204 + last_timestamp_start = 0

205 +

206 + if tokens_with_start[-2] < init_token and tokens_with_start[-1] >= init_token:

207 + tokens_with_start = tokens_with_start + (tokens_with_start[-1],)

208 + for idx, token in enumerate(tokens_with_start):

209 + # Timestamp tokens (e.g., <|0.00|>) are assumed to be sorted.

210 + # If the ordering is violated, this slicing may produce incorrect results.

211 + if (

212 + token >= init_token

213 + and idx != 0

214 + and tokens_with_start[idx - 1] >= init_token

215 + ):

216 + sliced_timestamp_tokens = tokens_with_start[last_timestamp_start:idx]

217 + start_timestamp = sliced_timestamp_tokens[0] - init_token

218 + end_timestamp = sliced_timestamp_tokens[-1] - init_token

219 +

220 + casting_segment = cast(

221 + SpeechToTextSegment,

222 + segment_class(

223 + id=len(segments),

224 + seek=start_time,

225 + start=start_time + BASE_OFFSET * start_timestamp,

226 + end=start_time + BASE_OFFSET * end_timestamp,

227 + temperature=request.temperature,

228 + text=self.tokenizer.decode(sliced_timestamp_tokens[1:-1]),

229 + tokens=sliced_timestamp_tokens[1:-1],

230 + ),

231 + )

232 + segments.append(casting_segment)

233 + last_timestamp_start = idx

234 + return segments

235 +

236 async def _create_speech_to_text(

237 self,

238 audio_data: bytes,

239 request: SpeechToTextRequest,

240 raw_request: Request,

144 - response_class: type[T],

241 + response_class: type[T | V],

242 stream_generator_method: Callable[..., AsyncGenerator[str, None]],

146 - ) -> T | AsyncGenerator[str, None] | ErrorResponse:

243 + ) -> T | V | AsyncGenerator[str, None] | ErrorResponse:

244 """Base method for speech-to-text operations like transcription and

245 translation."""

246 error_check_ret = await self._check_model(request)

@@ -156,11 +253,24 @@ class OpenAISpeechToText(OpenAIServing):

253 if self.engine_client.errored:

254 raise self.engine_client.dead_error

255

159 - if request.response_format not in ["text", "json"]:

256 + if request.response_format not in ["text", "json", "verbose_json"]:

257 return self.create_error_response(

161 - "Currently only support response_format `text` or `json`"

258 + ("Currently only support response_format")

259 + + ("`text`, `json` or `verbose_json`")

260 )

261

262 + if (

263 + request.response_format == "verbose_json"

264 + and not self.model_cls.supports_segment_timestamp

265 + ):

266 + return self.create_error_response(

267 + f"Currently do not support verbose_json for {request.model}"

268 + )

269 +

270 + if request.response_format == "verbose_json" and request.stream:

271 + return self.create_error_response(

272 + "verbose_json format doesn't support streaming case"

273 + )

274 request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"

275

276 request_metadata = RequestResponseMetadata(request_id=request_id)

@@ -215,25 +325,69 @@ class OpenAISpeechToText(OpenAIServing):

325 request, list_result_generator, request_id, request_metadata, duration_s

326 )

327 # Non-streaming response.

328 + total_segments = []

329 + text_parts = []

330 try:

331 assert list_result_generator is not None

332 + segments_types: dict[str, type[SpeechToTextSegment]] = {

333 + "transcribe": TranscriptionSegment,

334 + "translate": TranslationSegment,

335 + }

336 + segment_class: type[SpeechToTextSegment] = segments_types[self.task_type]

337 text = ""

221 - for result_generator in list_result_generator:

338 + for idx, result_generator in enumerate(list_result_generator):

339 async for op in result_generator:

223 - text += op.outputs[0].text

340 + if request.response_format == "verbose_json":

341 + segments: list[SpeechToTextSegment] = (

342 + self._get_verbose_segments(

343 + tokens=tuple(op.outputs[0].token_ids),

344 + segment_class=segment_class,

345 + request=request,

346 + start_time=idx * self.asr_config.max_audio_clip_s,

347 + )

348 + )

349

350 + total_segments.extend(segments)

351 + text_parts.extend([seg.text for seg in segments])

352 + else:

353 + text_parts.append(op.outputs[0].text)

354 + text = "".join(text_parts)

355 if self.task_type == "transcribe":

356 + final_response: ResponseType

357 # add usage in TranscriptionResponse.

358 usage = {

359 "type": "duration",

360 # rounded up as per openAI specs

361 "seconds": int(math.ceil(duration_s)),

362 }

232 - final_response = cast(T, response_class(text=text, usage=usage))

363 + if request.response_format != "verbose_json":

364 + final_response = cast(

365 + T, TranscriptionResponse(text=text, usage=usage)

366 + )

367 + else:

368 + final_response = cast(

369 + V,

370 + TranscriptionResponseVerbose(

371 + text=text,

372 + language=request.language,

373 + duration=str(duration_s),

374 + segments=total_segments,

375 + ),

376 + )

377 else:

378 # no usage in response for translation task

235 - final_response = cast(T, response_class(text=text)) # type: ignore[call-arg]

236 -

379 + if request.response_format != "verbose_json":

380 + final_response = cast(T, TranslationResponse(text=text))

381 + else:

382 + final_response = cast(

383 + V,

384 + TranslationResponseVerbose(

385 + text=text,

386 + language=request.language,

387 + duration=str(duration_s),

388 + segments=total_segments,

389 + ),

390 + )

391 return final_response

392 except asyncio.CancelledError:

393 return self.create_error_response("Client disconnected")

vllm/model_executor/models/interfaces.py

@@ -837,6 +837,10 @@ class SupportsTranscription(Protocol):

837 Transcription models can opt out of text generation by setting this to

838 `True`.

839 """

840 + supports_segment_timestamp: ClassVar[bool] = False

841 + """

842 + Enables the segment timestamp option for supported models by setting this to `True`.

843 + """

844

845 def __init_subclass__(cls, **kwargs):

846 super().__init_subclass__(**kwargs)

vllm/model_executor/models/whisper.py

@@ -791,6 +791,7 @@ class WhisperForConditionalGeneration(

791

792 # Whisper only supports audio-conditioned generation.

793 supports_transcription_only = True

794 + supports_segment_timestamp = True

795 supported_languages = ISO639_1_SUPPORTED_LANGS

796

797 @classmethod