A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
feat: add usage to TranscriptionResponse (text and json response_format) (#23576)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Aug 26, 2025 at 14:26 UTC
ebd5a77bb5a6b7643f047f61294da0ce92baf3f6
3 files changed
+32
-5
tests/entrypoints/openai/test_transcription_validation.py
+10
-4
@@ -69,8 +69,11 @@ async def test_basic_audio(mary_had_lamb, model_name):
69
language="en",
70
response_format="text",
71
temperature=0.0)
72
- out = json.loads(transcription)['text']
73
- assert "Mary had a little lamb," in out
72
+ out = json.loads(transcription)
73
+ out_text = out['text']
74
+ out_usage = out['usage']
75
+ assert "Mary had a little lamb," in out_text
76
+ assert out_usage["seconds"] == 16, out_usage["seconds"]
77
78
79
@pytest.mark.asyncio
@@ -116,9 +119,12 @@ async def test_long_audio_request(mary_had_lamb, client):
119
language="en",
120
response_format="text",
121
temperature=0.0)
119
- out = json.loads(transcription)['text']
120
- counts = out.count("Mary had a little lamb")
122
+ out = json.loads(transcription)
123
+ out_text = out['text']
124
+ out_usage = out['usage']
125
+ counts = out_text.count("Mary had a little lamb")
126
assert counts == 10, counts
127
+ assert out_usage["seconds"] == 161, out_usage["seconds"]
128
129
130
@pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+6
@@ -2232,9 +2232,15 @@ class TranscriptionRequest(OpenAIBaseModel):
2232
2233
2234
# Transcription response objects
2235
+class TranscriptionUsageAudio(OpenAIBaseModel):
2236
+ type: Literal["duration"] = "duration"
2237
+ seconds: int
2238
+
2239
+
2240
class TranscriptionResponse(OpenAIBaseModel):
2241
text: str
2242
"""The transcribed text."""
2243
+ usage: TranscriptionUsageAudio
2244
2245
2246
class TranscriptionWord(OpenAIBaseModel):
vllm/entrypoints/openai/speech_to_text.py
+16
-1
@@ -200,7 +200,22 @@ class OpenAISpeechToText(OpenAIServing):
200
for result_generator in list_result_generator:
201
async for op in result_generator:
202
text += op.outputs[0].text
203
- return cast(T, response_class(text=text))
203
+
204
+ if self.task_type == "transcribe":
205
+ # add usage in TranscriptionResponse.
206
+ usage = {
207
+ "type": "duration",
208
+ # rounded up as per openAI specs
209
+ "seconds": int(math.ceil(duration_s)),
210
+ }
211
+ final_response = cast(T, response_class(text=text,
212
+ usage=usage))
213
+ else:
214
+ # no usage in response for translation task
215
+ final_response = cast(
216
+ T, response_class(text=text)) # type: ignore[call-arg]
217
+
218
+ return final_response
219
except asyncio.CancelledError:
220
return self.create_error_response("Client disconnected")
221
except ValueError as e: