A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Frontend] Add `/v1/audio/transcriptions` OpenAI API endpoint (#12909)
Nicolò Lucchesi committed
Feb 13, 2025 at 16:23 UTC
d84cef76eb9e16190cfdd97ae24511c8c819f179
20 files changed
+910
-19
.buildkite/test-pipeline.yaml
+10
-2
@@ -117,7 +117,7 @@ steps:
117
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
118
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
119
- pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
120
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py
120
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/correctness/
121
- pytest -v -s entrypoints/test_chat_utils.py
122
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
123
@@ -205,7 +205,7 @@ steps:
205
- VLLM_USE_V1=1 pytest -v -s v1/e2e
206
# Integration test for streaming correctness (requires special branch).
207
- pip install -U git+https://github.com/robertgshaw2-neuralmagic/lm-evaluation-harness.git@streaming-api
208
- - pytest -v -s entrypoints/openai/test_accuracy.py::test_lm_eval_accuracy_v1_engine
208
+ - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
209
210
- label: Examples Test # 25min
211
working_dir: "/vllm-workspace/examples"
@@ -339,6 +339,14 @@ steps:
339
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
340
- bash ./run-tests.sh -c configs/models-small.txt -t 1
341
342
+- label: OpenAI API correctness
343
+ source_file_dependencies:
344
+ - csrc/
345
+ - vllm/entrypoints/openai/
346
+ - vllm/model_executor/models/whisper.py
347
+ commands: # LMEval+Transcription WER check
348
+ - pytest -s entrypoints/openai/correctness/
349
+
350
- label: Encoder Decoder tests # 5min
351
source_file_dependencies:
352
- vllm/
docs/source/serving/openai_compatible_server.md
+13
@@ -41,6 +41,8 @@ We currently support the following OpenAI APIs:
41
- *Note: `parallel_tool_calls` and `user` parameters are ignored.*
42
- [Embeddings API](#embeddings-api) (`/v1/embeddings`)
43
- Only applicable to [embedding models](../models/pooling_models.md) (`--task embed`).
44
+- [Transcriptions API](#transcriptions-api) (`/v1/audio/transcriptions`)
45
+ - Only applicable to Automatic Speech Recognition (ASR) models (OpenAI Whisper) (`--task generate`).
46
47
In addition, we have the following custom APIs:
48
@@ -296,6 +298,17 @@ For chat-like input (i.e. if `messages` is passed), these extra parameters are s
298
:end-before: end-chat-embedding-extra-params
299
:::
300
301
+(transcriptions-api)=
302
+
303
+### Transcriptions API
304
+
305
+Our Transcriptions API is compatible with [OpenAI's Transcriptions API](https://platform.openai.com/docs/api-reference/audio/createTranscription);
306
+you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
307
+
308
+<!-- TODO: api enforced limits + uploading audios -->
309
+
310
+Code example: <gh-file:examples/online_serving/openai_transcription_client.py>
311
+
312
(tokenizer-api)=
313
314
### Tokenizer API
examples/online_serving/openai_transcription_client.py
new
+23
@@ -0,0 +1,23 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from openai import OpenAI
3
+
4
+from vllm.assets.audio import AudioAsset
5
+
6
+mary_had_lamb = AudioAsset('mary_had_lamb').get_local_path()
7
+winning_call = AudioAsset('winning_call').get_local_path()
8
+
9
+# Modify OpenAI's API key and API base to use vLLM's API server.
10
+openai_api_key = "EMPTY"
11
+openai_api_base = "http://localhost:8000/v1"
12
+client = OpenAI(
13
+ api_key=openai_api_key,
14
+ base_url=openai_api_base,
15
+)
16
+with open(str(mary_had_lamb), "rb") as f:
17
+ transcription = client.audio.transcriptions.create(
18
+ file=f,
19
+ model="openai/whisper-large-v3",
20
+ language="en",
21
+ response_format="text",
22
+ temperature=0.0)
23
+ print("transcription result:", transcription)
requirements-common.txt
+3
-4
@@ -8,12 +8,11 @@ py-cpuinfo
8
transformers >= 4.48.2 # Required for Bamba model and Transformers backend.
9
tokenizers >= 0.19.1 # Required for Llama 3.
10
protobuf # Required by LlamaTokenizer.
11
-fastapi >= 0.107.0, < 0.113.0; python_version < '3.9'
12
-fastapi >= 0.107.0, != 0.113.*, != 0.114.0; python_version >= '3.9'
11
+fastapi[standard] >= 0.107.0, < 0.113.0; python_version < '3.9'
12
+fastapi[standard] >= 0.107.0, != 0.113.*, != 0.114.0; python_version >= '3.9'
13
aiohttp
14
openai >= 1.52.0 # Ensure modern openai package (ensure types module present and max_completion_tokens field support)
15
-uvicorn[standard]
16
-pydantic >= 2.9 # Required for fastapi >= 0.113.0
15
+pydantic >= 2.9
16
prometheus_client >= 0.18.0
17
pillow # Required for image processing
18
prometheus-fastapi-instrumentator >= 7.0.0
requirements-test.in
+1
@@ -19,6 +19,7 @@ pqdm
19
ray[adag]==2.40.0
20
sentence-transformers # required for embedding tests
21
soundfile # required for audio tests
22
+jiwer # required for audio tests
23
timm # required for internvl test
24
torch==2.5.1
25
torchaudio==2.5.1
requirements-test.txt
+5
@@ -66,6 +66,7 @@ charset-normalizer==3.4.0
66
click==8.1.7
67
# via
68
# black
69
+ # jiwer
70
# nltk
71
# ray
72
colorama==0.4.6
@@ -187,6 +188,8 @@ jinja2==3.1.4
188
# via
189
# datamodel-code-generator
190
# torch
191
+jiwer==3.0.5
192
+ # via -r requirements-test.in
193
jmespath==1.0.1
194
# via
195
# boto3
@@ -470,6 +473,8 @@ pyyaml==6.0.2
473
# timm
474
# transformers
475
# vocos
476
+rapidfuzz==3.12.1
477
+ # via jiwer
478
ray[adag]==2.40.0
479
# via -r requirements-test.in
480
redis==5.2.0
tests/entrypoints/openai/correctness/__init__.py
tests/entrypoints/openai/correctness/test_lmeval.py
renamed
+1
-1
@@ -13,7 +13,7 @@ import pytest
13
14
from vllm.platforms import current_platform
15
16
-from ...utils import RemoteOpenAIServer
16
+from ....utils import RemoteOpenAIServer
17
18
MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
19
NUM_CONCURRENT = 500
tests/entrypoints/openai/correctness/test_transcription_api_correctness.py
new
+166
@@ -0,0 +1,166 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""
3
+Evaluate Transcription API correctness by computing Word Error Rate (WER)
4
+on a given ASR dataset. When provided, it will also compare the WER against
5
+a baseline.
6
+This simulates real work usage of the API and makes sure that the frontend and
7
+AsyncLLMEngine are working correctly.
8
+"""
9
+import asyncio
10
+import io
11
+import time
12
+from statistics import mean, median
13
+from typing import List
14
+
15
+import librosa
16
+import pytest
17
+import soundfile
18
+import torch
19
+from datasets import load_dataset
20
+from evaluate import load
21
+from transformers import AutoTokenizer
22
+
23
+from ....utils import RemoteOpenAIServer
24
+
25
+
26
+def to_bytes(y, sr):
27
+ buffer = io.BytesIO()
28
+ soundfile.write(buffer, y, sr, format="WAV")
29
+ buffer.seek(0)
30
+ return buffer
31
+
32
+
33
+async def transcribe_audio(client, tokenizer, y, sr):
34
+ # Send loaded audio directly instead of loading from disk,
35
+ # dont account for that time though
36
+ with to_bytes(y, sr) as f:
37
+ start_time = time.perf_counter()
38
+ transcription = await client.audio.transcriptions.create(
39
+ file=f,
40
+ model=tokenizer.name_or_path,
41
+ language="en",
42
+ temperature=0.0,
43
+ )
44
+ end_time = time.perf_counter()
45
+ # NOTE there's no streaming in transcriptions, can't measure ttft
46
+ latency = end_time - start_time
47
+ num_output_tokens = len(
48
+ tokenizer(transcription.text, add_special_tokens=False).input_ids)
49
+ return latency, num_output_tokens, transcription.text
50
+
51
+
52
+async def bound_transcribe(model_name, sem, client, audio, reference):
53
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
54
+ # Use semaphore to limit concurrent requests.
55
+ async with sem:
56
+ result = await transcribe_audio(client, tokenizer, *audio)
57
+ # Normalize *english* output/reference for evaluation.
58
+ out = tokenizer.normalize(result[2])
59
+ ref = tokenizer.normalize(reference)
60
+ return result[:2] + (out, ref)
61
+
62
+
63
+async def process_dataset(model, client, data, concurrent_request):
64
+ sem = asyncio.Semaphore(concurrent_request)
65
+
66
+ # Warmup call as the first `librosa.load` server-side is quite slow.
67
+ audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
68
+ _ = await bound_transcribe(model, sem, client, (audio, sr), "")
69
+
70
+ tasks: List[asyncio.Task] = []
71
+ for sample in data:
72
+ audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
73
+ task = asyncio.create_task(
74
+ bound_transcribe(model, sem, client, (audio, sr), sample["text"]))
75
+ tasks.append(task)
76
+ return await asyncio.gather(*tasks)
77
+
78
+
79
+def print_performance_metrics(results, total_time):
80
+ latencies = [res[0] for res in results]
81
+ total_tokens = sum([res[1] for res in results])
82
+
83
+ total = len(results)
84
+ print(f"Total Requests: {total}")
85
+ print(f"Successful Requests: {len(latencies)}")
86
+ print(f"Average Latency: {mean(latencies):.4f} seconds")
87
+ print(f"Median Latency: {median(latencies):.4f} seconds")
88
+ perc = sorted(latencies)[int(len(latencies) * 0.95) - 1]
89
+ print(f"95th Percentile Latency: {perc:.4f} seconds")
90
+ # Throughput
91
+ req_throughput = len(latencies) / total_time
92
+ print(f"Estimated req_Throughput: {req_throughput:.2f} requests/s")
93
+ throughput = total_tokens / total_time
94
+ print(f"Estimated Throughput: {throughput:.2f} tok/s")
95
+
96
+
97
+def add_duration(sample):
98
+ y, sr = sample['audio']["array"], sample['audio']["sampling_rate"]
99
+ sample['duration_ms'] = librosa.get_duration(y=y, sr=sr) * 1000
100
+ return sample
101
+
102
+
103
+def load_hf_dataset(dataset_repo: str, split='validation', **hf_kwargs):
104
+ ## Load and filter the dataset
105
+ dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
106
+ if 'duration_ms' not in dataset[0]:
107
+ # compute duration to filter
108
+ dataset = dataset.map(add_duration)
109
+
110
+ # Whisper max supported duration
111
+ dataset = dataset.filter(lambda example: example['duration_ms'] < 30000)
112
+ return dataset
113
+
114
+
115
+def run_evaluation(model: str,
116
+ client,
117
+ dataset,
118
+ max_concurrent_reqs: int,
119
+ n_examples: int = -1,
120
+ print_metrics: bool = True):
121
+ if n_examples > 0:
122
+ dataset = dataset.select(range(n_examples))
123
+ start = time.perf_counter()
124
+ results = asyncio.run(
125
+ process_dataset(model, client, dataset, max_concurrent_reqs))
126
+ end = time.perf_counter()
127
+ total_time = end - start
128
+ print(f"Total Test Time: {total_time:.4f} seconds")
129
+ if print_metrics:
130
+ print_performance_metrics(results, total_time)
131
+ # Compute WER
132
+ predictions = [res[2] for res in results]
133
+ references = [res[3] for res in results]
134
+ wer = load("wer")
135
+ wer_score = 100 * wer.compute(references=references,
136
+ predictions=predictions)
137
+ print("WER:", wer_score)
138
+ return wer_score
139
+
140
+
141
+# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
142
+@pytest.mark.parametrize("model_name", ["openai/whisper-large-v3"])
143
+# Original dataset is 20GB+ in size, hence we use a pre-filtered slice.
144
+@pytest.mark.parametrize(
145
+ "dataset_repo", ["D4nt3/esb-datasets-earnings22-validation-tiny-filtered"])
146
+# NOTE: Expected WER measured with equivalent hf.transformers args:
147
+# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
148
+@pytest.mark.parametrize("expected_wer", [12.744980])
149
+def test_wer_correctness(model_name,
150
+ dataset_repo,
151
+ expected_wer,
152
+ n_examples=-1,
153
+ max_concurrent_request=None):
154
+ with RemoteOpenAIServer(model_name, ['--enforce-eager']) as remote_server:
155
+ dataset = load_hf_dataset(dataset_repo)
156
+
157
+ if not max_concurrent_request:
158
+ # No max concurrency
159
+ max_concurrent_request = n_examples if n_examples > 0\
160
+ else len(dataset)
161
+
162
+ client = remote_server.get_async_client()
163
+ wer = run_evaluation(model_name, client, dataset,
164
+ max_concurrent_request, n_examples)
165
+ if expected_wer:
166
+ torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
tests/entrypoints/openai/test_transcription_validation.py
new
+122
@@ -0,0 +1,122 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# imports for guided decoding tests
4
+import io
5
+import json
6
+
7
+import librosa
8
+import numpy as np
9
+import openai
10
+import pytest
11
+import soundfile as sf
12
+
13
+from vllm.assets.audio import AudioAsset
14
+
15
+from ...utils import RemoteOpenAIServer
16
+
17
+
18
+@pytest.fixture
19
+def mary_had_lamb():
20
+ path = AudioAsset('mary_had_lamb').get_local_path()
21
+ with open(str(path), "rb") as f:
22
+ yield f
23
+
24
+
25
+@pytest.fixture
26
+def winning_call():
27
+ path = AudioAsset('winning_call').get_local_path()
28
+ with open(str(path), "rb") as f:
29
+ yield f
30
+
31
+
32
+@pytest.mark.asyncio
33
+async def test_basic_audio(mary_had_lamb):
34
+ model_name = "openai/whisper-large-v3-turbo"
35
+ server_args = ["--enforce-eager"]
36
+ # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
37
+ prompt = "THE FIRST WORDS I SPOKE"
38
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
39
+ client = remote_server.get_async_client()
40
+ transcription = await client.audio.transcriptions.create(
41
+ model=model_name,
42
+ file=mary_had_lamb,
43
+ language="en",
44
+ response_format="text",
45
+ temperature=0.0)
46
+ out = json.loads(transcription)['text']
47
+ assert "Mary had a little lamb," in out
48
+ # This should "force" whisper to continue prompt in all caps
49
+ transcription_wprompt = await client.audio.transcriptions.create(
50
+ model=model_name,
51
+ file=mary_had_lamb,
52
+ language="en",
53
+ response_format="text",
54
+ prompt=prompt,
55
+ temperature=0.0)
56
+ out_capital = json.loads(transcription_wprompt)['text']
57
+ assert prompt not in out_capital
58
+
59
+
60
+@pytest.mark.asyncio
61
+async def test_bad_requests(mary_had_lamb):
62
+ model_name = "openai/whisper-small"
63
+ server_args = ["--enforce-eager"]
64
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
65
+ client = remote_server.get_async_client()
66
+
67
+ # invalid language
68
+ with pytest.raises(openai.BadRequestError):
69
+ await client.audio.transcriptions.create(model=model_name,
70
+ file=mary_had_lamb,
71
+ language="hh",
72
+ temperature=0.0)
73
+
74
+ # Expect audio too long: repeat the timeseries
75
+ mary_had_lamb.seek(0)
76
+ audio, sr = librosa.load(mary_had_lamb)
77
+ repeated_audio = np.tile(audio, 10)
78
+ # Repeated audio to buffer
79
+ buffer = io.BytesIO()
80
+ sf.write(buffer, repeated_audio, sr, format='WAV')
81
+ buffer.seek(0)
82
+ with pytest.raises(openai.BadRequestError):
83
+ await client.audio.transcriptions.create(model=model_name,
84
+ file=buffer,
85
+ language="en",
86
+ temperature=0.0)
87
+
88
+
89
+@pytest.mark.asyncio
90
+async def test_non_asr_model(winning_call):
91
+ # text to text model
92
+ model_name = "JackFram/llama-68m"
93
+ server_args = ["--enforce-eager"]
94
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
95
+ client = remote_server.get_async_client()
96
+ res = await client.audio.transcriptions.create(model=model_name,
97
+ file=winning_call,
98
+ language="en",
99
+ temperature=0.0)
100
+ assert res.code == 400 and not res.text
101
+ assert res.message == "The model does not support Transcriptions API"
102
+
103
+
104
+@pytest.mark.asyncio
105
+async def test_completion_endpoints():
106
+ # text to text model
107
+ model_name = "openai/whisper-small"
108
+ server_args = ["--enforce-eager"]
109
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
110
+ client = remote_server.get_async_client()
111
+ res = await client.chat.completions.create(
112
+ model=model_name,
113
+ messages=[{
114
+ "role": "system",
115
+ "content": "You are a helpful assistant."
116
+ }])
117
+ assert res.code == 400
118
+ assert res.message == "The model does not support Chat Completions API"
119
+
120
+ res = await client.completions.create(model=model_name, prompt="Hello")
121
+ assert res.code == 400
122
+ assert res.message == "The model does not support Completions API"
tests/test_config.py
+1
@@ -17,6 +17,7 @@ from vllm.platforms import current_platform
17
("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
18
("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),
19
("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
20
+ ("openai/whisper-small", "transcription", "transcription"),
21
],
22
)
23
def test_auto_task(model_id, expected_runner_type, expected_task):
vllm/assets/audio.py
+5
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from dataclasses import dataclass
4
+from pathlib import Path
5
from typing import Literal
6
from urllib.parse import urljoin
7
@@ -28,6 +29,10 @@ class AudioAsset:
29
s3_prefix=ASSET_DIR)
30
return librosa.load(audio_path, sr=None)
31
32
+ def get_local_path(self) -> Path:
33
+ return get_vllm_public_assets(filename=f"{self.name}.ogg",
34
+ s3_prefix=ASSET_DIR)
35
+
36
@property
37
def url(self) -> str:
38
return urljoin(VLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
vllm/config.py
+8
-3
@@ -54,17 +54,18 @@ _POOLING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
54
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
55
56
TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",
57
- "score", "reward"]
57
+ "score", "reward", "transcription"]
58
59
_ResolvedTask = Literal["generate", "embed", "classify", "score", "reward",
60
- "draft"]
60
+ "draft", "transcription"]
61
62
-RunnerType = Literal["generate", "pooling", "draft"]
62
+RunnerType = Literal["generate", "pooling", "draft", "transcription"]
63
64
_RUNNER_TASKS: Dict[RunnerType, List[_ResolvedTask]] = {
65
"generate": ["generate"],
66
"pooling": ["embed", "classify", "score", "reward"],
67
"draft": ["draft"],
68
+ "transcription": ["transcription"],
69
}
70
71
_TASK_RUNNER: Dict[_ResolvedTask, RunnerType] = {
@@ -484,6 +485,8 @@ class ModelConfig:
485
return "embed"
486
if ModelRegistry.is_cross_encoder_model(architectures):
487
return "score"
488
+ if ModelRegistry.is_transcription_model(architectures):
489
+ return "transcription"
490
491
suffix_to_preferred_task: List[Tuple[str, _ResolvedTask]] = [
492
# Other models follow this pattern
@@ -516,6 +519,8 @@ class ModelConfig:
519
runner_support: Dict[RunnerType, bool] = {
520
# NOTE: Listed from highest to lowest priority,
521
# in case the model supports multiple of them
522
+ "transcription":
523
+ ModelRegistry.is_transcription_model(architectures),
524
"generate": ModelRegistry.is_text_generation_model(architectures),
525
"pooling": ModelRegistry.is_pooling_model(architectures),
526
}
vllm/entrypoints/openai/api_server.py
+41
-2
@@ -16,10 +16,10 @@ from argparse import Namespace
16
from contextlib import asynccontextmanager
17
from functools import partial
18
from http import HTTPStatus
19
-from typing import AsyncIterator, Dict, Optional, Set, Tuple, Union
19
+from typing import Annotated, AsyncIterator, Dict, Optional, Set, Tuple, Union
20
21
import uvloop
22
-from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
22
+from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
23
from fastapi.exceptions import RequestValidationError
24
from fastapi.middleware.cors import CORSMiddleware
25
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -61,6 +61,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
61
ScoreRequest, ScoreResponse,
62
TokenizeRequest,
63
TokenizeResponse,
64
+ TranscriptionRequest,
65
+ TranscriptionResponse,
66
UnloadLoraAdapterRequest)
67
from vllm.entrypoints.openai.reasoning_parsers import ReasoningParserManager
68
# yapf: enable
@@ -75,6 +77,8 @@ from vllm.entrypoints.openai.serving_rerank import JinaAIServingRerank
77
from vllm.entrypoints.openai.serving_score import OpenAIServingScores
78
from vllm.entrypoints.openai.serving_tokenization import (
79
OpenAIServingTokenization)
80
+from vllm.entrypoints.openai.serving_transcription import (
81
+ OpenAIServingTranscription)
82
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
83
from vllm.entrypoints.utils import with_cancellation
84
from vllm.logger import init_logger
@@ -327,6 +331,10 @@ def tokenization(request: Request) -> OpenAIServingTokenization:
331
return request.app.state.openai_serving_tokenization
332
333
334
+def transcription(request: Request) -> OpenAIServingTranscription:
335
+ return request.app.state.openai_serving_transcription
336
+
337
+
338
def engine_client(request: Request) -> EngineClient:
339
return request.app.state.engine_client
340
@@ -520,6 +528,31 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
528
return await create_score(request, raw_request)
529
530
531
+@router.post("/v1/audio/transcriptions")
532
+@with_cancellation
533
+async def create_transcriptions(request: Annotated[TranscriptionRequest,
534
+ Form()],
535
+ raw_request: Request):
536
+
537
+ handler = transcription(raw_request)
538
+ if handler is None:
539
+ return base(raw_request).create_error_response(
540
+ message="The model does not support Transcriptions API")
541
+
542
+ audio_data = await request.file.read()
543
+ generator = await handler.create_transcription(audio_data, request,
544
+ raw_request)
545
+
546
+ if isinstance(generator, ErrorResponse):
547
+ return JSONResponse(content=generator.model_dump(),
548
+ status_code=generator.code)
549
+
550
+ elif isinstance(generator, TranscriptionResponse):
551
+ return JSONResponse(content=generator.model_dump())
552
+
553
+ return StreamingResponse(content=generator, media_type="text/event-stream")
554
+
555
+
556
@router.post("/rerank", dependencies=[Depends(validate_json_request)])
557
@with_cancellation
558
async def do_rerank(request: RerankRequest, raw_request: Request):
@@ -832,6 +865,12 @@ async def init_app_state(
865
chat_template=resolved_chat_template,
866
chat_template_content_format=args.chat_template_content_format,
867
)
868
+ state.openai_serving_transcription = OpenAIServingTranscription(
869
+ engine_client,
870
+ model_config,
871
+ state.openai_serving_models,
872
+ request_logger=request_logger,
873
+ ) if model_config.runner_type == "transcription" else None
874
state.task = model_config.task
875
876
vllm/entrypoints/openai/protocol.py
+162
-1
@@ -8,9 +8,10 @@ from argparse import Namespace
8
from typing import Any, ClassVar, Dict, List, Literal, Optional, Set, Union
9
10
import torch
11
+from fastapi import UploadFile
12
from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
13
ValidationInfo, field_validator, model_validator)
13
-from typing_extensions import Annotated
14
+from typing_extensions import Annotated, TypeAlias
15
16
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
17
from vllm.logger import init_logger
@@ -1426,3 +1427,163 @@ class LoadLoraAdapterRequest(BaseModel):
1427
class UnloadLoraAdapterRequest(BaseModel):
1428
lora_name: str
1429
lora_int_id: Optional[int] = Field(default=None)
1430
+
1431
+
1432
+## Protocols for Audio
1433
+AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json",
1434
+ "vtt"]
1435
+
1436
+
1437
+class TranscriptionRequest(OpenAIBaseModel):
1438
+ # Ordered by official OpenAI API documentation
1439
+ #https://platform.openai.com/docs/api-reference/audio/createTranscription
1440
+
1441
+ file: UploadFile
1442
+ """
1443
+ The audio file object (not file name) to transcribe, in one of these
1444
+ formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
1445
+ """
1446
+
1447
+ model: str
1448
+ """ID of the model to use.
1449
+ """
1450
+
1451
+ language: Optional[str] = None
1452
+ """The language of the input audio.
1453
+
1454
+ Supplying the input language in
1455
+ [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
1456
+ will improve accuracy and latency.
1457
+ """
1458
+
1459
+ prompt: str = Field(default="")
1460
+ """An optional text to guide the model's style or continue a previous audio
1461
+ segment.
1462
+
1463
+ The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
1464
+ should match the audio language.
1465
+ """
1466
+
1467
+ response_format: AudioResponseFormat = Field(default="json")
1468
+ """
1469
+ The format of the output, in one of these options: `json`, `text`, `srt`,
1470
+ `verbose_json`, or `vtt`.
1471
+ """
1472
+
1473
+ ## TODO (varun) : Support if set to 0, certain thresholds are met !!
1474
+ temperature: float = Field(default=0.0)
1475
+ """The sampling temperature, between 0 and 1.
1476
+
1477
+ Higher values like 0.8 will make the output more random, while lower values
1478
+ like 0.2 will make it more focused / deterministic. If set to 0, the model
1479
+ will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
1480
+ to automatically increase the temperature until certain thresholds are hit.
1481
+ """
1482
+
1483
+ timestamp_granularities: List[Literal["word", "segment"]] = Field(
1484
+ alias="timestamp_granularities[]", default=[])
1485
+ """The timestamp granularities to populate for this transcription.
1486
+
1487
+ `response_format` must be set `verbose_json` to use timestamp granularities.
1488
+ Either or both of these options are supported: `word`, or `segment`. Note:
1489
+ There is no additional latency for segment timestamps, but generating word
1490
+ timestamps incurs additional latency.
1491
+ """
1492
+
1493
+ # Default sampling parameters for transcription requests.
1494
+ _DEFAULT_SAMPLING_PARAMS: dict = {
1495
+ "temperature": 0,
1496
+ }
1497
+
1498
+ def to_sampling_params(
1499
+ self,
1500
+ default_max_tokens: int,
1501
+ default_sampling_params: Optional[dict] = None) -> SamplingParams:
1502
+ # TODO(#9845): remove max_tokens when field is removed from OpenAI API
1503
+ max_tokens = default_max_tokens
1504
+
1505
+ if default_sampling_params is None:
1506
+ default_sampling_params = {}
1507
+ # Default parameters
1508
+ if (temperature := self.temperature) is None:
1509
+ temperature = default_sampling_params.get(
1510
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
1511
+
1512
+ return SamplingParams.from_optional(temperature=temperature,
1513
+ max_tokens=max_tokens)
1514
+
1515
+
1516
+# Transcription response objects
1517
+class TranscriptionResponse(OpenAIBaseModel):
1518
+ text: str
1519
+ """The transcribed text."""
1520
+
1521
+
1522
+class TranscriptionWord(OpenAIBaseModel):
1523
+ end: float
1524
+ """End time of the word in seconds."""
1525
+
1526
+ start: float
1527
+ """Start time of the word in seconds."""
1528
+
1529
+ word: str
1530
+ """The text content of the word."""
1531
+
1532
+
1533
+class TranscriptionSegment(OpenAIBaseModel):
1534
+ id: int
1535
+ """Unique identifier of the segment."""
1536
+
1537
+ avg_logprob: float
1538
+ """Average logprob of the segment.
1539
+
1540
+ If the value is lower than -1, consider the logprobs failed.
1541
+ """
1542
+
1543
+ compression_ratio: float
1544
+ """Compression ratio of the segment.
1545
+
1546
+ If the value is greater than 2.4, consider the compression failed.
1547
+ """
1548
+
1549
+ end: float
1550
+ """End time of the segment in seconds."""
1551
+
1552
+ no_speech_prob: float
1553
+ """Probability of no speech in the segment.
1554
+
1555
+ If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
1556
+ this segment silent.
1557
+ """
1558
+
1559
+ seek: int
1560
+ """Seek offset of the segment."""
1561
+
1562
+ start: float
1563
+ """Start time of the segment in seconds."""
1564
+
1565
+ temperature: float
1566
+ """Temperature parameter used for generating the segment."""
1567
+
1568
+ text: str
1569
+ """Text content of the segment."""
1570
+
1571
+ tokens: List[int]
1572
+ """Array of token IDs for the text content."""
1573
+
1574
+
1575
+class TranscriptionResponseVerbose(OpenAIBaseModel):
1576
+ duration: str
1577
+ """The duration of the input audio."""
1578
+
1579
+ language: str
1580
+ """The language of the input audio."""
1581
+
1582
+ text: str
1583
+ """The transcribed text."""
1584
+
1585
+ segments: Optional[List[TranscriptionSegment]] = None
1586
+ """Segments of the transcribed text and their corresponding details."""
1587
+
1588
+ words: Optional[List[TranscriptionWord]] = None
1589
+ """Extracted words and their corresponding timestamps."""
vllm/entrypoints/openai/serving_engine.py
+4
-2
@@ -31,7 +31,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
31
ErrorResponse, RerankRequest,
32
ScoreRequest,
33
TokenizeChatRequest,
34
- TokenizeCompletionRequest)
34
+ TokenizeCompletionRequest,
35
+ TranscriptionRequest)
36
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
37
from vllm.entrypoints.openai.tool_parsers import ToolParser
38
# yapf: enable
@@ -57,7 +58,8 @@ CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
58
ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
59
TokenizeChatRequest]
60
60
-AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest]
61
+AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest,
62
+ TranscriptionRequest]
63
64
65
class TextTokensPrompt(TypedDict):
vllm/entrypoints/openai/serving_transcription.py
new
+305
@@ -0,0 +1,305 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import asyncio
3
+import io
4
+from typing import AsyncGenerator, Optional, Union, cast
5
+
6
+from fastapi import Request
7
+
8
+from vllm.config import ModelConfig
9
+from vllm.engine.protocol import EngineClient
10
+from vllm.entrypoints.logger import RequestLogger
11
+from vllm.entrypoints.openai.protocol import (ErrorResponse,
12
+ RequestResponseMetadata,
13
+ TranscriptionRequest,
14
+ TranscriptionResponse,
15
+ TranscriptionResponseVerbose)
16
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
17
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
18
+from vllm.inputs.data import PromptType
19
+from vllm.logger import init_logger
20
+from vllm.outputs import RequestOutput
21
+from vllm.utils import PlaceholderModule
22
+
23
+try:
24
+ import librosa
25
+except ImportError:
26
+ librosa = PlaceholderModule("librosa") # type: ignore[assignment]
27
+
28
+logger = init_logger(__name__)
29
+
30
+# From https://platform.openai.com/docs/guides/speech-to-text/supported-languages#supported-languages
31
+# TODO these configs should live somewhere with the model so we can support
32
+# additional ones
33
+
34
+ISO639_1_SUPPORTED_LANGS = {
35
+ "af": "Afrikaans",
36
+ "ar": "Arabic",
37
+ "hy": "Armenian",
38
+ "az": "Azerbaijani",
39
+ "be": "Belarusian",
40
+ "bs": "Bosnian",
41
+ "bg": "Bulgarian",
42
+ "ca": "Catalan",
43
+ "zh": "Chinese",
44
+ "hr": "Croatian",
45
+ "cs": "Czech",
46
+ "da": "Danish",
47
+ "nl": "Dutch",
48
+ "en": "English",
49
+ "et": "Estonian",
50
+ "fi": "Finnish",
51
+ "fr": "French",
52
+ "gl": "Galician",
53
+ "de": "German",
54
+ "el": "Greek",
55
+ "he": "Hebrew",
56
+ "hi": "Hindi",
57
+ "hu": "Hungarian",
58
+ "is": "Icelandic",
59
+ "id": "Indonesian",
60
+ "it": "Italian",
61
+ "ja": "Japanese",
62
+ "kn": "Kannada",
63
+ "kk": "Kazakh",
64
+ "ko": "Korean",
65
+ "lv": "Latvian",
66
+ "lt": "Lithuanian",
67
+ "mk": "Macedonian",
68
+ "ms": "Malay",
69
+ "mr": "Marathi",
70
+ "mi": "Maori",
71
+ "ne": "Nepali",
72
+ "no": "Norwegian",
73
+ "fa": "Persian",
74
+ "pl": "Polish",
75
+ "pt": "Portuguese",
76
+ "ro": "Romanian",
77
+ "ru": "Russian",
78
+ "sr": "Serbian",
79
+ "sk": "Slovak",
80
+ "sl": "Slovenian",
81
+ "es": "Spanish",
82
+ "sw": "Swahili",
83
+ "sv": "Swedish",
84
+ "tl": "Tagalog",
85
+ "ta": "Tamil",
86
+ "th": "Thai",
87
+ "tr": "Turkish",
88
+ "uk": "Ukrainian",
89
+ "ur": "Urdu",
90
+ "vi": "Vietnamese",
91
+ "cy": "Welsh"
92
+}
93
+ISO639_1_OTHER_LANGS = {
94
+ "lo": "Lao",
95
+ "jw": "Javanese",
96
+ "tk": "Turkmen",
97
+ "yi": "Yiddish",
98
+ "so": "Somali",
99
+ "bn": "Bengali",
100
+ "nn": "Norwegian Nynorsk",
101
+ "si": "Sinhala",
102
+ "yo": "Yoruba",
103
+ "sa": "Sanskrit",
104
+ "mi": "Māori",
105
+ "fo": "Faroese", # codespell:ignore
106
+ "mt": "Maltese",
107
+ "tg": "Tajik",
108
+ "mg": "Malagasy",
109
+ "haw": "Hawaiian",
110
+ "km": "Khmer",
111
+ "br": "Breton",
112
+ "ps": "Pashto",
113
+ "ln": "Lingala",
114
+ "la": "Latin",
115
+ "ml": "Malayalam",
116
+ "sq": "Albanian",
117
+ "su": "Sundanese",
118
+ "eu": "Basque",
119
+ "ka": "Georgian",
120
+ "uz": "Uzbek",
121
+ "sn": "Shona",
122
+ "ht": "Haitian",
123
+ "as": "Assamese",
124
+ "mn": "Mongolian",
125
+ "te": "Telugu",
126
+ "pa": "Panjabi",
127
+ "tt": "Tatar",
128
+ "gu": "Gujarati",
129
+ "oc": "Occitan",
130
+ "ha": "Hausa",
131
+ "ba": "Bashkir",
132
+ "my": "Burmese",
133
+ "sd": "Sindhi",
134
+ "am": "Amharic",
135
+ "lb": "Luxembourgish",
136
+ "bo": "Tibetan"
137
+}
138
+
139
+# As per https://platform.openai.com/docs/guides/speech-to-text#overview.
140
+# TODO configurable
141
+MAX_AUDIO_CLIP_FILESIZE_MB = 25
142
+# TODO get from processor.feature_extractor.chunk_length
143
+MAX_AUDIO_CLIP_DURATION_S = 30
144
+
145
+
146
+class OpenAIServingTranscription(OpenAIServing):
147
+
148
+ def __init__(
149
+ self,
150
+ engine_client: EngineClient,
151
+ model_config: ModelConfig,
152
+ models: OpenAIServingModels,
153
+ *,
154
+ request_logger: Optional[RequestLogger],
155
+ return_tokens_as_token_ids: bool = False,
156
+ ):
157
+ super().__init__(engine_client=engine_client,
158
+ model_config=model_config,
159
+ models=models,
160
+ request_logger=request_logger,
161
+ return_tokens_as_token_ids=return_tokens_as_token_ids)
162
+
163
+ diff_sampling_param = self.model_config.get_diff_sampling_param()
164
+ if diff_sampling_param:
165
+ logger.info(
166
+ "Overwriting default completion sampling param with: %s",
167
+ diff_sampling_param)
168
+
169
+ async def _preprocess_transcription(
170
+ self,
171
+ request: TranscriptionRequest,
172
+ audio_data: bytes,
173
+ ) -> PromptType:
174
+ # Validate request
175
+ # TODO language should be optional and can be guessed.
176
+ # For now we default to en. See
177
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
178
+ lang_token = f"<|{request.language}|>" if request.language else "<|en|>"
179
+ if request.language:
180
+ if request.language in ISO639_1_SUPPORTED_LANGS:
181
+ pass
182
+ elif request.language in ISO639_1_OTHER_LANGS:
183
+ logger.warning(
184
+ "The selected language %s has limited accuracy with"
185
+ " reported WER>=0.5. Results may be less accurate "
186
+ "for this choice.", request.language)
187
+ else:
188
+ raise ValueError(
189
+ f"Unsupported language: {request.language}."
190
+ "Language should be one of:" +
191
+ f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +
192
+ f"or {list(ISO639_1_OTHER_LANGS.values())}")
193
+
194
+ if len(audio_data) / 1024**2 > MAX_AUDIO_CLIP_FILESIZE_MB:
195
+ raise ValueError("Maximum file size exceeded.")
196
+
197
+ with io.BytesIO(audio_data) as bytes_:
198
+ y, sr = librosa.load(bytes_)
199
+ if librosa.get_duration(y=y, sr=sr) > MAX_AUDIO_CLIP_DURATION_S:
200
+ raise ValueError(
201
+ f"Maximum clip duration ({MAX_AUDIO_CLIP_DURATION_S}s) "
202
+ "exceeded.")
203
+
204
+ prompt = {
205
+ "encoder_prompt": {
206
+ "prompt": "",
207
+ "multi_modal_data": {
208
+ "audio": (y, sr),
209
+ },
210
+ },
211
+ "decoder_prompt":
212
+ f"<|startoftranscript|>{lang_token}<|transcribe|><|notimestamps|>{request.prompt}"
213
+ }
214
+ return cast(PromptType, prompt)
215
+
216
+ # TODO (varun) : Make verbose response work !
217
+ async def create_transcription(
218
+ self, audio_data: bytes, request: TranscriptionRequest,
219
+ raw_request: Request
220
+ ) -> Union[TranscriptionResponse, TranscriptionResponseVerbose,
221
+ ErrorResponse]:
222
+ """Transcription API similar to OpenAI's API.
223
+
224
+ See https://platform.openai.com/docs/api-reference/audio/createTranscription
225
+ for the API specification. This API mimics the OpenAI transcription API.
226
+ """
227
+ error_check_ret = await self._check_model(request)
228
+ if error_check_ret is not None:
229
+ return error_check_ret
230
+
231
+ # If the engine is dead, raise the engine's DEAD_ERROR.
232
+ # This is required for the streaming case, where we return a
233
+ # success status before we actually start generating text :).
234
+ if self.engine_client.errored:
235
+ raise self.engine_client.dead_error
236
+
237
+ if request.response_format not in ['text', 'json']:
238
+ return self.create_error_response(
239
+ "Currently only support response_format `text` or `json`")
240
+
241
+ # TODO cmpl->transcription?
242
+ request_id = f"cmpl-{self._base_request_id(raw_request)}"
243
+
244
+ request_metadata = RequestResponseMetadata(request_id=request_id)
245
+ if raw_request:
246
+ raw_request.state.request_metadata = request_metadata
247
+
248
+ try:
249
+ (
250
+ lora_request,
251
+ prompt_adapter_request,
252
+ ) = self._maybe_get_adapters(request)
253
+
254
+ if lora_request:
255
+ return self.create_error_response(
256
+ "Currently do not support LoRA for Transcription.")
257
+ if prompt_adapter_request:
258
+ return self.create_error_response(
259
+ "Currently do not support PromptAdapter for Transcription."
260
+ )
261
+
262
+ prompt = await self._preprocess_transcription(
263
+ request=request,
264
+ audio_data=audio_data,
265
+ )
266
+
267
+ except ValueError as e:
268
+ logger.exception("Error in preprocessing prompt inputs")
269
+ return self.create_error_response(str(e))
270
+
271
+ result_generator: Optional[AsyncGenerator[RequestOutput, None]] = None
272
+ try:
273
+ # TODO(rob): subtract len of tokenized prompt.
274
+ default_max_tokens = self.model_config.max_model_len
275
+ default_params = self.model_config.get_diff_sampling_param()
276
+ sampling_params = request.to_sampling_params(
277
+ default_max_tokens, default_params)
278
+
279
+ self._log_inputs(
280
+ request_id,
281
+ prompt['decoder_prompt'], # type: ignore
282
+ params=sampling_params,
283
+ lora_request=None,
284
+ prompt_adapter_request=None)
285
+
286
+ result_generator = self.engine_client.generate(
287
+ prompt,
288
+ sampling_params,
289
+ request_id,
290
+ )
291
+ except ValueError as e:
292
+ # TODO: Use a vllm-specific Validation Error
293
+ return self.create_error_response(str(e))
294
+
295
+ # TODO(rob): figure out a way to pipe streaming in.
296
+ # Non-streaming response.
297
+ try:
298
+ async for op in result_generator:
299
+ result = op
300
+ return TranscriptionResponse(text=result.outputs[0].text)
301
+ except asyncio.CancelledError:
302
+ return self.create_error_response("Client disconnected")
303
+ except ValueError as e:
304
+ # TODO: Use a vllm-specific Validation Error
305
+ return self.create_error_response(str(e))
vllm/model_executor/models/interfaces.py
+27
@@ -441,3 +441,30 @@ def supports_cross_encoding(
441
model: Union[Type[object], object],
442
) -> Union[TypeIs[Type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
443
return is_pooling_model(model) and _supports_cross_encoding(model)
444
+
445
+
446
+@runtime_checkable
447
+class SupportsTranscription(Protocol):
448
+ """The interface required for all models that support transcription."""
449
+
450
+ supports_transcription: ClassVar[Literal[True]] = True
451
+
452
+
453
+@overload
454
+def supports_transcription(
455
+ model: Type[object]) -> TypeIs[Type[SupportsTranscription]]:
456
+ ...
457
+
458
+
459
+@overload
460
+def supports_transcription(model: object) -> TypeIs[SupportsTranscription]:
461
+ ...
462
+
463
+
464
+def supports_transcription(
465
+ model: Union[Type[object], object],
466
+) -> Union[TypeIs[Type[SupportsTranscription]], TypeIs[SupportsTranscription]]:
467
+ if isinstance(model, type):
468
+ return isinstance(model, SupportsTranscription)
469
+
470
+ return isinstance(model, SupportsTranscription)
vllm/model_executor/models/registry.py
+10
-2
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22
23
from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
24
supports_cross_encoding, supports_multimodal,
25
- supports_pp)
25
+ supports_pp, supports_transcription)
26
from .interfaces_base import is_text_generation_model
27
28
logger = init_logger(__name__)
@@ -224,6 +224,7 @@ class _ModelInfo:
224
has_inner_state: bool
225
is_attention_free: bool
226
is_hybrid: bool
227
+ supports_transcription: bool
228
229
@staticmethod
230
def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
@@ -237,7 +238,7 @@ class _ModelInfo:
238
has_inner_state=has_inner_state(model),
239
is_attention_free=is_attention_free(model),
240
is_hybrid=is_hybrid(model),
240
- )
241
+ supports_transcription=supports_transcription(model))
242
243
244
class _BaseRegisteredModel(ABC):
@@ -485,6 +486,13 @@ class _ModelRegistry:
486
model_cls, _ = self.inspect_model_cls(architectures)
487
return model_cls.is_hybrid
488
489
+ def is_transcription_model(
490
+ self,
491
+ architectures: Union[str, List[str]],
492
+ ) -> bool:
493
+ model_cls, _ = self.inspect_model_cls(architectures)
494
+ return model_cls.supports_transcription
495
+
496
497
ModelRegistry = _ModelRegistry({
498
model_arch:
vllm/model_executor/models/whisper.py
+3
-2
@@ -31,7 +31,7 @@ from vllm.multimodal.audio import resample_audio
31
from vllm.sequence import SequenceData
32
from vllm.transformers_utils.processor import cached_get_processor
33
34
-from .interfaces import SupportsMultiModal
34
+from .interfaces import SupportsMultiModal, SupportsTranscription
35
from .utils import AutoWeightsLoader, WeightsMapper, make_layers
36
37
logger = init_logger(__name__)
@@ -637,7 +637,8 @@ def input_mapper_for_whisper(
637
@MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_whisper)
638
@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
639
"audio", get_max_whisper_audio_tokens)
640
-class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
640
+class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
641
+ SupportsMultiModal):
642
packed_modules_mapping = {
643
"self_attn.qkv_proj": [
644
"self_attn.q_proj",