@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d84cef76eb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Frontend] Add `/v1/audio/transcriptions` OpenAI API endpoint (#12909)

Nicolò Lucchesi committed Feb 13, 2025 at 16:23 UTC d84cef76eb9e16190cfdd97ae24511c8c819f179
20 files changed +910 -19
.buildkite/test-pipeline.yaml
+10 -2
@@ -117,7 +117,7 @@ steps:
117 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
118 - pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
119 - pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
120 - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py
120 + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/correctness/
121 - pytest -v -s entrypoints/test_chat_utils.py
122 - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
123
@@ -205,7 +205,7 @@ steps:
205 - VLLM_USE_V1=1 pytest -v -s v1/e2e
206 # Integration test for streaming correctness (requires special branch).
207 - pip install -U git+https://github.com/robertgshaw2-neuralmagic/lm-evaluation-harness.git@streaming-api
208 - - pytest -v -s entrypoints/openai/test_accuracy.py::test_lm_eval_accuracy_v1_engine
208 + - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
209
210 - label: Examples Test # 25min
211 working_dir: "/vllm-workspace/examples"
@@ -339,6 +339,14 @@ steps:
339 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
340 - bash ./run-tests.sh -c configs/models-small.txt -t 1
341
342 +- label: OpenAI API correctness
343 + source_file_dependencies:
344 + - csrc/
345 + - vllm/entrypoints/openai/
346 + - vllm/model_executor/models/whisper.py
347 + commands: # LMEval+Transcription WER check
348 + - pytest -s entrypoints/openai/correctness/
349 +
350 - label: Encoder Decoder tests # 5min
351 source_file_dependencies:
352 - vllm/
docs/source/serving/openai_compatible_server.md
+13
@@ -41,6 +41,8 @@ We currently support the following OpenAI APIs:
41 - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
42 - [Embeddings API](#embeddings-api) (`/v1/embeddings`)
43 - Only applicable to [embedding models](../models/pooling_models.md) (`--task embed`).
44 +- [Transcriptions API](#transcriptions-api) (`/v1/audio/transcriptions`)
45 + - Only applicable to Automatic Speech Recognition (ASR) models (OpenAI Whisper) (`--task generate`).
46
47 In addition, we have the following custom APIs:
48
@@ -296,6 +298,17 @@ For chat-like input (i.e. if `messages` is passed), these extra parameters are s
298 :end-before: end-chat-embedding-extra-params
299 :::
300
301 +(transcriptions-api)=
302 +
303 +### Transcriptions API
304 +
305 +Our Transcriptions API is compatible with [OpenAI's Transcriptions API](https://platform.openai.com/docs/api-reference/audio/createTranscription);
306 +you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
307 +
308 +<!-- TODO: api enforced limits + uploading audios -->
309 +
310 +Code example: <gh-file:examples/online_serving/openai_transcription_client.py>
311 +
312 (tokenizer-api)=
313
314 ### Tokenizer API
examples/online_serving/openai_transcription_client.py new
+23
@@ -0,0 +1,23 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +from openai import OpenAI
3 +
4 +from vllm.assets.audio import AudioAsset
5 +
6 +mary_had_lamb = AudioAsset('mary_had_lamb').get_local_path()
7 +winning_call = AudioAsset('winning_call').get_local_path()
8 +
9 +# Modify OpenAI's API key and API base to use vLLM's API server.
10 +openai_api_key = "EMPTY"
11 +openai_api_base = "http://localhost:8000/v1"
12 +client = OpenAI(
13 + api_key=openai_api_key,
14 + base_url=openai_api_base,
15 +)
16 +with open(str(mary_had_lamb), "rb") as f:
17 + transcription = client.audio.transcriptions.create(
18 + file=f,
19 + model="openai/whisper-large-v3",
20 + language="en",
21 + response_format="text",
22 + temperature=0.0)
23 + print("transcription result:", transcription)
requirements-common.txt
+3 -4
@@ -8,12 +8,11 @@ py-cpuinfo
8 transformers >= 4.48.2 # Required for Bamba model and Transformers backend.
9 tokenizers >= 0.19.1 # Required for Llama 3.
10 protobuf # Required by LlamaTokenizer.
11 -fastapi >= 0.107.0, < 0.113.0; python_version < '3.9'
12 -fastapi >= 0.107.0, != 0.113.*, != 0.114.0; python_version >= '3.9'
11 +fastapi[standard] >= 0.107.0, < 0.113.0; python_version < '3.9'
12 +fastapi[standard] >= 0.107.0, != 0.113.*, != 0.114.0; python_version >= '3.9'
13 aiohttp
14 openai >= 1.52.0 # Ensure modern openai package (ensure types module present and max_completion_tokens field support)
15 -uvicorn[standard]
16 -pydantic >= 2.9 # Required for fastapi >= 0.113.0
15 +pydantic >= 2.9
16 prometheus_client >= 0.18.0
17 pillow # Required for image processing
18 prometheus-fastapi-instrumentator >= 7.0.0
requirements-test.in
+1
@@ -19,6 +19,7 @@ pqdm
19 ray[adag]==2.40.0
20 sentence-transformers # required for embedding tests
21 soundfile # required for audio tests
22 +jiwer # required for audio tests
23 timm # required for internvl test
24 torch==2.5.1
25 torchaudio==2.5.1
requirements-test.txt
+5
@@ -66,6 +66,7 @@ charset-normalizer==3.4.0
66 click==8.1.7
67 # via
68 # black
69 + # jiwer
70 # nltk
71 # ray
72 colorama==0.4.6
@@ -187,6 +188,8 @@ jinja2==3.1.4
188 # via
189 # datamodel-code-generator
190 # torch
191 +jiwer==3.0.5
192 + # via -r requirements-test.in
193 jmespath==1.0.1
194 # via
195 # boto3
@@ -470,6 +473,8 @@ pyyaml==6.0.2
473 # timm
474 # transformers
475 # vocos
476 +rapidfuzz==3.12.1
477 + # via jiwer
478 ray[adag]==2.40.0
479 # via -r requirements-test.in
480 redis==5.2.0
tests/entrypoints/openai/correctness/__init__.py
tests/entrypoints/openai/correctness/test_lmeval.py renamed
+1 -1
@@ -13,7 +13,7 @@ import pytest
13
14 from vllm.platforms import current_platform
15
16 -from ...utils import RemoteOpenAIServer
16 +from ....utils import RemoteOpenAIServer
17
18 MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
19 NUM_CONCURRENT = 500
tests/entrypoints/openai/correctness/test_transcription_api_correctness.py new
+166
@@ -0,0 +1,166 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""
3 +Evaluate Transcription API correctness by computing Word Error Rate (WER)
4 +on a given ASR dataset. When provided, it will also compare the WER against
5 +a baseline.
6 +This simulates real work usage of the API and makes sure that the frontend and
7 +AsyncLLMEngine are working correctly.
8 +"""
9 +import asyncio
10 +import io
11 +import time
12 +from statistics import mean, median
13 +from typing import List
14 +
15 +import librosa
16 +import pytest
17 +import soundfile
18 +import torch
19 +from datasets import load_dataset
20 +from evaluate import load
21 +from transformers import AutoTokenizer
22 +
23 +from ....utils import RemoteOpenAIServer
24 +
25 +
26 +def to_bytes(y, sr):
27 + buffer = io.BytesIO()
28 + soundfile.write(buffer, y, sr, format="WAV")
29 + buffer.seek(0)
30 + return buffer
31 +
32 +
33 +async def transcribe_audio(client, tokenizer, y, sr):
34 + # Send loaded audio directly instead of loading from disk,
35 + # dont account for that time though
36 + with to_bytes(y, sr) as f:
37 + start_time = time.perf_counter()
38 + transcription = await client.audio.transcriptions.create(
39 + file=f,
40 + model=tokenizer.name_or_path,
41 + language="en",
42 + temperature=0.0,
43 + )
44 + end_time = time.perf_counter()
45 + # NOTE there's no streaming in transcriptions, can't measure ttft
46 + latency = end_time - start_time
47 + num_output_tokens = len(
48 + tokenizer(transcription.text, add_special_tokens=False).input_ids)
49 + return latency, num_output_tokens, transcription.text
50 +
51 +
52 +async def bound_transcribe(model_name, sem, client, audio, reference):
53 + tokenizer = AutoTokenizer.from_pretrained(model_name)
54 + # Use semaphore to limit concurrent requests.
55 + async with sem:
56 + result = await transcribe_audio(client, tokenizer, *audio)
57 + # Normalize *english* output/reference for evaluation.
58 + out = tokenizer.normalize(result[2])
59 + ref = tokenizer.normalize(reference)
60 + return result[:2] + (out, ref)
61 +
62 +
63 +async def process_dataset(model, client, data, concurrent_request):
64 + sem = asyncio.Semaphore(concurrent_request)
65 +
66 + # Warmup call as the first `librosa.load` server-side is quite slow.
67 + audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
68 + _ = await bound_transcribe(model, sem, client, (audio, sr), "")
69 +
70 + tasks: List[asyncio.Task] = []
71 + for sample in data:
72 + audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
73 + task = asyncio.create_task(
74 + bound_transcribe(model, sem, client, (audio, sr), sample["text"]))
75 + tasks.append(task)
76 + return await asyncio.gather(*tasks)
77 +
78 +
79 +def print_performance_metrics(results, total_time):
80 + latencies = [res[0] for res in results]
81 + total_tokens = sum([res[1] for res in results])
82 +
83 + total = len(results)
84 + print(f"Total Requests: {total}")
85 + print(f"Successful Requests: {len(latencies)}")
86 + print(f"Average Latency: {mean(latencies):.4f} seconds")
87 + print(f"Median Latency: {median(latencies):.4f} seconds")
88 + perc = sorted(latencies)[int(len(latencies) * 0.95) - 1]
89 + print(f"95th Percentile Latency: {perc:.4f} seconds")
90 + # Throughput
91 + req_throughput = len(latencies) / total_time
92 + print(f"Estimated req_Throughput: {req_throughput:.2f} requests/s")
93 + throughput = total_tokens / total_time
94 + print(f"Estimated Throughput: {throughput:.2f} tok/s")
95 +
96 +
97 +def add_duration(sample):
98 + y, sr = sample['audio']["array"], sample['audio']["sampling_rate"]
99 + sample['duration_ms'] = librosa.get_duration(y=y, sr=sr) * 1000
100 + return sample
101 +
102 +
103 +def load_hf_dataset(dataset_repo: str, split='validation', **hf_kwargs):
104 + ## Load and filter the dataset
105 + dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
106 + if 'duration_ms' not in dataset[0]:
107 + # compute duration to filter
108 + dataset = dataset.map(add_duration)
109 +
110 + # Whisper max supported duration
111 + dataset = dataset.filter(lambda example: example['duration_ms'] < 30000)
112 + return dataset
113 +
114 +
115 +def run_evaluation(model: str,
116 + client,
117 + dataset,
118 + max_concurrent_reqs: int,
119 + n_examples: int = -1,
120 + print_metrics: bool = True):
121 + if n_examples > 0:
122 + dataset = dataset.select(range(n_examples))
123 + start = time.perf_counter()
124 + results = asyncio.run(
125 + process_dataset(model, client, dataset, max_concurrent_reqs))
126 + end = time.perf_counter()
127 + total_time = end - start
128 + print(f"Total Test Time: {total_time:.4f} seconds")
129 + if print_metrics:
130 + print_performance_metrics(results, total_time)
131 + # Compute WER
132 + predictions = [res[2] for res in results]
133 + references = [res[3] for res in results]
134 + wer = load("wer")
135 + wer_score = 100 * wer.compute(references=references,
136 + predictions=predictions)
137 + print("WER:", wer_score)
138 + return wer_score
139 +
140 +
141 +# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
142 +@pytest.mark.parametrize("model_name", ["openai/whisper-large-v3"])
143 +# Original dataset is 20GB+ in size, hence we use a pre-filtered slice.
144 +@pytest.mark.parametrize(
145 + "dataset_repo", ["D4nt3/esb-datasets-earnings22-validation-tiny-filtered"])
146 +# NOTE: Expected WER measured with equivalent hf.transformers args:
147 +# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
148 +@pytest.mark.parametrize("expected_wer", [12.744980])
149 +def test_wer_correctness(model_name,
150 + dataset_repo,
151 + expected_wer,
152 + n_examples=-1,
153 + max_concurrent_request=None):
154 + with RemoteOpenAIServer(model_name, ['--enforce-eager']) as remote_server:
155 + dataset = load_hf_dataset(dataset_repo)
156 +
157 + if not max_concurrent_request:
158 + # No max concurrency
159 + max_concurrent_request = n_examples if n_examples > 0\
160 + else len(dataset)
161 +
162 + client = remote_server.get_async_client()
163 + wer = run_evaluation(model_name, client, dataset,
164 + max_concurrent_request, n_examples)
165 + if expected_wer:
166 + torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
tests/entrypoints/openai/test_transcription_validation.py new
+122
@@ -0,0 +1,122 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# imports for guided decoding tests
4 +import io
5 +import json
6 +
7 +import librosa
8 +import numpy as np
9 +import openai
10 +import pytest
11 +import soundfile as sf
12 +
13 +from vllm.assets.audio import AudioAsset
14 +
15 +from ...utils import RemoteOpenAIServer
16 +
17 +
18 +@pytest.fixture
19 +def mary_had_lamb():
20 + path = AudioAsset('mary_had_lamb').get_local_path()
21 + with open(str(path), "rb") as f:
22 + yield f
23 +
24 +
25 +@pytest.fixture
26 +def winning_call():
27 + path = AudioAsset('winning_call').get_local_path()
28 + with open(str(path), "rb") as f:
29 + yield f
30 +
31 +
32 +@pytest.mark.asyncio
33 +async def test_basic_audio(mary_had_lamb):
34 + model_name = "openai/whisper-large-v3-turbo"
35 + server_args = ["--enforce-eager"]
36 + # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
37 + prompt = "THE FIRST WORDS I SPOKE"
38 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
39 + client = remote_server.get_async_client()
40 + transcription = await client.audio.transcriptions.create(
41 + model=model_name,
42 + file=mary_had_lamb,
43 + language="en",
44 + response_format="text",
45 + temperature=0.0)
46 + out = json.loads(transcription)['text']
47 + assert "Mary had a little lamb," in out
48 + # This should "force" whisper to continue prompt in all caps
49 + transcription_wprompt = await client.audio.transcriptions.create(
50 + model=model_name,
51 + file=mary_had_lamb,
52 + language="en",
53 + response_format="text",
54 + prompt=prompt,
55 + temperature=0.0)
56 + out_capital = json.loads(transcription_wprompt)['text']
57 + assert prompt not in out_capital
58 +
59 +
60 +@pytest.mark.asyncio
61 +async def test_bad_requests(mary_had_lamb):
62 + model_name = "openai/whisper-small"
63 + server_args = ["--enforce-eager"]
64 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
65 + client = remote_server.get_async_client()
66 +
67 + # invalid language
68 + with pytest.raises(openai.BadRequestError):
69 + await client.audio.transcriptions.create(model=model_name,
70 + file=mary_had_lamb,
71 + language="hh",
72 + temperature=0.0)
73 +
74 + # Expect audio too long: repeat the timeseries
75 + mary_had_lamb.seek(0)
76 + audio, sr = librosa.load(mary_had_lamb)
77 + repeated_audio = np.tile(audio, 10)
78 + # Repeated audio to buffer
79 + buffer = io.BytesIO()
80 + sf.write(buffer, repeated_audio, sr, format='WAV')
81 + buffer.seek(0)
82 + with pytest.raises(openai.BadRequestError):
83 + await client.audio.transcriptions.create(model=model_name,
84 + file=buffer,
85 + language="en",
86 + temperature=0.0)
87 +
88 +
89 +@pytest.mark.asyncio
90 +async def test_non_asr_model(winning_call):
91 + # text to text model
92 + model_name = "JackFram/llama-68m"
93 + server_args = ["--enforce-eager"]
94 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
95 + client = remote_server.get_async_client()
96 + res = await client.audio.transcriptions.create(model=model_name,
97 + file=winning_call,
98 + language="en",
99 + temperature=0.0)
100 + assert res.code == 400 and not res.text
101 + assert res.message == "The model does not support Transcriptions API"
102 +
103 +
104 +@pytest.mark.asyncio
105 +async def test_completion_endpoints():
106 + # text to text model
107 + model_name = "openai/whisper-small"
108 + server_args = ["--enforce-eager"]
109 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
110 + client = remote_server.get_async_client()
111 + res = await client.chat.completions.create(
112 + model=model_name,
113 + messages=[{
114 + "role": "system",
115 + "content": "You are a helpful assistant."
116 + }])
117 + assert res.code == 400
118 + assert res.message == "The model does not support Chat Completions API"
119 +
120 + res = await client.completions.create(model=model_name, prompt="Hello")
121 + assert res.code == 400
122 + assert res.message == "The model does not support Completions API"
tests/test_config.py
+1
@@ -17,6 +17,7 @@ from vllm.platforms import current_platform
17 ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
18 ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),
19 ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
20 + ("openai/whisper-small", "transcription", "transcription"),
21 ],
22 )
23 def test_auto_task(model_id, expected_runner_type, expected_task):
vllm/assets/audio.py
+5
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from dataclasses import dataclass
4 +from pathlib import Path
5 from typing import Literal
6 from urllib.parse import urljoin
7
@@ -28,6 +29,10 @@ class AudioAsset:
29 s3_prefix=ASSET_DIR)
30 return librosa.load(audio_path, sr=None)
31
32 + def get_local_path(self) -> Path:
33 + return get_vllm_public_assets(filename=f"{self.name}.ogg",
34 + s3_prefix=ASSET_DIR)
35 +
36 @property
37 def url(self) -> str:
38 return urljoin(VLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
vllm/config.py
+8 -3
@@ -54,17 +54,18 @@ _POOLING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
54 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
55
56 TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",
57 - "score", "reward"]
57 + "score", "reward", "transcription"]
58
59 _ResolvedTask = Literal["generate", "embed", "classify", "score", "reward",
60 - "draft"]
60 + "draft", "transcription"]
61
62 -RunnerType = Literal["generate", "pooling", "draft"]
62 +RunnerType = Literal["generate", "pooling", "draft", "transcription"]
63
64 _RUNNER_TASKS: Dict[RunnerType, List[_ResolvedTask]] = {
65 "generate": ["generate"],
66 "pooling": ["embed", "classify", "score", "reward"],
67 "draft": ["draft"],
68 + "transcription": ["transcription"],
69 }
70
71 _TASK_RUNNER: Dict[_ResolvedTask, RunnerType] = {
@@ -484,6 +485,8 @@ class ModelConfig:
485 return "embed"
486 if ModelRegistry.is_cross_encoder_model(architectures):
487 return "score"
488 + if ModelRegistry.is_transcription_model(architectures):
489 + return "transcription"
490
491 suffix_to_preferred_task: List[Tuple[str, _ResolvedTask]] = [
492 # Other models follow this pattern
@@ -516,6 +519,8 @@ class ModelConfig:
519 runner_support: Dict[RunnerType, bool] = {
520 # NOTE: Listed from highest to lowest priority,
521 # in case the model supports multiple of them
522 + "transcription":
523 + ModelRegistry.is_transcription_model(architectures),
524 "generate": ModelRegistry.is_text_generation_model(architectures),
525 "pooling": ModelRegistry.is_pooling_model(architectures),
526 }
vllm/entrypoints/openai/api_server.py
+41 -2
@@ -16,10 +16,10 @@ from argparse import Namespace
16 from contextlib import asynccontextmanager
17 from functools import partial
18 from http import HTTPStatus
19 -from typing import AsyncIterator, Dict, Optional, Set, Tuple, Union
19 +from typing import Annotated, AsyncIterator, Dict, Optional, Set, Tuple, Union
20
21 import uvloop
22 -from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
22 +from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
23 from fastapi.exceptions import RequestValidationError
24 from fastapi.middleware.cors import CORSMiddleware
25 from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -61,6 +61,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
61 ScoreRequest, ScoreResponse,
62 TokenizeRequest,
63 TokenizeResponse,
64 + TranscriptionRequest,
65 + TranscriptionResponse,
66 UnloadLoraAdapterRequest)
67 from vllm.entrypoints.openai.reasoning_parsers import ReasoningParserManager
68 # yapf: enable
@@ -75,6 +77,8 @@ from vllm.entrypoints.openai.serving_rerank import JinaAIServingRerank
77 from vllm.entrypoints.openai.serving_score import OpenAIServingScores
78 from vllm.entrypoints.openai.serving_tokenization import (
79 OpenAIServingTokenization)
80 +from vllm.entrypoints.openai.serving_transcription import (
81 + OpenAIServingTranscription)
82 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
83 from vllm.entrypoints.utils import with_cancellation
84 from vllm.logger import init_logger
@@ -327,6 +331,10 @@ def tokenization(request: Request) -> OpenAIServingTokenization:
331 return request.app.state.openai_serving_tokenization
332
333
334 +def transcription(request: Request) -> OpenAIServingTranscription:
335 + return request.app.state.openai_serving_transcription
336 +
337 +
338 def engine_client(request: Request) -> EngineClient:
339 return request.app.state.engine_client
340
@@ -520,6 +528,31 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
528 return await create_score(request, raw_request)
529
530
531 +@router.post("/v1/audio/transcriptions")
532 +@with_cancellation
533 +async def create_transcriptions(request: Annotated[TranscriptionRequest,
534 + Form()],
535 + raw_request: Request):
536 +
537 + handler = transcription(raw_request)
538 + if handler is None:
539 + return base(raw_request).create_error_response(
540 + message="The model does not support Transcriptions API")
541 +
542 + audio_data = await request.file.read()
543 + generator = await handler.create_transcription(audio_data, request,
544 + raw_request)
545 +
546 + if isinstance(generator, ErrorResponse):
547 + return JSONResponse(content=generator.model_dump(),
548 + status_code=generator.code)
549 +
550 + elif isinstance(generator, TranscriptionResponse):
551 + return JSONResponse(content=generator.model_dump())
552 +
553 + return StreamingResponse(content=generator, media_type="text/event-stream")
554 +
555 +
556 @router.post("/rerank", dependencies=[Depends(validate_json_request)])
557 @with_cancellation
558 async def do_rerank(request: RerankRequest, raw_request: Request):
@@ -832,6 +865,12 @@ async def init_app_state(
865 chat_template=resolved_chat_template,
866 chat_template_content_format=args.chat_template_content_format,
867 )
868 + state.openai_serving_transcription = OpenAIServingTranscription(
869 + engine_client,
870 + model_config,
871 + state.openai_serving_models,
872 + request_logger=request_logger,
873 + ) if model_config.runner_type == "transcription" else None
874 state.task = model_config.task
875
876
vllm/entrypoints/openai/protocol.py
+162 -1
@@ -8,9 +8,10 @@ from argparse import Namespace
8 from typing import Any, ClassVar, Dict, List, Literal, Optional, Set, Union
9
10 import torch
11 +from fastapi import UploadFile
12 from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
13 ValidationInfo, field_validator, model_validator)
13 -from typing_extensions import Annotated
14 +from typing_extensions import Annotated, TypeAlias
15
16 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
17 from vllm.logger import init_logger
@@ -1426,3 +1427,163 @@ class LoadLoraAdapterRequest(BaseModel):
1427 class UnloadLoraAdapterRequest(BaseModel):
1428 lora_name: str
1429 lora_int_id: Optional[int] = Field(default=None)
1430 +
1431 +
1432 +## Protocols for Audio
1433 +AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json",
1434 + "vtt"]
1435 +
1436 +
1437 +class TranscriptionRequest(OpenAIBaseModel):
1438 + # Ordered by official OpenAI API documentation
1439 + #https://platform.openai.com/docs/api-reference/audio/createTranscription
1440 +
1441 + file: UploadFile
1442 + """
1443 + The audio file object (not file name) to transcribe, in one of these
1444 + formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
1445 + """
1446 +
1447 + model: str
1448 + """ID of the model to use.
1449 + """
1450 +
1451 + language: Optional[str] = None
1452 + """The language of the input audio.
1453 +
1454 + Supplying the input language in
1455 + [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
1456 + will improve accuracy and latency.
1457 + """
1458 +
1459 + prompt: str = Field(default="")
1460 + """An optional text to guide the model's style or continue a previous audio
1461 + segment.
1462 +
1463 + The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
1464 + should match the audio language.
1465 + """
1466 +
1467 + response_format: AudioResponseFormat = Field(default="json")
1468 + """
1469 + The format of the output, in one of these options: `json`, `text`, `srt`,
1470 + `verbose_json`, or `vtt`.
1471 + """
1472 +
1473 + ## TODO (varun) : Support if set to 0, certain thresholds are met !!
1474 + temperature: float = Field(default=0.0)
1475 + """The sampling temperature, between 0 and 1.
1476 +
1477 + Higher values like 0.8 will make the output more random, while lower values
1478 + like 0.2 will make it more focused / deterministic. If set to 0, the model
1479 + will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
1480 + to automatically increase the temperature until certain thresholds are hit.
1481 + """
1482 +
1483 + timestamp_granularities: List[Literal["word", "segment"]] = Field(
1484 + alias="timestamp_granularities[]", default=[])
1485 + """The timestamp granularities to populate for this transcription.
1486 +
1487 + `response_format` must be set `verbose_json` to use timestamp granularities.
1488 + Either or both of these options are supported: `word`, or `segment`. Note:
1489 + There is no additional latency for segment timestamps, but generating word
1490 + timestamps incurs additional latency.
1491 + """
1492 +
1493 + # Default sampling parameters for transcription requests.
1494 + _DEFAULT_SAMPLING_PARAMS: dict = {
1495 + "temperature": 0,
1496 + }
1497 +
1498 + def to_sampling_params(
1499 + self,
1500 + default_max_tokens: int,
1501 + default_sampling_params: Optional[dict] = None) -> SamplingParams:
1502 + # TODO(#9845): remove max_tokens when field is removed from OpenAI API
1503 + max_tokens = default_max_tokens
1504 +
1505 + if default_sampling_params is None:
1506 + default_sampling_params = {}
1507 + # Default parameters
1508 + if (temperature := self.temperature) is None:
1509 + temperature = default_sampling_params.get(
1510 + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
1511 +
1512 + return SamplingParams.from_optional(temperature=temperature,
1513 + max_tokens=max_tokens)
1514 +
1515 +
1516 +# Transcription response objects
1517 +class TranscriptionResponse(OpenAIBaseModel):
1518 + text: str
1519 + """The transcribed text."""
1520 +
1521 +
1522 +class TranscriptionWord(OpenAIBaseModel):
1523 + end: float
1524 + """End time of the word in seconds."""
1525 +
1526 + start: float
1527 + """Start time of the word in seconds."""
1528 +
1529 + word: str
1530 + """The text content of the word."""
1531 +
1532 +
1533 +class TranscriptionSegment(OpenAIBaseModel):
1534 + id: int
1535 + """Unique identifier of the segment."""
1536 +
1537 + avg_logprob: float
1538 + """Average logprob of the segment.
1539 +
1540 + If the value is lower than -1, consider the logprobs failed.
1541 + """
1542 +
1543 + compression_ratio: float
1544 + """Compression ratio of the segment.
1545 +
1546 + If the value is greater than 2.4, consider the compression failed.
1547 + """
1548 +
1549 + end: float
1550 + """End time of the segment in seconds."""
1551 +
1552 + no_speech_prob: float
1553 + """Probability of no speech in the segment.
1554 +
1555 + If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
1556 + this segment silent.
1557 + """
1558 +
1559 + seek: int
1560 + """Seek offset of the segment."""
1561 +
1562 + start: float
1563 + """Start time of the segment in seconds."""
1564 +
1565 + temperature: float
1566 + """Temperature parameter used for generating the segment."""
1567 +
1568 + text: str
1569 + """Text content of the segment."""
1570 +
1571 + tokens: List[int]
1572 + """Array of token IDs for the text content."""
1573 +
1574 +
1575 +class TranscriptionResponseVerbose(OpenAIBaseModel):
1576 + duration: str
1577 + """The duration of the input audio."""
1578 +
1579 + language: str
1580 + """The language of the input audio."""
1581 +
1582 + text: str
1583 + """The transcribed text."""
1584 +
1585 + segments: Optional[List[TranscriptionSegment]] = None
1586 + """Segments of the transcribed text and their corresponding details."""
1587 +
1588 + words: Optional[List[TranscriptionWord]] = None
1589 + """Extracted words and their corresponding timestamps."""
vllm/entrypoints/openai/serving_engine.py
+4 -2
@@ -31,7 +31,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
31 ErrorResponse, RerankRequest,
32 ScoreRequest,
33 TokenizeChatRequest,
34 - TokenizeCompletionRequest)
34 + TokenizeCompletionRequest,
35 + TranscriptionRequest)
36 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
37 from vllm.entrypoints.openai.tool_parsers import ToolParser
38 # yapf: enable
@@ -57,7 +58,8 @@ CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
58 ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
59 TokenizeChatRequest]
60
60 -AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest]
61 +AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest,
62 + TranscriptionRequest]
63
64
65 class TextTokensPrompt(TypedDict):
vllm/entrypoints/openai/serving_transcription.py new
+305
@@ -0,0 +1,305 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import asyncio
3 +import io
4 +from typing import AsyncGenerator, Optional, Union, cast
5 +
6 +from fastapi import Request
7 +
8 +from vllm.config import ModelConfig
9 +from vllm.engine.protocol import EngineClient
10 +from vllm.entrypoints.logger import RequestLogger
11 +from vllm.entrypoints.openai.protocol import (ErrorResponse,
12 + RequestResponseMetadata,
13 + TranscriptionRequest,
14 + TranscriptionResponse,
15 + TranscriptionResponseVerbose)
16 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
17 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
18 +from vllm.inputs.data import PromptType
19 +from vllm.logger import init_logger
20 +from vllm.outputs import RequestOutput
21 +from vllm.utils import PlaceholderModule
22 +
23 +try:
24 + import librosa
25 +except ImportError:
26 + librosa = PlaceholderModule("librosa") # type: ignore[assignment]
27 +
28 +logger = init_logger(__name__)
29 +
30 +# From https://platform.openai.com/docs/guides/speech-to-text/supported-languages#supported-languages
31 +# TODO these configs should live somewhere with the model so we can support
32 +# additional ones
33 +
34 +ISO639_1_SUPPORTED_LANGS = {
35 + "af": "Afrikaans",
36 + "ar": "Arabic",
37 + "hy": "Armenian",
38 + "az": "Azerbaijani",
39 + "be": "Belarusian",
40 + "bs": "Bosnian",
41 + "bg": "Bulgarian",
42 + "ca": "Catalan",
43 + "zh": "Chinese",
44 + "hr": "Croatian",
45 + "cs": "Czech",
46 + "da": "Danish",
47 + "nl": "Dutch",
48 + "en": "English",
49 + "et": "Estonian",
50 + "fi": "Finnish",
51 + "fr": "French",
52 + "gl": "Galician",
53 + "de": "German",
54 + "el": "Greek",
55 + "he": "Hebrew",
56 + "hi": "Hindi",
57 + "hu": "Hungarian",
58 + "is": "Icelandic",
59 + "id": "Indonesian",
60 + "it": "Italian",
61 + "ja": "Japanese",
62 + "kn": "Kannada",
63 + "kk": "Kazakh",
64 + "ko": "Korean",
65 + "lv": "Latvian",
66 + "lt": "Lithuanian",
67 + "mk": "Macedonian",
68 + "ms": "Malay",
69 + "mr": "Marathi",
70 + "mi": "Maori",
71 + "ne": "Nepali",
72 + "no": "Norwegian",
73 + "fa": "Persian",
74 + "pl": "Polish",
75 + "pt": "Portuguese",
76 + "ro": "Romanian",
77 + "ru": "Russian",
78 + "sr": "Serbian",
79 + "sk": "Slovak",
80 + "sl": "Slovenian",
81 + "es": "Spanish",
82 + "sw": "Swahili",
83 + "sv": "Swedish",
84 + "tl": "Tagalog",
85 + "ta": "Tamil",
86 + "th": "Thai",
87 + "tr": "Turkish",
88 + "uk": "Ukrainian",
89 + "ur": "Urdu",
90 + "vi": "Vietnamese",
91 + "cy": "Welsh"
92 +}
93 +ISO639_1_OTHER_LANGS = {
94 + "lo": "Lao",
95 + "jw": "Javanese",
96 + "tk": "Turkmen",
97 + "yi": "Yiddish",
98 + "so": "Somali",
99 + "bn": "Bengali",
100 + "nn": "Norwegian Nynorsk",
101 + "si": "Sinhala",
102 + "yo": "Yoruba",
103 + "sa": "Sanskrit",
104 + "mi": "Māori",
105 + "fo": "Faroese", # codespell:ignore
106 + "mt": "Maltese",
107 + "tg": "Tajik",
108 + "mg": "Malagasy",
109 + "haw": "Hawaiian",
110 + "km": "Khmer",
111 + "br": "Breton",
112 + "ps": "Pashto",
113 + "ln": "Lingala",
114 + "la": "Latin",
115 + "ml": "Malayalam",
116 + "sq": "Albanian",
117 + "su": "Sundanese",
118 + "eu": "Basque",
119 + "ka": "Georgian",
120 + "uz": "Uzbek",
121 + "sn": "Shona",
122 + "ht": "Haitian",
123 + "as": "Assamese",
124 + "mn": "Mongolian",
125 + "te": "Telugu",
126 + "pa": "Panjabi",
127 + "tt": "Tatar",
128 + "gu": "Gujarati",
129 + "oc": "Occitan",
130 + "ha": "Hausa",
131 + "ba": "Bashkir",
132 + "my": "Burmese",
133 + "sd": "Sindhi",
134 + "am": "Amharic",
135 + "lb": "Luxembourgish",
136 + "bo": "Tibetan"
137 +}
138 +
139 +# As per https://platform.openai.com/docs/guides/speech-to-text#overview.
140 +# TODO configurable
141 +MAX_AUDIO_CLIP_FILESIZE_MB = 25
142 +# TODO get from processor.feature_extractor.chunk_length
143 +MAX_AUDIO_CLIP_DURATION_S = 30
144 +
145 +
146 +class OpenAIServingTranscription(OpenAIServing):
147 +
148 + def __init__(
149 + self,
150 + engine_client: EngineClient,
151 + model_config: ModelConfig,
152 + models: OpenAIServingModels,
153 + *,
154 + request_logger: Optional[RequestLogger],
155 + return_tokens_as_token_ids: bool = False,
156 + ):
157 + super().__init__(engine_client=engine_client,
158 + model_config=model_config,
159 + models=models,
160 + request_logger=request_logger,
161 + return_tokens_as_token_ids=return_tokens_as_token_ids)
162 +
163 + diff_sampling_param = self.model_config.get_diff_sampling_param()
164 + if diff_sampling_param:
165 + logger.info(
166 + "Overwriting default completion sampling param with: %s",
167 + diff_sampling_param)
168 +
169 + async def _preprocess_transcription(
170 + self,
171 + request: TranscriptionRequest,
172 + audio_data: bytes,
173 + ) -> PromptType:
174 + # Validate request
175 + # TODO language should be optional and can be guessed.
176 + # For now we default to en. See
177 + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
178 + lang_token = f"<|{request.language}|>" if request.language else "<|en|>"
179 + if request.language:
180 + if request.language in ISO639_1_SUPPORTED_LANGS:
181 + pass
182 + elif request.language in ISO639_1_OTHER_LANGS:
183 + logger.warning(
184 + "The selected language %s has limited accuracy with"
185 + " reported WER>=0.5. Results may be less accurate "
186 + "for this choice.", request.language)
187 + else:
188 + raise ValueError(
189 + f"Unsupported language: {request.language}."
190 + "Language should be one of:" +
191 + f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +
192 + f"or {list(ISO639_1_OTHER_LANGS.values())}")
193 +
194 + if len(audio_data) / 1024**2 > MAX_AUDIO_CLIP_FILESIZE_MB:
195 + raise ValueError("Maximum file size exceeded.")
196 +
197 + with io.BytesIO(audio_data) as bytes_:
198 + y, sr = librosa.load(bytes_)
199 + if librosa.get_duration(y=y, sr=sr) > MAX_AUDIO_CLIP_DURATION_S:
200 + raise ValueError(
201 + f"Maximum clip duration ({MAX_AUDIO_CLIP_DURATION_S}s) "
202 + "exceeded.")
203 +
204 + prompt = {
205 + "encoder_prompt": {
206 + "prompt": "",
207 + "multi_modal_data": {
208 + "audio": (y, sr),
209 + },
210 + },
211 + "decoder_prompt":
212 + f"<|startoftranscript|>{lang_token}<|transcribe|><|notimestamps|>{request.prompt}"
213 + }
214 + return cast(PromptType, prompt)
215 +
216 + # TODO (varun) : Make verbose response work !
217 + async def create_transcription(
218 + self, audio_data: bytes, request: TranscriptionRequest,
219 + raw_request: Request
220 + ) -> Union[TranscriptionResponse, TranscriptionResponseVerbose,
221 + ErrorResponse]:
222 + """Transcription API similar to OpenAI's API.
223 +
224 + See https://platform.openai.com/docs/api-reference/audio/createTranscription
225 + for the API specification. This API mimics the OpenAI transcription API.
226 + """
227 + error_check_ret = await self._check_model(request)
228 + if error_check_ret is not None:
229 + return error_check_ret
230 +
231 + # If the engine is dead, raise the engine's DEAD_ERROR.
232 + # This is required for the streaming case, where we return a
233 + # success status before we actually start generating text :).
234 + if self.engine_client.errored:
235 + raise self.engine_client.dead_error
236 +
237 + if request.response_format not in ['text', 'json']:
238 + return self.create_error_response(
239 + "Currently only support response_format `text` or `json`")
240 +
241 + # TODO cmpl->transcription?
242 + request_id = f"cmpl-{self._base_request_id(raw_request)}"
243 +
244 + request_metadata = RequestResponseMetadata(request_id=request_id)
245 + if raw_request:
246 + raw_request.state.request_metadata = request_metadata
247 +
248 + try:
249 + (
250 + lora_request,
251 + prompt_adapter_request,
252 + ) = self._maybe_get_adapters(request)
253 +
254 + if lora_request:
255 + return self.create_error_response(
256 + "Currently do not support LoRA for Transcription.")
257 + if prompt_adapter_request:
258 + return self.create_error_response(
259 + "Currently do not support PromptAdapter for Transcription."
260 + )
261 +
262 + prompt = await self._preprocess_transcription(
263 + request=request,
264 + audio_data=audio_data,
265 + )
266 +
267 + except ValueError as e:
268 + logger.exception("Error in preprocessing prompt inputs")
269 + return self.create_error_response(str(e))
270 +
271 + result_generator: Optional[AsyncGenerator[RequestOutput, None]] = None
272 + try:
273 + # TODO(rob): subtract len of tokenized prompt.
274 + default_max_tokens = self.model_config.max_model_len
275 + default_params = self.model_config.get_diff_sampling_param()
276 + sampling_params = request.to_sampling_params(
277 + default_max_tokens, default_params)
278 +
279 + self._log_inputs(
280 + request_id,
281 + prompt['decoder_prompt'], # type: ignore
282 + params=sampling_params,
283 + lora_request=None,
284 + prompt_adapter_request=None)
285 +
286 + result_generator = self.engine_client.generate(
287 + prompt,
288 + sampling_params,
289 + request_id,
290 + )
291 + except ValueError as e:
292 + # TODO: Use a vllm-specific Validation Error
293 + return self.create_error_response(str(e))
294 +
295 + # TODO(rob): figure out a way to pipe streaming in.
296 + # Non-streaming response.
297 + try:
298 + async for op in result_generator:
299 + result = op
300 + return TranscriptionResponse(text=result.outputs[0].text)
301 + except asyncio.CancelledError:
302 + return self.create_error_response("Client disconnected")
303 + except ValueError as e:
304 + # TODO: Use a vllm-specific Validation Error
305 + return self.create_error_response(str(e))
vllm/model_executor/models/interfaces.py
+27
@@ -441,3 +441,30 @@ def supports_cross_encoding(
441 model: Union[Type[object], object],
442 ) -> Union[TypeIs[Type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
443 return is_pooling_model(model) and _supports_cross_encoding(model)
444 +
445 +
446 +@runtime_checkable
447 +class SupportsTranscription(Protocol):
448 + """The interface required for all models that support transcription."""
449 +
450 + supports_transcription: ClassVar[Literal[True]] = True
451 +
452 +
453 +@overload
454 +def supports_transcription(
455 + model: Type[object]) -> TypeIs[Type[SupportsTranscription]]:
456 + ...
457 +
458 +
459 +@overload
460 +def supports_transcription(model: object) -> TypeIs[SupportsTranscription]:
461 + ...
462 +
463 +
464 +def supports_transcription(
465 + model: Union[Type[object], object],
466 +) -> Union[TypeIs[Type[SupportsTranscription]], TypeIs[SupportsTranscription]]:
467 + if isinstance(model, type):
468 + return isinstance(model, SupportsTranscription)
469 +
470 + return isinstance(model, SupportsTranscription)
vllm/model_executor/models/registry.py
+10 -2
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22
23 from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
24 supports_cross_encoding, supports_multimodal,
25 - supports_pp)
25 + supports_pp, supports_transcription)
26 from .interfaces_base import is_text_generation_model
27
28 logger = init_logger(__name__)
@@ -224,6 +224,7 @@ class _ModelInfo:
224 has_inner_state: bool
225 is_attention_free: bool
226 is_hybrid: bool
227 + supports_transcription: bool
228
229 @staticmethod
230 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
@@ -237,7 +238,7 @@ class _ModelInfo:
238 has_inner_state=has_inner_state(model),
239 is_attention_free=is_attention_free(model),
240 is_hybrid=is_hybrid(model),
240 - )
241 + supports_transcription=supports_transcription(model))
242
243
244 class _BaseRegisteredModel(ABC):
@@ -485,6 +486,13 @@ class _ModelRegistry:
486 model_cls, _ = self.inspect_model_cls(architectures)
487 return model_cls.is_hybrid
488
489 + def is_transcription_model(
490 + self,
491 + architectures: Union[str, List[str]],
492 + ) -> bool:
493 + model_cls, _ = self.inspect_model_cls(architectures)
494 + return model_cls.supports_transcription
495 +
496
497 ModelRegistry = _ModelRegistry({
498 model_arch:
vllm/model_executor/models/whisper.py
+3 -2
@@ -31,7 +31,7 @@ from vllm.multimodal.audio import resample_audio
31 from vllm.sequence import SequenceData
32 from vllm.transformers_utils.processor import cached_get_processor
33
34 -from .interfaces import SupportsMultiModal
34 +from .interfaces import SupportsMultiModal, SupportsTranscription
35 from .utils import AutoWeightsLoader, WeightsMapper, make_layers
36
37 logger = init_logger(__name__)
@@ -637,7 +637,8 @@ def input_mapper_for_whisper(
637 @MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_whisper)
638 @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
639 "audio", get_max_whisper_audio_tokens)
640 -class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
640 +class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
641 + SupportsMultiModal):
642 packed_modules_mapping = {
643 "self_attn.qkv_proj": [
644 "self_attn.q_proj",