@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2cb2340f7a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 16:30 UTC

[Frontend]Add support for transcriptions and translations to run_batch (#33934)

Signed-off-by: Pooya Davoodi <pooya.davoodi@parasail.io> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Pooya Davoodi committed Feb 7, 2026 at 05:24 UTC 2cb2340f7a91a2f6629aacc45563af20897812c0
3 files changed +555 -101
tests/entrypoints/openai/test_run_batch.py
+140
@@ -7,6 +7,7 @@ import tempfile
7
8 import pytest
9
10 +from vllm.assets.audio import AudioAsset
11 from vllm.entrypoints.openai.run_batch import BatchRequestOutput
12
13 MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@@ -42,6 +43,27 @@ INPUT_RERANK_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/re
43 INPUT_REASONING_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Solve this math problem: 2+2=?"}]}}
44 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "What is the capital of France?"}]}}"""
45
46 +# This is a valid but minimal audio file for testing
47 +MINIMAL_WAV_BASE64 = "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQAAAAA="
48 +INPUT_TRANSCRIPTION_BATCH = (
49 + '{{"custom_id": "request-1", "method": "POST", "url": "/v1/audio/transcriptions", '
50 + '"body": {{"model": "openai/whisper-large-v3", "file_url": "data:audio/wav;base64,{}", '
51 + '"response_format": "json"}}}}\n'
52 +).format(MINIMAL_WAV_BASE64)
53 +
54 +INPUT_TRANSCRIPTION_HTTP_BATCH = (
55 + '{{"custom_id": "request-1", "method": "POST", "url": "/v1/audio/transcriptions", '
56 + '"body": {{"model": "openai/whisper-large-v3", "file_url": "{}", '
57 + '"response_format": "json"}}}}\n'
58 +).format(AudioAsset("mary_had_lamb").url)
59 +
60 +INPUT_TRANSLATION_BATCH = (
61 + '{{"custom_id": "request-1", "method": "POST", "url": "/v1/audio/translations", '
62 + '"body": {{"model": "openai/whisper-small", "file_url": "{}", '
63 + '"response_format": "text", "language": "it", "to_language": "en", '
64 + '"temperature": 0.0}}}}\n'
65 +).format(AudioAsset("mary_had_lamb").url)
66 +
67
68 def test_empty_file():
69 with (
@@ -238,3 +260,121 @@ def test_reasoning_parser():
260 ]
261 assert reasoning is not None
262 assert len(reasoning) > 0
263 +
264 +
265 +def test_transcription():
266 + with (
267 + tempfile.NamedTemporaryFile("w") as input_file,
268 + tempfile.NamedTemporaryFile("r") as output_file,
269 + ):
270 + input_file.write(INPUT_TRANSCRIPTION_BATCH)
271 + input_file.flush()
272 + proc = subprocess.Popen(
273 + [
274 + "vllm",
275 + "run-batch",
276 + "-i",
277 + input_file.name,
278 + "-o",
279 + output_file.name,
280 + "--model",
281 + "openai/whisper-large-v3",
282 + ],
283 + )
284 + proc.communicate()
285 + proc.wait()
286 + assert proc.returncode == 0, f"{proc=}"
287 +
288 + contents = output_file.read()
289 + print(f"\n\ncontents: {contents}\n\n")
290 + for line in contents.strip().split("\n"):
291 + BatchRequestOutput.model_validate_json(line)
292 +
293 + line_dict = json.loads(line)
294 + assert isinstance(line_dict, dict)
295 + assert line_dict["error"] is None
296 +
297 + response_body = line_dict["response"]["body"]
298 + assert response_body is not None
299 + assert "text" in response_body
300 + assert "usage" in response_body
301 +
302 +
303 +def test_transcription_http_url():
304 + with (
305 + tempfile.NamedTemporaryFile("w") as input_file,
306 + tempfile.NamedTemporaryFile("r") as output_file,
307 + ):
308 + input_file.write(INPUT_TRANSCRIPTION_HTTP_BATCH)
309 + input_file.flush()
310 + proc = subprocess.Popen(
311 + [
312 + "vllm",
313 + "run-batch",
314 + "-i",
315 + input_file.name,
316 + "-o",
317 + output_file.name,
318 + "--model",
319 + "openai/whisper-large-v3",
320 + ],
321 + )
322 + proc.communicate()
323 + proc.wait()
324 + assert proc.returncode == 0, f"{proc=}"
325 +
326 + contents = output_file.read()
327 + for line in contents.strip().split("\n"):
328 + BatchRequestOutput.model_validate_json(line)
329 +
330 + line_dict = json.loads(line)
331 + assert isinstance(line_dict, dict)
332 + assert line_dict["error"] is None
333 +
334 + response_body = line_dict["response"]["body"]
335 + assert response_body is not None
336 + assert "text" in response_body
337 + assert "usage" in response_body
338 +
339 + transcription_text = response_body["text"]
340 + assert "Mary had a little lamb" in transcription_text
341 +
342 +
343 +def test_translation():
344 + with (
345 + tempfile.NamedTemporaryFile("w") as input_file,
346 + tempfile.NamedTemporaryFile("r") as output_file,
347 + ):
348 + input_file.write(INPUT_TRANSLATION_BATCH)
349 + input_file.flush()
350 + proc = subprocess.Popen(
351 + [
352 + "vllm",
353 + "run-batch",
354 + "-i",
355 + input_file.name,
356 + "-o",
357 + output_file.name,
358 + "--model",
359 + "openai/whisper-small",
360 + ],
361 + )
362 + proc.communicate()
363 + proc.wait()
364 + assert proc.returncode == 0, f"{proc=}"
365 +
366 + contents = output_file.read()
367 + for line in contents.strip().split("\n"):
368 + BatchRequestOutput.model_validate_json(line)
369 +
370 + line_dict = json.loads(line)
371 + assert isinstance(line_dict, dict)
372 + assert line_dict["error"] is None
373 +
374 + response_body = line_dict["response"]["body"]
375 + assert response_body is not None
376 + assert "text" in response_body
377 +
378 + translation_text = response_body["text"]
379 + translation_text_lower = str(translation_text).strip().lower()
380 + assert "mary" in translation_text_lower or "lamb" in translation_text_lower
vllm/entrypoints/openai/run_batch.py
+407 -99
@@ -2,17 +2,20 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import asyncio
5 +import base64
6 import tempfile
7 from argparse import Namespace
8 from collections.abc import Awaitable, Callable
9 from http import HTTPStatus
9 -from io import StringIO
10 +from io import BytesIO, StringIO
11 from typing import Any, TypeAlias
12 +from urllib.parse import urlparse
13
14 import aiohttp
15 import torch
16 +from fastapi import UploadFile
17 from prometheus_client import start_http_server
15 -from pydantic import TypeAdapter, field_validator
18 +from pydantic import Field, TypeAdapter, field_validator, model_validator
19 from pydantic_core.core_schema import ValidationInfo
20 from tqdm import tqdm
21
@@ -25,12 +28,28 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
28 )
29 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
30 from vllm.entrypoints.openai.engine.protocol import (
31 + ErrorInfo,
32 ErrorResponse,
33 OpenAIBaseModel,
34 )
35 from vllm.entrypoints.openai.models.protocol import BaseModelPath
36 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
33 -from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest, EmbeddingResponse
37 +from vllm.entrypoints.openai.speech_to_text.protocol import (
38 + TranscriptionRequest,
39 + TranscriptionResponse,
40 + TranscriptionResponseVerbose,
41 + TranslationRequest,
42 + TranslationResponse,
43 + TranslationResponseVerbose,
44 +)
45 +from vllm.entrypoints.openai.speech_to_text.serving import (
46 + OpenAIServingTranscription,
47 + OpenAIServingTranslation,
48 +)
49 +from vllm.entrypoints.pooling.embed.protocol import (
50 + EmbeddingRequest,
51 + EmbeddingResponse,
52 +)
53 from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
54 from vllm.entrypoints.pooling.score.protocol import (
55 RerankRequest,
@@ -41,6 +60,7 @@ from vllm.entrypoints.pooling.score.protocol import (
60 from vllm.entrypoints.pooling.score.serving import ServingScores
61 from vllm.logger import init_logger
62 from vllm.reasoning import ReasoningParserManager
63 +from vllm.tasks import SupportedTask
64 from vllm.utils import random_uuid
65 from vllm.utils.argparse_utils import FlexibleArgumentParser
66 from vllm.version import __version__ as VLLM_VERSION
@@ -48,8 +68,73 @@ from vllm.version import __version__ as VLLM_VERSION
68 logger = init_logger(__name__)
69
70
71 +class BatchTranscriptionRequest(TranscriptionRequest):
72 + """
73 + Batch transcription request that uses file_url instead of file.
74 +
75 + This class extends TranscriptionRequest but replaces the file field
76 + with file_url to support batch processing from audio files written in JSON format.
77 + """
78 +
79 + file_url: str = Field(
80 + ...,
81 + description=(
82 + "Either a URL of the audio or a data URL with base64 encoded audio data. "
83 + ),
84 + )
85 +
86 + # Override file to be optional and unused for batch processing
87 + file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
88 +
89 + @model_validator(mode="before")
90 + @classmethod
91 + def validate_no_file(cls, data: Any):
92 + """Ensure file field is not provided in batch requests."""
93 + if isinstance(data, dict) and "file" in data:
94 + raise ValueError(
95 + "The 'file' field is not supported in batch requests. "
96 + "Use 'file_url' instead."
97 + )
98 + return data
99 +
100 +
101 +class BatchTranslationRequest(TranslationRequest):
102 + """
103 + Batch translation request that uses file_url instead of file.
104 +
105 + This class extends TranslationRequest but replaces the file field
106 + with file_url to support batch processing from audio files written in JSON format.
107 + """
108 +
109 + file_url: str = Field(
110 + ...,
111 + description=(
112 + "Either a URL of the audio or a data URL with base64 encoded audio data. "
113 + ),
114 + )
115 +
116 + # Override file to be optional and unused for batch processing
117 + file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
118 +
119 + @model_validator(mode="before")
120 + @classmethod
121 + def validate_no_file(cls, data: Any):
122 + """Ensure file field is not provided in batch requests."""
123 + if isinstance(data, dict) and "file" in data:
124 + raise ValueError(
125 + "The 'file' field is not supported in batch requests. "
126 + "Use 'file_url' instead."
127 + )
128 + return data
129 +
130 +
131 BatchRequestInputBody: TypeAlias = (
52 - ChatCompletionRequest | EmbeddingRequest | ScoreRequest | RerankRequest
132 + ChatCompletionRequest
133 + | EmbeddingRequest
134 + | ScoreRequest
135 + | RerankRequest
136 + | BatchTranscriptionRequest
137 + | BatchTranslationRequest
138 )
139
140
@@ -88,6 +173,10 @@ class BatchRequestInput(OpenAIBaseModel):
173 return TypeAdapter(ScoreRequest).validate_python(value)
174 if url.endswith("/rerank"):
175 return RerankRequest.model_validate(value)
176 + if url == "/v1/audio/transcriptions":
177 + return BatchTranscriptionRequest.model_validate(value)
178 + if url == "/v1/audio/translations":
179 + return BatchTranslationRequest.model_validate(value)
180 return TypeAdapter(BatchRequestInputBody).validate_python(value)
181
182
@@ -104,6 +193,10 @@ class BatchResponseData(OpenAIBaseModel):
193 | EmbeddingResponse
194 | ScoreResponse
195 | RerankResponse
196 + | TranscriptionResponse
197 + | TranscriptionResponseVerbose
198 + | TranslationResponse
199 + | TranslationResponseVerbose
200 | None
201 ) = None
202
@@ -361,6 +454,49 @@ async def write_file(
454 await write_local_file(path_or_url, batch_outputs)
455
456
457 +async def download_bytes_from_url(url: str) -> bytes:
458 + """
459 + Download data from a URL or decode from a data URL.
460 +
461 + Args:
462 + url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
463 +
464 + Returns:
465 + Data as bytes
466 + """
467 + parsed = urlparse(url)
468 +
469 + # Handle data URLs (base64 encoded)
470 + if parsed.scheme == "data":
471 + # Format: data:...;base64,<base64_data>
472 + if "," in url:
473 + header, data = url.split(",", 1)
474 + if "base64" in header:
475 + return base64.b64decode(data)
476 + else:
477 + raise ValueError(f"Unsupported data URL encoding: {header}")
478 + else:
479 + raise ValueError(f"Invalid data URL format: {url}")
480 +
481 + # Handle HTTP/HTTPS URLs
482 + elif parsed.scheme in ("http", "https"):
483 + async with (
484 + aiohttp.ClientSession() as session,
485 + session.get(url) as resp,
486 + ):
487 + if resp.status != 200:
488 + raise Exception(
489 + f"Failed to download data from URL: {url}. Status: {resp.status}"
490 + )
491 + return await resp.read()
492 +
493 + else:
494 + raise ValueError(
495 + f"Unsupported URL scheme: {parsed.scheme}. "
496 + "Supported schemes: http, https, data"
497 + )
498 +
499 +
500 def make_error_request_output(
501 request: BatchRequestInput, error_msg: str
502 ) -> BatchRequestOutput:
@@ -391,7 +527,16 @@ async def run_request(
527
528 if isinstance(
529 response,
394 - (ChatCompletionResponse, EmbeddingResponse, ScoreResponse, RerankResponse),
530 + (
531 + ChatCompletionResponse,
532 + EmbeddingResponse,
533 + ScoreResponse,
534 + RerankResponse,
535 + TranscriptionResponse,
536 + TranscriptionResponseVerbose,
537 + TranslationResponse,
538 + TranslationResponseVerbose,
539 + ),
540 ):
541 batch_output = BatchRequestOutput(
542 id=f"vllm-{random_uuid()}",
@@ -420,38 +565,130 @@ async def run_request(
565 return batch_output
566
567
423 -def validate_run_batch_args(args):
424 - valid_reasoning_parsers = ReasoningParserManager.list_registered()
425 - if (
426 - reasoning_parser := args.structured_outputs_config.reasoning_parser
427 - ) and reasoning_parser not in valid_reasoning_parsers:
428 - raise KeyError(
429 - f"invalid reasoning parser: {reasoning_parser} "
430 - f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
431 - )
568 +def handle_endpoint_request(
569 + request: BatchRequestInput,
570 + tracker: BatchProgressTracker,
571 + url_matcher: Callable[[str], bool],
572 + handler_getter: Callable[[], Callable | None],
573 + wrapper_fn: Callable[[Callable], Callable] | None = None,
574 +) -> Awaitable[BatchRequestOutput] | None:
575 + """
576 + Generic handler for endpoint requests.
577 +
578 + Args:
579 + request: The batch request input
580 + tracker: Progress tracker for the batch
581 + url_matcher: Function that takes a URL and returns True if it matches
582 + handler_getter: Function that returns the handler function or None
583 + wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
584 +
585 + Returns:
586 + Awaitable[BatchRequestOutput] if the request was handled,
587 + None if URL didn't match
588 + """
589 + if not url_matcher(request.url):
590 + return None
591
592 + handler_fn = handler_getter()
593 + if handler_fn is None:
594 + error_msg = f"Model does not support endpoint: {request.url}"
595 + return make_async_error_request_output(request, error_msg=error_msg)
596
434 -async def run_batch(
597 + # Apply wrapper if provided (e.g., for transcriptions/translations)
598 + if wrapper_fn is not None:
599 + handler_fn = wrapper_fn(handler_fn)
600 +
601 + tracker.submitted()
602 + return run_request(handler_fn, request, tracker)
603 +
604 +
605 +def make_transcription_wrapper(is_translation: bool):
606 + """
607 + Factory function to create a wrapper for transcription/translation handlers.
608 + The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
609 + to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
610 +
611 + Args:
612 + is_translation: If True, process as translation; otherwise process
613 + as transcription
614 +
615 + Returns:
616 + A function that takes a handler and returns a wrapped handler
617 + """
618 +
619 + def wrapper(handler_fn: Callable):
620 + async def transcription_wrapper(
621 + batch_request_body: (BatchTranscriptionRequest | BatchTranslationRequest),
622 + ) -> (
623 + TranscriptionResponse
624 + | TranscriptionResponseVerbose
625 + | TranslationResponse
626 + | TranslationResponseVerbose
627 + | ErrorResponse
628 + ):
629 + try:
630 + # Download data from URL
631 + audio_data = await download_bytes_from_url(batch_request_body.file_url)
632 +
633 + # Create a mock file from the downloaded audio data
634 + mock_file = UploadFile(
635 + file=BytesIO(audio_data),
636 + filename="audio.bin",
637 + )
638 +
639 + # Convert batch request to regular request
640 + # by copying all fields except file_url and setting file to mock_file
641 + request_dict = batch_request_body.model_dump(exclude={"file_url"})
642 + request_dict["file"] = mock_file
643 +
644 + if is_translation:
645 + # Create TranslationRequest from BatchTranslationRequest
646 + translation_request = TranslationRequest.model_validate(
647 + request_dict
648 + )
649 + return await handler_fn(audio_data, translation_request)
650 + else:
651 + # Create TranscriptionRequest from BatchTranscriptionRequest
652 + transcription_request = TranscriptionRequest.model_validate(
653 + request_dict
654 + )
655 + return await handler_fn(audio_data, transcription_request)
656 + except Exception as e:
657 + operation = "translation" if is_translation else "transcription"
658 + return ErrorResponse(
659 + error=ErrorInfo(
660 + message=f"Failed to process {operation}: {str(e)}",
661 + type="BadRequestError",
662 + code=HTTPStatus.BAD_REQUEST.value,
663 + )
664 + )
665 +
666 + return transcription_wrapper
667 +
668 + return wrapper
669 +
670 +
671 +def build_endpoint_registry(
672 engine_client: EngineClient,
673 args: Namespace,
437 -) -> None:
438 - if args.served_model_name is not None:
439 - served_model_names = args.served_model_name
440 - else:
441 - served_model_names = [args.model]
442 -
443 - if args.enable_log_requests:
444 - request_logger = RequestLogger(max_log_len=args.max_log_len)
445 - else:
446 - request_logger = None
674 + base_model_paths: list[BaseModelPath],
675 + request_logger: RequestLogger | None,
676 + supported_tasks: tuple[SupportedTask, ...],
677 +) -> dict[str, dict[str, Any]]:
678 + """
679 + Build the endpoint registry with all serving objects and handler configurations.
680
448 - base_model_paths = [
449 - BaseModelPath(name=name, model_path=args.model) for name in served_model_names
450 - ]
681 + Args:
682 + engine_client: The engine client
683 + args: Command line arguments
684 + base_model_paths: List of base model paths
685 + request_logger: Optional request logger
686 + supported_tasks: Tuple of supported tasks
687
688 + Returns:
689 + Dictionary mapping endpoint keys to their configurations
690 + """
691 model_config = engine_client.model_config
453 - supported_tasks = await engine_client.get_supported_tasks()
454 - logger.info("Supported tasks: %s", supported_tasks)
692
693 # Create the openai serving objects.
694 openai_serving_models = OpenAIServingModels(
@@ -507,97 +744,168 @@ async def run_batch(
744 else None
745 )
746
510 - tracker = BatchProgressTracker()
511 - logger.info("Reading batch from %s...", args.input_file)
512 -
513 - # Submit all requests in the file to the engine "concurrently".
514 - response_futures: list[Awaitable[BatchRequestOutput]] = []
515 - for request_json in (await read_file(args.input_file)).strip().split("\n"):
516 - # Skip empty lines.
517 - request_json = request_json.strip()
518 - if not request_json:
519 - continue
747 + openai_serving_transcription = (
748 + OpenAIServingTranscription(
749 + engine_client,
750 + openai_serving_models,
751 + request_logger=request_logger,
752 + enable_force_include_usage=args.enable_force_include_usage,
753 + )
754 + if "transcription" in supported_tasks
755 + else None
756 + )
757
521 - request = BatchRequestInput.model_validate_json(request_json)
758 + openai_serving_translation = (
759 + OpenAIServingTranslation(
760 + engine_client,
761 + openai_serving_models,
762 + request_logger=request_logger,
763 + enable_force_include_usage=args.enable_force_include_usage,
764 + )
765 + if "transcription" in supported_tasks
766 + else None
767 + )
768
523 - # Determine the type of request and run it.
524 - if request.url == "/v1/chat/completions":
525 - chat_handler_fn = (
769 + # Registry of endpoint configurations
770 + endpoint_registry: dict[str, dict[str, Any]] = {
771 + "completions": {
772 + "url_matcher": lambda url: url == "/v1/chat/completions",
773 + "handler_getter": lambda: (
774 openai_serving_chat.create_chat_completion
775 if openai_serving_chat is not None
776 else None
529 - )
530 - if chat_handler_fn is None:
531 - response_futures.append(
532 - make_async_error_request_output(
533 - request,
534 - error_msg="The model does not support Chat Completions API",
535 - )
536 - )
537 - continue
538 -
539 - response_futures.append(run_request(chat_handler_fn, request, tracker))
540 - tracker.submitted()
541 - elif request.url == "/v1/embeddings":
542 - embed_handler_fn = (
777 + ),
778 + "wrapper_fn": None,
779 + },
780 + "embeddings": {
781 + "url_matcher": lambda url: url == "/v1/embeddings",
782 + "handler_getter": lambda: (
783 openai_serving_embedding.create_embedding
784 if openai_serving_embedding is not None
785 else None
546 - )
547 - if embed_handler_fn is None:
548 - response_futures.append(
549 - make_async_error_request_output(
550 - request,
551 - error_msg="The model does not support Embeddings API",
552 - )
553 - )
554 - continue
555 -
556 - response_futures.append(run_request(embed_handler_fn, request, tracker))
557 - tracker.submitted()
558 - elif request.url.endswith("/score"):
559 - score_handler_fn = (
786 + ),
787 + "wrapper_fn": None,
788 + },
789 + "score": {
790 + "url_matcher": lambda url: url.endswith("/score"),
791 + "handler_getter": lambda: (
792 openai_serving_scores.create_score
793 if openai_serving_scores is not None
794 else None
563 - )
564 - if score_handler_fn is None:
565 - response_futures.append(
566 - make_async_error_request_output(
567 - request,
568 - error_msg="The model does not support Scores API",
569 - )
570 - )
571 - continue
572 -
573 - response_futures.append(run_request(score_handler_fn, request, tracker))
574 - tracker.submitted()
575 - elif request.url.endswith("/rerank"):
576 - rerank_handler_fn = (
795 + ),
796 + "wrapper_fn": None,
797 + },
798 + "rerank": {
799 + "url_matcher": lambda url: url.endswith("/rerank"),
800 + "handler_getter": lambda: (
801 openai_serving_scores.do_rerank
802 if openai_serving_scores is not None
803 else None
804 + ),
805 + "wrapper_fn": None,
806 + },
807 + "transcriptions": {
808 + "url_matcher": lambda url: url == "/v1/audio/transcriptions",
809 + "handler_getter": lambda: (
810 + openai_serving_transcription.create_transcription
811 + if openai_serving_transcription is not None
812 + else None
813 + ),
814 + "wrapper_fn": make_transcription_wrapper(is_translation=False),
815 + },
816 + "translations": {
817 + "url_matcher": lambda url: url == "/v1/audio/translations",
818 + "handler_getter": lambda: (
819 + openai_serving_translation.create_translation
820 + if openai_serving_translation is not None
821 + else None
822 + ),
823 + "wrapper_fn": make_transcription_wrapper(is_translation=True),
824 + },
825 + }
826 +
827 + return endpoint_registry
828 +
829 +
830 +def validate_run_batch_args(args):
831 + valid_reasoning_parsers = ReasoningParserManager.list_registered()
832 + if (
833 + reasoning_parser := args.structured_outputs_config.reasoning_parser
834 + ) and reasoning_parser not in valid_reasoning_parsers:
835 + raise KeyError(
836 + f"invalid reasoning parser: {reasoning_parser} "
837 + f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
838 + )
839 +
840 +
841 +async def run_batch(
842 + engine_client: EngineClient,
843 + args: Namespace,
844 +) -> None:
845 + if args.served_model_name is not None:
846 + served_model_names = args.served_model_name
847 + else:
848 + served_model_names = [args.model]
849 +
850 + if args.enable_log_requests:
851 + request_logger = RequestLogger(max_log_len=args.max_log_len)
852 + else:
853 + request_logger = None
854 +
855 + base_model_paths = [
856 + BaseModelPath(name=name, model_path=args.model) for name in served_model_names
857 + ]
858 +
859 + supported_tasks = await engine_client.get_supported_tasks()
860 + logger.info("Supported tasks: %s", supported_tasks)
861 +
862 + endpoint_registry = build_endpoint_registry(
863 + engine_client=engine_client,
864 + args=args,
865 + base_model_paths=base_model_paths,
866 + request_logger=request_logger,
867 + supported_tasks=supported_tasks,
868 + )
869 +
870 + tracker = BatchProgressTracker()
871 + logger.info("Reading batch from %s...", args.input_file)
872 +
873 + # Submit all requests in the file to the engine "concurrently".
874 + response_futures: list[Awaitable[BatchRequestOutput]] = []
875 + for request_json in (await read_file(args.input_file)).strip().split("\n"):
876 + # Skip empty lines.
877 + request_json = request_json.strip()
878 + if not request_json:
879 + continue
880 +
881 + request = BatchRequestInput.model_validate_json(request_json)
882 +
883 + # Use the last segment of the URL as the endpoint key.
884 + # More advanced URL matching is done in url_matcher of endpoint_registry.
885 + endpoint_key = request.url.split("/")[-1]
886 +
887 + result = None
888 + if endpoint_key in endpoint_registry:
889 + endpoint_config = endpoint_registry[endpoint_key]
890 + result = handle_endpoint_request(
891 + request,
892 + tracker,
893 + url_matcher=endpoint_config["url_matcher"],
894 + handler_getter=endpoint_config["handler_getter"],
895 + wrapper_fn=endpoint_config["wrapper_fn"],
896 )
581 - if rerank_handler_fn is None:
582 - response_futures.append(
583 - make_async_error_request_output(
584 - request,
585 - error_msg="The model does not support Rerank API",
586 - )
587 - )
588 - continue
897
590 - response_futures.append(run_request(rerank_handler_fn, request, tracker))
591 - tracker.submitted()
898 + if result is not None:
899 + response_futures.append(result)
900 else:
901 response_futures.append(
902 make_async_error_request_output(
903 request,
904 error_msg=f"URL {request.url} was used. "
905 "Supported endpoints: /v1/chat/completions, /v1/embeddings,"
598 - " /score, /rerank ."
599 - "See vllm/entrypoints/openai/api_server.py for supported "
600 - "score/rerank versions.",
906 + " /v1/audio/transcriptions, /v1/audio/translations, /score, "
907 + " /rerank. See vllm/entrypoints/openai/api_server.py "
908 + "for supported score/rerank versions.",
909 )
910 )
911
vllm/entrypoints/openai/speech_to_text/serving.py
+8 -2
@@ -54,7 +54,10 @@ class OpenAIServingTranscription(OpenAISpeechToText):
54 )
55
56 async def create_transcription(
57 - self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request
57 + self,
58 + audio_data: bytes,
59 + request: TranscriptionRequest,
60 + raw_request: Request | None = None,
61 ) -> (
62 TranscriptionResponse
63 | TranscriptionResponseVerbose
@@ -124,7 +127,10 @@ class OpenAIServingTranslation(OpenAISpeechToText):
127 )
128
129 async def create_translation(
127 - self, audio_data: bytes, request: TranslationRequest, raw_request: Request
130 + self,
131 + audio_data: bytes,
132 + request: TranslationRequest,
133 + raw_request: Request | None = None,
134 ) -> (
135 TranslationResponse
136 | TranslationResponseVerbose