[Frontend] Move run_batch.py out openai folder (#53500)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Aug 24, 2026 at 23:04 UTC
9cef631f307dd2a96937bf97b591e64dc0ce85a4
7 files changed
+937
-900
docs/mkdocs/gen_files/generate_argparse.py
+2
-2
@@ -180,7 +180,7 @@ launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION")
180
RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand")
181
sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS")
182
openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args")
183
-openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch")
183
+run_batch = auto_mock("vllm.entrypoints.launchers", "run_batch")
184
185
if TYPE_CHECKING:
186
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -329,7 +329,7 @@ pages = {
329
"cli/serve.md": (create_parser(openai_cli_args.make_arg_parser), True),
330
"cli/chat.md": (create_parser(ChatCommand.add_cli_args), False),
331
"cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False),
332
- "cli/run-batch.md": (create_parser(openai_run_batch.make_arg_parser), True),
332
+ "cli/run-batch.md": (create_parser(run_batch.make_arg_parser), True),
333
"cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True),
334
"cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True),
335
# URL kept as `mm_processor` for back-compat; command name is `mm-processor`
examples/features/openai_batch/README.md
+3
-3
@@ -48,7 +48,7 @@ The batch running tool is designed to be used from the command line.
48
You can run the batch with the following command, which will write its results to a file called `results.jsonl`
49
50
```bash
51
-python -m vllm.entrypoints.openai.run_batch \
51
+python -m vllm.entrypoints.launchers.run_batch \
52
-i features/openai_batch/openai_example_batch.jsonl \
53
-o results.jsonl \
54
--model meta-llama/Meta-Llama-3-8B-Instruct
@@ -80,7 +80,7 @@ The batch runner supports remote input and output urls that are accessible via h
80
For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl`, you can run
81
82
```bash
83
-python -m vllm.entrypoints.openai.run_batch \
83
+python -m vllm.entrypoints.launchers.run_batch \
84
-i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl \
85
-o results.jsonl \
86
--model meta-llama/Meta-Llama-3-8B-Instruct
@@ -190,7 +190,7 @@ output_url='https://s3.us-west-2.amazonaws.com/MY_BUCKET/MY_OUTPUT_FILE.jsonl?AW
190
You can now run the batch runner, using the urls generated in the previous section.
191
192
```bash
193
-python -m vllm.entrypoints.openai.run_batch \
193
+python -m vllm.entrypoints.launchers.run_batch \
194
-i "https://s3.us-west-2.amazonaws.com/MY_BUCKET/MY_INPUT_FILE.jsonl?AWSAccessKeyId=ABCDEFGHIJKLMNOPQRST&Signature=abcdefghijklmnopqrstuvwxyz12345&Expires=1715800091" \
195
-o "https://s3.us-west-2.amazonaws.com/MY_BUCKET/MY_OUTPUT_FILE.jsonl?AWSAccessKeyId=ABCDEFGHIJKLMNOPQRST&Signature=abcdefghijklmnopqrstuvwxyz12345&Expires=1715800091" \
196
--model meta-llama/Meta-Llama-3-8B-Instruct
tests/entrypoints/launchers/test_run_batch.py
renamed
+10
-10
@@ -14,14 +14,14 @@ import pytest
14
import vllm.envs as envs
15
from vllm.assets.audio import AudioAsset
16
from vllm.connections import HTTPConnection
17
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
18
-from vllm.entrypoints.openai.run_batch import (
17
+from vllm.entrypoints.launchers.run_batch import (
18
BatchRequestOutput,
19
BatchTranscriptionRequest,
20
download_bytes_from_url,
21
make_transcription_wrapper,
22
upload_data,
23
)
24
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
25
from vllm.exceptions import VLLMValidationError
26
from vllm.utils.mem_constants import MiB_bytes
27
@@ -920,7 +920,7 @@ async def test_download_bytes_allows_permitted_domain():
920
mock_session = _make_aiohttp_mocks(expected)
921
922
with patch(
923
- "vllm.entrypoints.openai.run_batch.aiohttp.ClientSession",
923
+ "vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
924
return_value=mock_session,
925
):
926
result = await download_bytes_from_url(
@@ -937,7 +937,7 @@ async def test_download_bytes_no_allowlist_permits_any_domain():
937
mock_session = _make_aiohttp_mocks(expected)
938
939
with patch(
940
- "vllm.entrypoints.openai.run_batch.aiohttp.ClientSession",
940
+ "vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
941
return_value=mock_session,
942
):
943
result = await download_bytes_from_url(url, allowed_media_domains=None)
@@ -994,7 +994,7 @@ async def test_transcription_wrapper_rejects_oversized_data_url_before_decode(
994
}
995
)
996
997
- with patch("vllm.entrypoints.openai.run_batch.base64.b64decode") as decode:
997
+ with patch("vllm.entrypoints.launchers.run_batch.base64.b64decode") as decode:
998
response = await wrapped_handler(request)
999
1000
assert isinstance(response, ErrorResponse)
@@ -1011,7 +1011,7 @@ async def test_download_bytes_allows_http_body_at_audio_limit(
1011
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
1012
connection = HTTPConnection()
1013
monkeypatch.setattr(
1014
- "vllm.entrypoints.openai.run_batch.global_http_connection",
1014
+ "vllm.entrypoints.launchers.run_batch.global_http_connection",
1015
connection,
1016
)
1017
@@ -1032,7 +1032,7 @@ async def test_transcription_wrapper_rejects_oversized_http_before_handler(
1032
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
1033
connection = HTTPConnection()
1034
monkeypatch.setattr(
1035
- "vllm.entrypoints.openai.run_batch.global_http_connection",
1035
+ "vllm.entrypoints.launchers.run_batch.global_http_connection",
1036
connection,
1037
)
1038
handler = AsyncMock()
@@ -1080,7 +1080,7 @@ async def test_upload_data_uploads_once_on_success():
1080
"""A successful upload must not be retried (regression guard)."""
1081
session = _make_aiohttp_put_session(status=200)
1082
with patch(
1083
- "vllm.entrypoints.openai.run_batch.aiohttp.ClientSession",
1083
+ "vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
1084
return_value=session,
1085
):
1086
await upload_data(
@@ -1095,11 +1095,11 @@ async def test_upload_data_error_includes_awaited_response_body():
1095
session = _make_aiohttp_put_session(status=500, body_text="server-error-detail")
1096
with (
1097
patch(
1098
- "vllm.entrypoints.openai.run_batch.aiohttp.ClientSession",
1098
+ "vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
1099
return_value=session,
1100
),
1101
patch(
1102
- "vllm.entrypoints.openai.run_batch.asyncio.sleep",
1102
+ "vllm.entrypoints.launchers.run_batch.asyncio.sleep",
1103
AsyncMock(),
1104
),
1105
pytest.raises(Exception) as exc_info,
tests/entrypoints/serve/instrumentator/test_metrics.py
+1
-1
@@ -450,7 +450,7 @@ def test_metrics_exist_run_batch():
450
[
451
sys.executable,
452
"-m",
453
- "vllm.entrypoints.openai.run_batch",
453
+ "vllm.entrypoints.launchers.run_batch",
454
"-i",
455
input_file.name,
456
"-o",
vllm/entrypoints/cli/run_batch.py
+2
-2
@@ -25,7 +25,7 @@ class RunBatchSubcommand(CLISubcommand):
25
26
@staticmethod
27
def cmd(args: argparse.Namespace) -> None:
28
- from vllm.entrypoints.openai.run_batch import main as run_batch_main
28
+ from vllm.entrypoints.launchers.run_batch import main as run_batch_main
29
30
logger.info(
31
"vLLM batch processing API version %s", importlib.metadata.version("vllm")
@@ -48,7 +48,7 @@ class RunBatchSubcommand(CLISubcommand):
48
def subparser_init(
49
self, subparsers: argparse._SubParsersAction
50
) -> FlexibleArgumentParser:
51
- from vllm.entrypoints.openai.run_batch import make_arg_parser
51
+ from vllm.entrypoints.launchers.run_batch import make_arg_parser
52
53
run_batch_parser = subparsers.add_parser(
54
self.name,
vllm/entrypoints/launchers/run_batch.py
new
+910
@@ -0,0 +1,910 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import asyncio
5
+import contextlib
6
+import json
7
+import sys
8
+import tempfile
9
+from argparse import Namespace
10
+from collections.abc import Awaitable, Callable
11
+from http import HTTPStatus
12
+from io import BytesIO, StringIO
13
+from typing import Any, NoReturn, TypeAlias
14
+from urllib.parse import urlparse
15
+
16
+import aiohttp
17
+import pybase64 as base64
18
+import pydantic
19
+import torch
20
+from fastapi import UploadFile
21
+from prometheus_client import start_http_server
22
+from pydantic import Field, TypeAdapter, field_validator, model_validator
23
+from pydantic_core.core_schema import ValidationInfo
24
+from starlette.datastructures import State
25
+from starlette.responses import JSONResponse
26
+from tqdm import tqdm
27
+from urllib3.util import parse_url
28
+
29
+import vllm.envs as envs
30
+from vllm.config import config
31
+from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
32
+from vllm.engine.arg_utils import AsyncEngineArgs
33
+from vllm.engine.protocol import EngineClient
34
+from vllm.entrypoints.launchers.api_server.entry import init_app_state
35
+from vllm.entrypoints.openai.chat_completion.protocol import (
36
+ ChatCompletionRequest,
37
+ ChatCompletionResponse,
38
+)
39
+from vllm.entrypoints.openai.cli_args import BaseFrontendArgs
40
+from vllm.entrypoints.openai.engine.protocol import (
41
+ ErrorInfo,
42
+ ErrorResponse,
43
+ OpenAIBaseModel,
44
+)
45
+from vllm.entrypoints.pooling.embed.protocol import (
46
+ EmbeddingRequest,
47
+ EmbeddingResponse,
48
+)
49
+from vllm.entrypoints.pooling.scoring.protocol import (
50
+ RerankRequest,
51
+ RerankResponse,
52
+ ScoreRequest,
53
+ ScoreResponse,
54
+)
55
+from vllm.entrypoints.serve import create_error_response
56
+from vllm.entrypoints.speech_to_text.transcription.protocol import (
57
+ TranscriptionRequest,
58
+ TranscriptionResponse,
59
+ TranscriptionResponseVerbose,
60
+)
61
+from vllm.entrypoints.speech_to_text.translation.protocol import (
62
+ TranslationRequest,
63
+ TranslationResponse,
64
+ TranslationResponseVerbose,
65
+)
66
+from vllm.exceptions import VLLMValidationError
67
+from vllm.logger import init_logger
68
+from vllm.reasoning import ReasoningParserManager
69
+from vllm.utils import random_uuid
70
+from vllm.utils.argparse_utils import FlexibleArgumentParser
71
+from vllm.utils.mem_constants import MiB_bytes
72
+from vllm.version import __version__ as VLLM_VERSION
73
+
74
+logger = init_logger(__name__)
75
+
76
+
77
+def _get_audio_download_limit_bytes() -> int:
78
+ return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
79
+
80
+
81
+def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
82
+ raise VLLMValidationError(
83
+ "Maximum file size exceeded",
84
+ parameter="audio_filesize_mb",
85
+ value=size_bytes / MiB_bytes,
86
+ )
87
+
88
+
89
+class BatchTranscriptionRequest(TranscriptionRequest):
90
+ """
91
+ Batch transcription request that uses file_url instead of file.
92
+
93
+ This class extends TranscriptionRequest but replaces the file field
94
+ with file_url to support batch processing from audio files written in JSON format.
95
+ """
96
+
97
+ file_url: str = Field(
98
+ ...,
99
+ description=(
100
+ "Either a URL of the audio or a data URL with base64 encoded audio data. "
101
+ ),
102
+ )
103
+
104
+ # Override file to be optional and unused for batch processing
105
+ file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
106
+
107
+ @model_validator(mode="before")
108
+ @classmethod
109
+ def validate_no_file(cls, data: Any):
110
+ """Ensure file field is not provided in batch requests."""
111
+ if isinstance(data, dict) and "file" in data:
112
+ raise VLLMValidationError(
113
+ "The 'file' field is not supported in batch requests. "
114
+ "Use 'file_url' instead.",
115
+ parameter="file",
116
+ )
117
+ return data
118
+
119
+
120
+class BatchTranslationRequest(TranslationRequest):
121
+ """
122
+ Batch translation request that uses file_url instead of file.
123
+
124
+ This class extends TranslationRequest but replaces the file field
125
+ with file_url to support batch processing from audio files written in JSON format.
126
+ """
127
+
128
+ file_url: str = Field(
129
+ ...,
130
+ description=(
131
+ "Either a URL of the audio or a data URL with base64 encoded audio data. "
132
+ ),
133
+ )
134
+
135
+ # Override file to be optional and unused for batch processing
136
+ file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
137
+
138
+ @model_validator(mode="before")
139
+ @classmethod
140
+ def validate_no_file(cls, data: Any):
141
+ """Ensure file field is not provided in batch requests."""
142
+ if isinstance(data, dict) and "file" in data:
143
+ raise VLLMValidationError(
144
+ "The 'file' field is not supported in batch requests. "
145
+ "Use 'file_url' instead.",
146
+ parameter="file",
147
+ )
148
+ return data
149
+
150
+
151
+BatchRequestInputBody: TypeAlias = (
152
+ ChatCompletionRequest
153
+ | EmbeddingRequest
154
+ | ScoreRequest
155
+ | RerankRequest
156
+ | BatchTranscriptionRequest
157
+ | BatchTranslationRequest
158
+)
159
+
160
+
161
+class BatchRequestInput(OpenAIBaseModel):
162
+ """
163
+ The per-line object of the batch input file.
164
+
165
+ NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
166
+ """
167
+
168
+ # A developer-provided per-request id that will be used to match outputs to
169
+ # inputs. Must be unique for each request in a batch.
170
+ custom_id: str
171
+
172
+ # The HTTP method to be used for the request. Currently only POST is
173
+ # supported.
174
+ method: str
175
+
176
+ # The OpenAI API relative URL to be used for the request. Currently
177
+ # /v1/chat/completions is supported.
178
+ url: str
179
+
180
+ # The parameters of the request.
181
+ body: BatchRequestInputBody
182
+
183
+ @field_validator("body", mode="plain")
184
+ @classmethod
185
+ def check_type_for_url(cls, value: Any, info: ValidationInfo):
186
+ # Use url to disambiguate models
187
+ url: str = info.data["url"]
188
+ if url == "/v1/chat/completions":
189
+ return ChatCompletionRequest.model_validate(value)
190
+ if url == "/v1/embeddings":
191
+ return TypeAdapter(EmbeddingRequest).validate_python(value)
192
+ if url.endswith("/score"):
193
+ return TypeAdapter(ScoreRequest).validate_python(value)
194
+ if url.endswith("/rerank"):
195
+ return RerankRequest.model_validate(value)
196
+ if url == "/v1/audio/transcriptions":
197
+ return BatchTranscriptionRequest.model_validate(value)
198
+ if url == "/v1/audio/translations":
199
+ return BatchTranslationRequest.model_validate(value)
200
+ return TypeAdapter(BatchRequestInputBody).validate_python(value)
201
+
202
+
203
+AllResponse: TypeAlias = (
204
+ ChatCompletionResponse
205
+ | EmbeddingResponse
206
+ | ScoreResponse
207
+ | RerankResponse
208
+ | TranscriptionResponse
209
+ | TranscriptionResponseVerbose
210
+ | TranslationResponse
211
+ | TranslationResponseVerbose
212
+)
213
+
214
+
215
+class BatchResponseData(OpenAIBaseModel):
216
+ # HTTP status code of the response.
217
+ status_code: int = 200
218
+
219
+ # An unique identifier for the API request.
220
+ request_id: str
221
+
222
+ # The body of the response.
223
+ body: AllResponse | None = None
224
+
225
+
226
+class BatchRequestOutput(OpenAIBaseModel):
227
+ """
228
+ The per-line object of the batch output and error files
229
+ """
230
+
231
+ id: str
232
+
233
+ # A developer-provided per-request id that will be used to match outputs to
234
+ # inputs.
235
+ custom_id: str
236
+
237
+ response: BatchResponseData | None
238
+
239
+ # For requests that failed with a non-HTTP error, this will contain more
240
+ # information on the cause of the failure.
241
+ error: Any | None
242
+
243
+
244
+@config
245
+class BatchFrontendArgs(BaseFrontendArgs):
246
+ """Arguments for the batch runner frontend."""
247
+
248
+ input_file: str | None = None
249
+ """The path or url to a single input file. Currently supports local file
250
+ paths, or the http protocol (http or https). If a URL is specified,
251
+ the file should be available via HTTP GET."""
252
+ output_file: str | None = None
253
+ """The path or url to a single output file. Currently supports
254
+ local file paths, or web (http or https) urls. If a URL is specified,
255
+ the file should be available via HTTP PUT."""
256
+ output_tmp_dir: str | None = None
257
+ """The directory to store the output file before uploading it
258
+ to the output URL."""
259
+ enable_metrics: bool = False
260
+ """Enable Prometheus metrics"""
261
+ host: str | None = None
262
+ """Host name for the Prometheus metrics server
263
+ (only needed if enable-metrics is set)."""
264
+ port: int = 8000
265
+ """Port number for the Prometheus metrics server
266
+ (only needed if enable-metrics is set)."""
267
+ url: str = "0.0.0.0"
268
+ """[DEPRECATED] Host name for the Prometheus metrics server
269
+ (only needed if enable-metrics is set). Use --host instead."""
270
+
271
+ @classmethod
272
+ def _customize_cli_kwargs(
273
+ cls,
274
+ frontend_kwargs: dict[str, Any],
275
+ ) -> dict[str, Any]:
276
+ frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
277
+
278
+ frontend_kwargs["input_file"]["flags"] = ["-i"]
279
+ frontend_kwargs["input_file"]["required"] = True
280
+ frontend_kwargs["output_file"]["flags"] = ["-o"]
281
+ frontend_kwargs["output_file"]["required"] = True
282
+
283
+ frontend_kwargs["enable_metrics"]["action"] = "store_true"
284
+
285
+ frontend_kwargs["url"]["deprecated"] = True
286
+ return frontend_kwargs
287
+
288
+
289
+def make_arg_parser(parser: FlexibleArgumentParser):
290
+ parser = BatchFrontendArgs.add_cli_args(parser)
291
+ parser = AsyncEngineArgs.add_cli_args(parser)
292
+ return parser
293
+
294
+
295
+def parse_args():
296
+ parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
297
+ args = make_arg_parser(parser).parse_args()
298
+
299
+ # Backward compatibility: If --url is set, use it for host
300
+ url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
301
+ host_explicit = any(
302
+ arg == "--host" or arg.startswith("--host=") for arg in sys.argv
303
+ )
304
+ if url_explicit and hasattr(args, "url") and not host_explicit:
305
+ args.host = args.url
306
+ logger.warning_once(
307
+ "Using --url for metrics is deprecated. Please use --host instead."
308
+ )
309
+
310
+ return args
311
+
312
+
313
+# explicitly use pure text format, with a newline at the end
314
+# this makes it impossible to see the animation in the progress bar
315
+# but will avoid messing up with ray or multiprocessing, which wraps
316
+# each line of output with some prefix.
317
+_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
318
+
319
+
320
+class BatchProgressTracker:
321
+ def __init__(self):
322
+ self._total = 0
323
+ self._pbar: tqdm | None = None
324
+
325
+ def submitted(self):
326
+ self._total += 1
327
+
328
+ def completed(self):
329
+ if self._pbar:
330
+ self._pbar.update()
331
+
332
+ def pbar(self) -> tqdm:
333
+ enable_tqdm = (
334
+ not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
335
+ )
336
+ self._pbar = tqdm(
337
+ total=self._total,
338
+ unit="req",
339
+ desc="Running batch",
340
+ mininterval=5,
341
+ disable=not enable_tqdm,
342
+ bar_format=_BAR_FORMAT,
343
+ )
344
+ return self._pbar
345
+
346
+
347
+async def read_file(path_or_url: str) -> str:
348
+ if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
349
+ async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
350
+ resp.raise_for_status()
351
+ return await resp.text()
352
+ else:
353
+ with open(path_or_url, encoding="utf-8") as f:
354
+ return f.read()
355
+
356
+
357
+async def write_local_file(
358
+ output_path: str, batch_outputs: list[BatchRequestOutput]
359
+) -> None:
360
+ """
361
+ Write the responses to a local file.
362
+ output_path: The path to write the responses to.
363
+ batch_outputs: The list of batch outputs to write.
364
+ """
365
+ # We should make this async, but as long as run_batch runs as a
366
+ # standalone program, blocking the event loop won't affect performance.
367
+ with open(output_path, "w", encoding="utf-8") as f:
368
+ for o in batch_outputs:
369
+ print(o.model_dump_json(), file=f)
370
+
371
+
372
+async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
373
+ """
374
+ Upload a local file to a URL.
375
+ output_url: The URL to upload the file to.
376
+ data_or_file: Either the data to upload or the path to the file to upload.
377
+ from_file: If True, data_or_file is the path to the file to upload.
378
+ """
379
+ # Timeout is a common issue when uploading large files.
380
+ # We retry max_retries times before giving up.
381
+ max_retries = 5
382
+ # Number of seconds to wait before retrying.
383
+ delay = 5
384
+
385
+ for attempt in range(1, max_retries + 1):
386
+ try:
387
+ # We increase the timeout to 1000 seconds to allow
388
+ # for large files (default is 300).
389
+ async with aiohttp.ClientSession(
390
+ timeout=aiohttp.ClientTimeout(total=1000)
391
+ ) as session:
392
+ if from_file:
393
+ with open(data_or_file, "rb") as file:
394
+ async with session.put(output_url, data=file) as response:
395
+ if response.status != 200:
396
+ error_text = await response.text()
397
+ raise Exception(
398
+ f"Failed to upload file.\n"
399
+ f"Status: {response.status}\n"
400
+ f"Response: {error_text}"
401
+ )
402
+ else:
403
+ async with session.put(output_url, data=data_or_file) as response:
404
+ if response.status != 200:
405
+ error_text = await response.text()
406
+ raise Exception(
407
+ f"Failed to upload data.\n"
408
+ f"Status: {response.status}\n"
409
+ f"Response: {error_text}"
410
+ )
411
+ # Upload succeeded; do not retry.
412
+ return
413
+ except Exception as e:
414
+ if attempt < max_retries:
415
+ logger.error(
416
+ "Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
417
+ attempt,
418
+ e,
419
+ delay,
420
+ )
421
+ await asyncio.sleep(delay)
422
+ else:
423
+ raise Exception(
424
+ f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
425
+ ) from e
426
+
427
+
428
+async def write_file(
429
+ path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
430
+) -> None:
431
+ """
432
+ Write batch_outputs to a file or upload to a URL.
433
+ path_or_url: The path or URL to write batch_outputs to.
434
+ batch_outputs: The list of batch outputs to write.
435
+ output_tmp_dir: The directory to store the output file before uploading it
436
+ to the output URL.
437
+ """
438
+ if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
439
+ if output_tmp_dir is None:
440
+ logger.info("Writing outputs to memory buffer")
441
+ output_buffer = StringIO()
442
+ for o in batch_outputs:
443
+ print(o.model_dump_json(), file=output_buffer)
444
+ output_buffer.seek(0)
445
+ logger.info("Uploading outputs to %s", path_or_url)
446
+ await upload_data(
447
+ path_or_url,
448
+ output_buffer.read().strip().encode("utf-8"),
449
+ from_file=False,
450
+ )
451
+ else:
452
+ # Write responses to a temporary file and then upload it to the URL.
453
+ with tempfile.NamedTemporaryFile(
454
+ mode="w",
455
+ encoding="utf-8",
456
+ dir=output_tmp_dir,
457
+ prefix="tmp_batch_output_",
458
+ suffix=".jsonl",
459
+ ) as f:
460
+ logger.info("Writing outputs to temporary local file %s", f.name)
461
+ await write_local_file(f.name, batch_outputs)
462
+ logger.info("Uploading outputs to %s", path_or_url)
463
+ await upload_data(path_or_url, f.name, from_file=True)
464
+ else:
465
+ logger.info("Writing outputs to local file %s", path_or_url)
466
+ await write_local_file(path_or_url, batch_outputs)
467
+
468
+
469
+async def download_bytes_from_url(
470
+ url: str,
471
+ allowed_media_domains: list[str] | None = None,
472
+) -> bytes:
473
+ """
474
+ Download data from a URL or decode from a data URL.
475
+
476
+ Args:
477
+ url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
478
+ allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
479
+ is in this list are permitted. data: URLs are not subject to
480
+ this restriction.
481
+
482
+ Returns:
483
+ Data as bytes
484
+ """
485
+ parsed = urlparse(url)
486
+
487
+ # Handle data URLs (base64 encoded) - not subject to domain restrictions
488
+ if parsed.scheme == "data":
489
+ # Format: data:...;base64,<base64_data>
490
+ if "," in url:
491
+ header, data = url.split(",", 1)
492
+ if "base64" in header:
493
+ decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
494
+ if decoded_size_bytes > _get_audio_download_limit_bytes():
495
+ _raise_audio_filesize_limit(decoded_size_bytes)
496
+ return base64.b64decode(data)
497
+ else:
498
+ raise VLLMValidationError(
499
+ f"Unsupported data URL encoding: {header}",
500
+ parameter="url",
501
+ )
502
+ else:
503
+ raise VLLMValidationError(
504
+ f"Invalid data URL format: {url}",
505
+ parameter="url",
506
+ )
507
+
508
+ # Handle HTTP/HTTPS URLs
509
+ elif parsed.scheme in ("http", "https"):
510
+ if allowed_media_domains is not None:
511
+ url_spec = parse_url(url)
512
+ if url_spec.hostname not in allowed_media_domains:
513
+ raise VLLMValidationError(
514
+ f"The URL must be from one of the allowed domains: "
515
+ f"{allowed_media_domains}. Input URL domain: "
516
+ f"{url_spec.hostname}",
517
+ parameter="url",
518
+ value=url_spec.hostname,
519
+ )
520
+ # Use the normalized URL to prevent parsing discrepancies
521
+ # between urllib3 and aiohttp (e.g. backslash-@ attacks).
522
+ url = url_spec.url
523
+
524
+ try:
525
+ return await global_http_connection.async_get_bytes(
526
+ url,
527
+ allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
528
+ max_bytes=_get_audio_download_limit_bytes(),
529
+ )
530
+ except HTTPResponseSizeExceededError as exc:
531
+ _raise_audio_filesize_limit(exc.received_bytes)
532
+
533
+ else:
534
+ raise VLLMValidationError(
535
+ f"Unsupported URL scheme: {parsed.scheme}. "
536
+ "Supported schemes: http, https, data",
537
+ parameter="url",
538
+ value=parsed.scheme,
539
+ )
540
+
541
+
542
+def make_error_request_output(
543
+ request: BatchRequestInput, error_msg: str
544
+) -> BatchRequestOutput:
545
+ batch_output = BatchRequestOutput(
546
+ id=f"vllm-{random_uuid()}",
547
+ custom_id=request.custom_id,
548
+ response=BatchResponseData(
549
+ status_code=HTTPStatus.BAD_REQUEST,
550
+ request_id=f"vllm-batch-{random_uuid()}",
551
+ ),
552
+ error=error_msg,
553
+ )
554
+ return batch_output
555
+
556
+
557
+async def make_async_error_request_output(
558
+ request: BatchRequestInput, error_msg: str
559
+) -> BatchRequestOutput:
560
+ return make_error_request_output(request, error_msg)
561
+
562
+
563
+async def run_request(
564
+ serving_engine_func: Callable,
565
+ request: BatchRequestInput,
566
+ tracker: BatchProgressTracker,
567
+) -> BatchRequestOutput:
568
+ try:
569
+ response = await serving_engine_func(request.body)
570
+ except Exception as e:
571
+ response = create_error_response(e)
572
+
573
+ if isinstance(response, JSONResponse):
574
+ with contextlib.suppress(pydantic.ValidationError):
575
+ response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
576
+ json.loads(response.body)
577
+ )
578
+
579
+ if isinstance(response, AllResponse):
580
+ batch_output = BatchRequestOutput(
581
+ id=f"vllm-{random_uuid()}",
582
+ custom_id=request.custom_id,
583
+ response=BatchResponseData(
584
+ body=response, request_id=f"vllm-batch-{random_uuid()}"
585
+ ),
586
+ error=None,
587
+ )
588
+ elif isinstance(response, ErrorResponse):
589
+ batch_output = BatchRequestOutput(
590
+ id=f"vllm-{random_uuid()}",
591
+ custom_id=request.custom_id,
592
+ response=BatchResponseData(
593
+ status_code=response.error.code,
594
+ request_id=f"vllm-batch-{random_uuid()}",
595
+ ),
596
+ error=response,
597
+ )
598
+ else:
599
+ batch_output = make_error_request_output(
600
+ request, error_msg="Request must not be sent in stream mode"
601
+ )
602
+
603
+ tracker.completed()
604
+ return batch_output
605
+
606
+
607
+WrapperFn: TypeAlias = Callable[[Callable], Callable]
608
+
609
+
610
+def handle_endpoint_request(
611
+ request: BatchRequestInput,
612
+ tracker: BatchProgressTracker,
613
+ url_matcher: Callable[[str], bool],
614
+ handler_getter: Callable[[], Callable | None],
615
+ wrapper_fn: WrapperFn | None = None,
616
+) -> Awaitable[BatchRequestOutput] | None:
617
+ """
618
+ Generic handler for endpoint requests.
619
+
620
+ Args:
621
+ request: The batch request input
622
+ tracker: Progress tracker for the batch
623
+ url_matcher: Function that takes a URL and returns True if it matches
624
+ handler_getter: Function that returns the handler function or None
625
+ wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
626
+
627
+ Returns:
628
+ Awaitable[BatchRequestOutput] if the request was handled,
629
+ None if URL didn't match
630
+ """
631
+ if not url_matcher(request.url):
632
+ return None
633
+
634
+ handler_fn = handler_getter()
635
+ if handler_fn is None:
636
+ error_msg = f"Model does not support endpoint: {request.url}"
637
+ return make_async_error_request_output(request, error_msg=error_msg)
638
+
639
+ # Apply wrapper if provided (e.g., for transcriptions/translations)
640
+ if wrapper_fn is not None:
641
+ handler_fn = wrapper_fn(handler_fn)
642
+
643
+ tracker.submitted()
644
+ return run_request(handler_fn, request, tracker)
645
+
646
+
647
+def make_transcription_wrapper(
648
+ is_translation: bool,
649
+ allowed_media_domains: list[str] | None = None,
650
+) -> WrapperFn:
651
+ """
652
+ Factory function to create a wrapper for transcription/translation handlers.
653
+ The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
654
+ to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
655
+
656
+ Args:
657
+ is_translation: If True, process as translation; otherwise process
658
+ as transcription
659
+ allowed_media_domains: If set, only URLs from these domains are
660
+ permitted for HTTP/HTTPS fetches.
661
+
662
+ Returns:
663
+ A function that takes a handler and returns a wrapped handler
664
+ """
665
+
666
+ def wrapper(handler_fn: Callable):
667
+ async def transcription_wrapper(
668
+ batch_request_body: (BatchTranscriptionRequest | BatchTranslationRequest),
669
+ ) -> (
670
+ TranscriptionResponse
671
+ | TranscriptionResponseVerbose
672
+ | TranslationResponse
673
+ | TranslationResponseVerbose
674
+ | ErrorResponse
675
+ ):
676
+ try:
677
+ # Download data from URL
678
+ audio_data = await download_bytes_from_url(
679
+ batch_request_body.file_url,
680
+ allowed_media_domains=allowed_media_domains,
681
+ )
682
+
683
+ # Create a mock file from the downloaded audio data
684
+ mock_file = UploadFile(
685
+ file=BytesIO(audio_data),
686
+ filename="audio.bin",
687
+ )
688
+
689
+ # Convert batch request to regular request
690
+ # by copying all fields except file_url and setting file to mock_file
691
+ request_dict = batch_request_body.model_dump(exclude={"file_url"})
692
+ request_dict["file"] = mock_file
693
+
694
+ if is_translation:
695
+ # Create TranslationRequest from BatchTranslationRequest
696
+ translation_request = TranslationRequest.model_validate(
697
+ request_dict
698
+ )
699
+ return await handler_fn(audio_data, translation_request)
700
+ else:
701
+ # Create TranscriptionRequest from BatchTranscriptionRequest
702
+ transcription_request = TranscriptionRequest.model_validate(
703
+ request_dict
704
+ )
705
+ return await handler_fn(audio_data, transcription_request)
706
+ except Exception as e:
707
+ operation = "translation" if is_translation else "transcription"
708
+ return ErrorResponse(
709
+ error=ErrorInfo(
710
+ message=f"Failed to process {operation}: {str(e)}",
711
+ type="BadRequestError",
712
+ code=HTTPStatus.BAD_REQUEST.value,
713
+ )
714
+ )
715
+
716
+ return transcription_wrapper
717
+
718
+ return wrapper
719
+
720
+
721
+async def build_endpoint_registry(
722
+ engine_client: EngineClient,
723
+ args: Namespace,
724
+) -> dict[str, dict[str, Any]]:
725
+ """
726
+ Build the endpoint registry with all serving objects and handler configurations.
727
+
728
+ Args:
729
+ engine_client: The engine client
730
+ args: Command line arguments
731
+
732
+ Returns:
733
+ Dictionary mapping endpoint keys to their configurations
734
+ """
735
+ supported_tasks = await engine_client.get_supported_tasks()
736
+ logger.info("Supported tasks: %s", supported_tasks)
737
+
738
+ # Create a state object to hold serving objects
739
+ state = State()
740
+
741
+ # Initialize all serving objects using init_app_state
742
+ # This provides full functionality including chat template processing,
743
+ # LoRA support, tool servers, etc.
744
+ await init_app_state(engine_client, state, args, supported_tasks)
745
+
746
+ # Get serving objects from state (defaulting to None if not set)
747
+ openai_serving_chat = getattr(state, "openai_serving_chat", None)
748
+ openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
749
+ openai_serving_translation = getattr(state, "openai_serving_translation", None)
750
+ serving_embedding = getattr(state, "serving_embedding", None)
751
+ serving_scores = getattr(state, "serving_scores", None)
752
+
753
+ allowed_media_domains = getattr(args, "allowed_media_domains", None)
754
+
755
+ # Registry of endpoint configurations
756
+ endpoint_registry: dict[str, dict[str, Any]] = {
757
+ "completions": {
758
+ "url_matcher": lambda url: url == "/v1/chat/completions",
759
+ "handler_getter": lambda: (
760
+ openai_serving_chat.create_chat_completion
761
+ if openai_serving_chat is not None
762
+ else None
763
+ ),
764
+ "wrapper_fn": None,
765
+ },
766
+ "embeddings": {
767
+ "url_matcher": lambda url: url == "/v1/embeddings",
768
+ "handler_getter": lambda: (
769
+ serving_embedding if serving_embedding is not None else None
770
+ ),
771
+ "wrapper_fn": None,
772
+ },
773
+ "score": {
774
+ "url_matcher": lambda url: url.endswith("/score"),
775
+ "handler_getter": lambda: (
776
+ serving_scores if serving_scores is not None else None
777
+ ),
778
+ "wrapper_fn": None,
779
+ },
780
+ "rerank": {
781
+ "url_matcher": lambda url: url.endswith("/rerank"),
782
+ "handler_getter": lambda: (
783
+ serving_scores if serving_scores is not None else None
784
+ ),
785
+ "wrapper_fn": None,
786
+ },
787
+ "transcriptions": {
788
+ "url_matcher": lambda url: url == "/v1/audio/transcriptions",
789
+ "handler_getter": lambda: (
790
+ openai_serving_transcription.create_transcription
791
+ if openai_serving_transcription is not None
792
+ else None
793
+ ),
794
+ "wrapper_fn": make_transcription_wrapper(
795
+ is_translation=False,
796
+ allowed_media_domains=allowed_media_domains,
797
+ ),
798
+ },
799
+ "translations": {
800
+ "url_matcher": lambda url: url == "/v1/audio/translations",
801
+ "handler_getter": lambda: (
802
+ openai_serving_translation.create_translation
803
+ if openai_serving_translation is not None
804
+ else None
805
+ ),
806
+ "wrapper_fn": make_transcription_wrapper(
807
+ is_translation=True,
808
+ allowed_media_domains=allowed_media_domains,
809
+ ),
810
+ },
811
+ }
812
+
813
+ return endpoint_registry
814
+
815
+
816
+def validate_run_batch_args(args):
817
+ valid_reasoning_parsers = ReasoningParserManager.list_registered()
818
+ if (
819
+ reasoning_parser := args.structured_outputs_config.reasoning_parser
820
+ ) and reasoning_parser not in valid_reasoning_parsers:
821
+ raise KeyError(
822
+ f"invalid reasoning parser: {reasoning_parser} "
823
+ f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
824
+ )
825
+
826
+
827
+async def run_batch(
828
+ engine_client: EngineClient,
829
+ args: Namespace,
830
+) -> None:
831
+ endpoint_registry = await build_endpoint_registry(
832
+ engine_client=engine_client,
833
+ args=args,
834
+ )
835
+
836
+ tracker = BatchProgressTracker()
837
+ logger.info("Reading batch from %s...", args.input_file)
838
+
839
+ # Submit all requests in the file to the engine "concurrently".
840
+ response_futures: list[Awaitable[BatchRequestOutput]] = []
841
+ for request_json in (await read_file(args.input_file)).strip().split("\n"):
842
+ # Skip empty lines.
843
+ request_json = request_json.strip()
844
+ if not request_json:
845
+ continue
846
+
847
+ request = BatchRequestInput.model_validate_json(request_json)
848
+
849
+ # Use the last segment of the URL as the endpoint key.
850
+ # More advanced URL matching is done in url_matcher of endpoint_registry.
851
+ endpoint_key = request.url.split("/")[-1]
852
+
853
+ result = None
854
+ if endpoint_key in endpoint_registry:
855
+ endpoint_config = endpoint_registry[endpoint_key]
856
+ result = handle_endpoint_request(
857
+ request,
858
+ tracker,
859
+ url_matcher=endpoint_config["url_matcher"],
860
+ handler_getter=endpoint_config["handler_getter"],
861
+ wrapper_fn=endpoint_config["wrapper_fn"],
862
+ )
863
+
864
+ if result is not None:
865
+ response_futures.append(result)
866
+ else:
867
+ response_futures.append(
868
+ make_async_error_request_output(
869
+ request,
870
+ error_msg=f"URL {request.url} was used. "
871
+ "Supported endpoints: /v1/chat/completions, /v1/embeddings,"
872
+ " /v1/audio/transcriptions, /v1/audio/translations, /score, "
873
+ " /rerank.",
874
+ )
875
+ )
876
+
877
+ with tracker.pbar():
878
+ responses = await asyncio.gather(*response_futures)
879
+
880
+ await write_file(args.output_file, responses, args.output_tmp_dir)
881
+
882
+
883
+async def main(args: Namespace):
884
+ from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
885
+ from vllm.usage.usage_lib import UsageContext
886
+
887
+ validate_run_batch_args(args)
888
+
889
+ async with build_async_engine_client(
890
+ args,
891
+ usage_context=UsageContext.OPENAI_BATCH_RUNNER,
892
+ ) as engine_client:
893
+ await run_batch(engine_client, args)
894
+
895
+
896
+if __name__ == "__main__":
897
+ args = parse_args()
898
+
899
+ logger.info("vLLM batch processing API version %s", VLLM_VERSION)
900
+ logger.info("args: %s", args)
901
+
902
+ # Start the Prometheus metrics server. LLMEngine uses the Prometheus client
903
+ # to publish metrics at the /metrics endpoint.
904
+ if args.enable_metrics:
905
+ logger.info("Prometheus metrics enabled")
906
+ start_http_server(port=args.port, addr=args.host)
907
+ else:
908
+ logger.info("Prometheus metrics disabled")
909
+
910
+ asyncio.run(main(args))
vllm/entrypoints/openai/run_batch.py
+9
-882
@@ -1,899 +1,26 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
import asyncio
5
-import contextlib
6
-import json
7
-import sys
8
-import tempfile
9
-from argparse import Namespace
10
-from collections.abc import Awaitable, Callable
11
-from http import HTTPStatus
12
-from io import BytesIO, StringIO
13
-from typing import Any, NoReturn, TypeAlias
14
-from urllib.parse import urlparse
4
+import warnings
5
16
-import aiohttp
17
-import pybase64 as base64
18
-import pydantic
19
-import torch
20
-from fastapi import UploadFile
6
from prometheus_client import start_http_server
22
-from pydantic import Field, TypeAdapter, field_validator, model_validator
23
-from pydantic_core.core_schema import ValidationInfo
24
-from starlette.datastructures import State
25
-from starlette.responses import JSONResponse
26
-from tqdm import tqdm
27
-from urllib3.util import parse_url
7
29
-import vllm.envs as envs
30
-from vllm.config import config
31
-from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
32
-from vllm.engine.arg_utils import AsyncEngineArgs
33
-from vllm.engine.protocol import EngineClient
34
-from vllm.entrypoints.launchers.api_server.entry import init_app_state
35
-from vllm.entrypoints.openai.chat_completion.protocol import (
36
- ChatCompletionRequest,
37
- ChatCompletionResponse,
38
-)
39
-from vllm.entrypoints.openai.cli_args import BaseFrontendArgs
40
-from vllm.entrypoints.openai.engine.protocol import (
41
- ErrorInfo,
42
- ErrorResponse,
43
- OpenAIBaseModel,
44
-)
45
-from vllm.entrypoints.pooling.embed.protocol import (
46
- EmbeddingRequest,
47
- EmbeddingResponse,
48
-)
49
-from vllm.entrypoints.pooling.scoring.protocol import (
50
- RerankRequest,
51
- RerankResponse,
52
- ScoreRequest,
53
- ScoreResponse,
54
-)
55
-from vllm.entrypoints.serve import create_error_response
56
-from vllm.entrypoints.speech_to_text.transcription.protocol import (
57
- TranscriptionRequest,
58
- TranscriptionResponse,
59
- TranscriptionResponseVerbose,
60
-)
61
-from vllm.entrypoints.speech_to_text.translation.protocol import (
62
- TranslationRequest,
63
- TranslationResponse,
64
- TranslationResponseVerbose,
65
-)
66
-from vllm.exceptions import VLLMValidationError
8
from vllm.logger import init_logger
68
-from vllm.reasoning import ReasoningParserManager
69
-from vllm.utils import random_uuid
70
-from vllm.utils.argparse_utils import FlexibleArgumentParser
71
-from vllm.utils.mem_constants import MiB_bytes
9
from vllm.version import __version__ as VLLM_VERSION
10
74
-logger = init_logger(__name__)
75
-
76
-
77
-def _get_audio_download_limit_bytes() -> int:
78
- return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
79
-
80
-
81
-def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
82
- raise VLLMValidationError(
83
- "Maximum file size exceeded",
84
- parameter="audio_filesize_mb",
85
- value=size_bytes / MiB_bytes,
86
- )
87
-
88
-
89
-class BatchTranscriptionRequest(TranscriptionRequest):
90
- """
91
- Batch transcription request that uses file_url instead of file.
92
-
93
- This class extends TranscriptionRequest but replaces the file field
94
- with file_url to support batch processing from audio files written in JSON format.
95
- """
96
-
97
- file_url: str = Field(
98
- ...,
99
- description=(
100
- "Either a URL of the audio or a data URL with base64 encoded audio data. "
101
- ),
102
- )
103
-
104
- # Override file to be optional and unused for batch processing
105
- file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
106
-
107
- @model_validator(mode="before")
108
- @classmethod
109
- def validate_no_file(cls, data: Any):
110
- """Ensure file field is not provided in batch requests."""
111
- if isinstance(data, dict) and "file" in data:
112
- raise VLLMValidationError(
113
- "The 'file' field is not supported in batch requests. "
114
- "Use 'file_url' instead.",
115
- parameter="file",
116
- )
117
- return data
118
-
119
-
120
-class BatchTranslationRequest(TranslationRequest):
121
- """
122
- Batch translation request that uses file_url instead of file.
123
-
124
- This class extends TranslationRequest but replaces the file field
125
- with file_url to support batch processing from audio files written in JSON format.
126
- """
127
-
128
- file_url: str = Field(
129
- ...,
130
- description=(
131
- "Either a URL of the audio or a data URL with base64 encoded audio data. "
132
- ),
133
- )
134
-
135
- # Override file to be optional and unused for batch processing
136
- file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
137
-
138
- @model_validator(mode="before")
139
- @classmethod
140
- def validate_no_file(cls, data: Any):
141
- """Ensure file field is not provided in batch requests."""
142
- if isinstance(data, dict) and "file" in data:
143
- raise VLLMValidationError(
144
- "The 'file' field is not supported in batch requests. "
145
- "Use 'file_url' instead.",
146
- parameter="file",
147
- )
148
- return data
149
-
150
-
151
-BatchRequestInputBody: TypeAlias = (
152
- ChatCompletionRequest
153
- | EmbeddingRequest
154
- | ScoreRequest
155
- | RerankRequest
156
- | BatchTranscriptionRequest
157
- | BatchTranslationRequest
158
-)
159
-
160
-
161
-class BatchRequestInput(OpenAIBaseModel):
162
- """
163
- The per-line object of the batch input file.
164
-
165
- NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
166
- """
167
-
168
- # A developer-provided per-request id that will be used to match outputs to
169
- # inputs. Must be unique for each request in a batch.
170
- custom_id: str
171
-
172
- # The HTTP method to be used for the request. Currently only POST is
173
- # supported.
174
- method: str
175
-
176
- # The OpenAI API relative URL to be used for the request. Currently
177
- # /v1/chat/completions is supported.
178
- url: str
179
-
180
- # The parameters of the request.
181
- body: BatchRequestInputBody
182
-
183
- @field_validator("body", mode="plain")
184
- @classmethod
185
- def check_type_for_url(cls, value: Any, info: ValidationInfo):
186
- # Use url to disambiguate models
187
- url: str = info.data["url"]
188
- if url == "/v1/chat/completions":
189
- return ChatCompletionRequest.model_validate(value)
190
- if url == "/v1/embeddings":
191
- return TypeAdapter(EmbeddingRequest).validate_python(value)
192
- if url.endswith("/score"):
193
- return TypeAdapter(ScoreRequest).validate_python(value)
194
- if url.endswith("/rerank"):
195
- return RerankRequest.model_validate(value)
196
- if url == "/v1/audio/transcriptions":
197
- return BatchTranscriptionRequest.model_validate(value)
198
- if url == "/v1/audio/translations":
199
- return BatchTranslationRequest.model_validate(value)
200
- return TypeAdapter(BatchRequestInputBody).validate_python(value)
201
-
202
-
203
-AllResponse: TypeAlias = (
204
- ChatCompletionResponse
205
- | EmbeddingResponse
206
- | ScoreResponse
207
- | RerankResponse
208
- | TranscriptionResponse
209
- | TranscriptionResponseVerbose
210
- | TranslationResponse
211
- | TranslationResponseVerbose
11
+warnings.warn(
12
+ "The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
13
+ "and may be removed in a future release. Please use `vllm run_batch` instead.",
14
+ DeprecationWarning,
15
+ stacklevel=1,
16
)
17
18
215
-class BatchResponseData(OpenAIBaseModel):
216
- # HTTP status code of the response.
217
- status_code: int = 200
218
-
219
- # An unique identifier for the API request.
220
- request_id: str
221
-
222
- # The body of the response.
223
- body: AllResponse | None = None
224
-
225
-
226
-class BatchRequestOutput(OpenAIBaseModel):
227
- """
228
- The per-line object of the batch output and error files
229
- """
230
-
231
- id: str
232
-
233
- # A developer-provided per-request id that will be used to match outputs to
234
- # inputs.
235
- custom_id: str
236
-
237
- response: BatchResponseData | None
238
-
239
- # For requests that failed with a non-HTTP error, this will contain more
240
- # information on the cause of the failure.
241
- error: Any | None
242
-
243
-
244
-@config
245
-class BatchFrontendArgs(BaseFrontendArgs):
246
- """Arguments for the batch runner frontend."""
247
-
248
- input_file: str | None = None
249
- """The path or url to a single input file. Currently supports local file
250
- paths, or the http protocol (http or https). If a URL is specified,
251
- the file should be available via HTTP GET."""
252
- output_file: str | None = None
253
- """The path or url to a single output file. Currently supports
254
- local file paths, or web (http or https) urls. If a URL is specified,
255
- the file should be available via HTTP PUT."""
256
- output_tmp_dir: str | None = None
257
- """The directory to store the output file before uploading it
258
- to the output URL."""
259
- enable_metrics: bool = False
260
- """Enable Prometheus metrics"""
261
- host: str | None = None
262
- """Host name for the Prometheus metrics server
263
- (only needed if enable-metrics is set)."""
264
- port: int = 8000
265
- """Port number for the Prometheus metrics server
266
- (only needed if enable-metrics is set)."""
267
- url: str = "0.0.0.0"
268
- """[DEPRECATED] Host name for the Prometheus metrics server
269
- (only needed if enable-metrics is set). Use --host instead."""
270
-
271
- @classmethod
272
- def _customize_cli_kwargs(
273
- cls,
274
- frontend_kwargs: dict[str, Any],
275
- ) -> dict[str, Any]:
276
- frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
277
-
278
- frontend_kwargs["input_file"]["flags"] = ["-i"]
279
- frontend_kwargs["input_file"]["required"] = True
280
- frontend_kwargs["output_file"]["flags"] = ["-o"]
281
- frontend_kwargs["output_file"]["required"] = True
282
-
283
- frontend_kwargs["enable_metrics"]["action"] = "store_true"
284
-
285
- frontend_kwargs["url"]["deprecated"] = True
286
- return frontend_kwargs
287
-
288
-
289
-def make_arg_parser(parser: FlexibleArgumentParser):
290
- parser = BatchFrontendArgs.add_cli_args(parser)
291
- parser = AsyncEngineArgs.add_cli_args(parser)
292
- return parser
293
-
294
-
295
-def parse_args():
296
- parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
297
- args = make_arg_parser(parser).parse_args()
298
-
299
- # Backward compatibility: If --url is set, use it for host
300
- url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
301
- host_explicit = any(
302
- arg == "--host" or arg.startswith("--host=") for arg in sys.argv
303
- )
304
- if url_explicit and hasattr(args, "url") and not host_explicit:
305
- args.host = args.url
306
- logger.warning_once(
307
- "Using --url for metrics is deprecated. Please use --host instead."
308
- )
309
-
310
- return args
311
-
312
-
313
-# explicitly use pure text format, with a newline at the end
314
-# this makes it impossible to see the animation in the progress bar
315
-# but will avoid messing up with ray or multiprocessing, which wraps
316
-# each line of output with some prefix.
317
-_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
318
-
319
-
320
-class BatchProgressTracker:
321
- def __init__(self):
322
- self._total = 0
323
- self._pbar: tqdm | None = None
324
-
325
- def submitted(self):
326
- self._total += 1
327
-
328
- def completed(self):
329
- if self._pbar:
330
- self._pbar.update()
331
-
332
- def pbar(self) -> tqdm:
333
- enable_tqdm = (
334
- not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
335
- )
336
- self._pbar = tqdm(
337
- total=self._total,
338
- unit="req",
339
- desc="Running batch",
340
- mininterval=5,
341
- disable=not enable_tqdm,
342
- bar_format=_BAR_FORMAT,
343
- )
344
- return self._pbar
345
-
346
-
347
-async def read_file(path_or_url: str) -> str:
348
- if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
349
- async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
350
- resp.raise_for_status()
351
- return await resp.text()
352
- else:
353
- with open(path_or_url, encoding="utf-8") as f:
354
- return f.read()
355
-
356
-
357
-async def write_local_file(
358
- output_path: str, batch_outputs: list[BatchRequestOutput]
359
-) -> None:
360
- """
361
- Write the responses to a local file.
362
- output_path: The path to write the responses to.
363
- batch_outputs: The list of batch outputs to write.
364
- """
365
- # We should make this async, but as long as run_batch runs as a
366
- # standalone program, blocking the event loop won't affect performance.
367
- with open(output_path, "w", encoding="utf-8") as f:
368
- for o in batch_outputs:
369
- print(o.model_dump_json(), file=f)
370
-
371
-
372
-async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
373
- """
374
- Upload a local file to a URL.
375
- output_url: The URL to upload the file to.
376
- data_or_file: Either the data to upload or the path to the file to upload.
377
- from_file: If True, data_or_file is the path to the file to upload.
378
- """
379
- # Timeout is a common issue when uploading large files.
380
- # We retry max_retries times before giving up.
381
- max_retries = 5
382
- # Number of seconds to wait before retrying.
383
- delay = 5
384
-
385
- for attempt in range(1, max_retries + 1):
386
- try:
387
- # We increase the timeout to 1000 seconds to allow
388
- # for large files (default is 300).
389
- async with aiohttp.ClientSession(
390
- timeout=aiohttp.ClientTimeout(total=1000)
391
- ) as session:
392
- if from_file:
393
- with open(data_or_file, "rb") as file:
394
- async with session.put(output_url, data=file) as response:
395
- if response.status != 200:
396
- error_text = await response.text()
397
- raise Exception(
398
- f"Failed to upload file.\n"
399
- f"Status: {response.status}\n"
400
- f"Response: {error_text}"
401
- )
402
- else:
403
- async with session.put(output_url, data=data_or_file) as response:
404
- if response.status != 200:
405
- error_text = await response.text()
406
- raise Exception(
407
- f"Failed to upload data.\n"
408
- f"Status: {response.status}\n"
409
- f"Response: {error_text}"
410
- )
411
- # Upload succeeded; do not retry.
412
- return
413
- except Exception as e:
414
- if attempt < max_retries:
415
- logger.error(
416
- "Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
417
- attempt,
418
- e,
419
- delay,
420
- )
421
- await asyncio.sleep(delay)
422
- else:
423
- raise Exception(
424
- f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
425
- ) from e
426
-
427
-
428
-async def write_file(
429
- path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
430
-) -> None:
431
- """
432
- Write batch_outputs to a file or upload to a URL.
433
- path_or_url: The path or URL to write batch_outputs to.
434
- batch_outputs: The list of batch outputs to write.
435
- output_tmp_dir: The directory to store the output file before uploading it
436
- to the output URL.
437
- """
438
- if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
439
- if output_tmp_dir is None:
440
- logger.info("Writing outputs to memory buffer")
441
- output_buffer = StringIO()
442
- for o in batch_outputs:
443
- print(o.model_dump_json(), file=output_buffer)
444
- output_buffer.seek(0)
445
- logger.info("Uploading outputs to %s", path_or_url)
446
- await upload_data(
447
- path_or_url,
448
- output_buffer.read().strip().encode("utf-8"),
449
- from_file=False,
450
- )
451
- else:
452
- # Write responses to a temporary file and then upload it to the URL.
453
- with tempfile.NamedTemporaryFile(
454
- mode="w",
455
- encoding="utf-8",
456
- dir=output_tmp_dir,
457
- prefix="tmp_batch_output_",
458
- suffix=".jsonl",
459
- ) as f:
460
- logger.info("Writing outputs to temporary local file %s", f.name)
461
- await write_local_file(f.name, batch_outputs)
462
- logger.info("Uploading outputs to %s", path_or_url)
463
- await upload_data(path_or_url, f.name, from_file=True)
464
- else:
465
- logger.info("Writing outputs to local file %s", path_or_url)
466
- await write_local_file(path_or_url, batch_outputs)
467
-
468
-
469
-async def download_bytes_from_url(
470
- url: str,
471
- allowed_media_domains: list[str] | None = None,
472
-) -> bytes:
473
- """
474
- Download data from a URL or decode from a data URL.
475
-
476
- Args:
477
- url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
478
- allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
479
- is in this list are permitted. data: URLs are not subject to
480
- this restriction.
481
-
482
- Returns:
483
- Data as bytes
484
- """
485
- parsed = urlparse(url)
486
-
487
- # Handle data URLs (base64 encoded) - not subject to domain restrictions
488
- if parsed.scheme == "data":
489
- # Format: data:...;base64,<base64_data>
490
- if "," in url:
491
- header, data = url.split(",", 1)
492
- if "base64" in header:
493
- decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
494
- if decoded_size_bytes > _get_audio_download_limit_bytes():
495
- _raise_audio_filesize_limit(decoded_size_bytes)
496
- return base64.b64decode(data)
497
- else:
498
- raise VLLMValidationError(
499
- f"Unsupported data URL encoding: {header}",
500
- parameter="url",
501
- )
502
- else:
503
- raise VLLMValidationError(
504
- f"Invalid data URL format: {url}",
505
- parameter="url",
506
- )
507
-
508
- # Handle HTTP/HTTPS URLs
509
- elif parsed.scheme in ("http", "https"):
510
- if allowed_media_domains is not None:
511
- url_spec = parse_url(url)
512
- if url_spec.hostname not in allowed_media_domains:
513
- raise VLLMValidationError(
514
- f"The URL must be from one of the allowed domains: "
515
- f"{allowed_media_domains}. Input URL domain: "
516
- f"{url_spec.hostname}",
517
- parameter="url",
518
- value=url_spec.hostname,
519
- )
520
- # Use the normalized URL to prevent parsing discrepancies
521
- # between urllib3 and aiohttp (e.g. backslash-@ attacks).
522
- url = url_spec.url
523
-
524
- try:
525
- return await global_http_connection.async_get_bytes(
526
- url,
527
- allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
528
- max_bytes=_get_audio_download_limit_bytes(),
529
- )
530
- except HTTPResponseSizeExceededError as exc:
531
- _raise_audio_filesize_limit(exc.received_bytes)
532
-
533
- else:
534
- raise VLLMValidationError(
535
- f"Unsupported URL scheme: {parsed.scheme}. "
536
- "Supported schemes: http, https, data",
537
- parameter="url",
538
- value=parsed.scheme,
539
- )
540
-
541
-
542
-def make_error_request_output(
543
- request: BatchRequestInput, error_msg: str
544
-) -> BatchRequestOutput:
545
- batch_output = BatchRequestOutput(
546
- id=f"vllm-{random_uuid()}",
547
- custom_id=request.custom_id,
548
- response=BatchResponseData(
549
- status_code=HTTPStatus.BAD_REQUEST,
550
- request_id=f"vllm-batch-{random_uuid()}",
551
- ),
552
- error=error_msg,
553
- )
554
- return batch_output
555
-
556
-
557
-async def make_async_error_request_output(
558
- request: BatchRequestInput, error_msg: str
559
-) -> BatchRequestOutput:
560
- return make_error_request_output(request, error_msg)
561
-
562
-
563
-async def run_request(
564
- serving_engine_func: Callable,
565
- request: BatchRequestInput,
566
- tracker: BatchProgressTracker,
567
-) -> BatchRequestOutput:
568
- try:
569
- response = await serving_engine_func(request.body)
570
- except Exception as e:
571
- response = create_error_response(e)
572
-
573
- if isinstance(response, JSONResponse):
574
- with contextlib.suppress(pydantic.ValidationError):
575
- response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
576
- json.loads(response.body)
577
- )
578
-
579
- if isinstance(response, AllResponse):
580
- batch_output = BatchRequestOutput(
581
- id=f"vllm-{random_uuid()}",
582
- custom_id=request.custom_id,
583
- response=BatchResponseData(
584
- body=response, request_id=f"vllm-batch-{random_uuid()}"
585
- ),
586
- error=None,
587
- )
588
- elif isinstance(response, ErrorResponse):
589
- batch_output = BatchRequestOutput(
590
- id=f"vllm-{random_uuid()}",
591
- custom_id=request.custom_id,
592
- response=BatchResponseData(
593
- status_code=response.error.code,
594
- request_id=f"vllm-batch-{random_uuid()}",
595
- ),
596
- error=response,
597
- )
598
- else:
599
- batch_output = make_error_request_output(
600
- request, error_msg="Request must not be sent in stream mode"
601
- )
602
-
603
- tracker.completed()
604
- return batch_output
605
-
606
-
607
-WrapperFn: TypeAlias = Callable[[Callable], Callable]
608
-
609
-
610
-def handle_endpoint_request(
611
- request: BatchRequestInput,
612
- tracker: BatchProgressTracker,
613
- url_matcher: Callable[[str], bool],
614
- handler_getter: Callable[[], Callable | None],
615
- wrapper_fn: WrapperFn | None = None,
616
-) -> Awaitable[BatchRequestOutput] | None:
617
- """
618
- Generic handler for endpoint requests.
619
-
620
- Args:
621
- request: The batch request input
622
- tracker: Progress tracker for the batch
623
- url_matcher: Function that takes a URL and returns True if it matches
624
- handler_getter: Function that returns the handler function or None
625
- wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
626
-
627
- Returns:
628
- Awaitable[BatchRequestOutput] if the request was handled,
629
- None if URL didn't match
630
- """
631
- if not url_matcher(request.url):
632
- return None
633
-
634
- handler_fn = handler_getter()
635
- if handler_fn is None:
636
- error_msg = f"Model does not support endpoint: {request.url}"
637
- return make_async_error_request_output(request, error_msg=error_msg)
638
-
639
- # Apply wrapper if provided (e.g., for transcriptions/translations)
640
- if wrapper_fn is not None:
641
- handler_fn = wrapper_fn(handler_fn)
642
-
643
- tracker.submitted()
644
- return run_request(handler_fn, request, tracker)
645
-
646
-
647
-def make_transcription_wrapper(
648
- is_translation: bool,
649
- allowed_media_domains: list[str] | None = None,
650
-) -> WrapperFn:
651
- """
652
- Factory function to create a wrapper for transcription/translation handlers.
653
- The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
654
- to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
655
-
656
- Args:
657
- is_translation: If True, process as translation; otherwise process
658
- as transcription
659
- allowed_media_domains: If set, only URLs from these domains are
660
- permitted for HTTP/HTTPS fetches.
661
-
662
- Returns:
663
- A function that takes a handler and returns a wrapped handler
664
- """
665
-
666
- def wrapper(handler_fn: Callable):
667
- async def transcription_wrapper(
668
- batch_request_body: (BatchTranscriptionRequest | BatchTranslationRequest),
669
- ) -> (
670
- TranscriptionResponse
671
- | TranscriptionResponseVerbose
672
- | TranslationResponse
673
- | TranslationResponseVerbose
674
- | ErrorResponse
675
- ):
676
- try:
677
- # Download data from URL
678
- audio_data = await download_bytes_from_url(
679
- batch_request_body.file_url,
680
- allowed_media_domains=allowed_media_domains,
681
- )
682
-
683
- # Create a mock file from the downloaded audio data
684
- mock_file = UploadFile(
685
- file=BytesIO(audio_data),
686
- filename="audio.bin",
687
- )
688
-
689
- # Convert batch request to regular request
690
- # by copying all fields except file_url and setting file to mock_file
691
- request_dict = batch_request_body.model_dump(exclude={"file_url"})
692
- request_dict["file"] = mock_file
693
-
694
- if is_translation:
695
- # Create TranslationRequest from BatchTranslationRequest
696
- translation_request = TranslationRequest.model_validate(
697
- request_dict
698
- )
699
- return await handler_fn(audio_data, translation_request)
700
- else:
701
- # Create TranscriptionRequest from BatchTranscriptionRequest
702
- transcription_request = TranscriptionRequest.model_validate(
703
- request_dict
704
- )
705
- return await handler_fn(audio_data, transcription_request)
706
- except Exception as e:
707
- operation = "translation" if is_translation else "transcription"
708
- return ErrorResponse(
709
- error=ErrorInfo(
710
- message=f"Failed to process {operation}: {str(e)}",
711
- type="BadRequestError",
712
- code=HTTPStatus.BAD_REQUEST.value,
713
- )
714
- )
715
-
716
- return transcription_wrapper
717
-
718
- return wrapper
719
-
720
-
721
-async def build_endpoint_registry(
722
- engine_client: EngineClient,
723
- args: Namespace,
724
-) -> dict[str, dict[str, Any]]:
725
- """
726
- Build the endpoint registry with all serving objects and handler configurations.
727
-
728
- Args:
729
- engine_client: The engine client
730
- args: Command line arguments
731
-
732
- Returns:
733
- Dictionary mapping endpoint keys to their configurations
734
- """
735
- supported_tasks = await engine_client.get_supported_tasks()
736
- logger.info("Supported tasks: %s", supported_tasks)
737
-
738
- # Create a state object to hold serving objects
739
- state = State()
740
-
741
- # Initialize all serving objects using init_app_state
742
- # This provides full functionality including chat template processing,
743
- # LoRA support, tool servers, etc.
744
- await init_app_state(engine_client, state, args, supported_tasks)
745
-
746
- # Get serving objects from state (defaulting to None if not set)
747
- openai_serving_chat = getattr(state, "openai_serving_chat", None)
748
- openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
749
- openai_serving_translation = getattr(state, "openai_serving_translation", None)
750
- serving_embedding = getattr(state, "serving_embedding", None)
751
- serving_scores = getattr(state, "serving_scores", None)
752
-
753
- allowed_media_domains = getattr(args, "allowed_media_domains", None)
754
-
755
- # Registry of endpoint configurations
756
- endpoint_registry: dict[str, dict[str, Any]] = {
757
- "completions": {
758
- "url_matcher": lambda url: url == "/v1/chat/completions",
759
- "handler_getter": lambda: (
760
- openai_serving_chat.create_chat_completion
761
- if openai_serving_chat is not None
762
- else None
763
- ),
764
- "wrapper_fn": None,
765
- },
766
- "embeddings": {
767
- "url_matcher": lambda url: url == "/v1/embeddings",
768
- "handler_getter": lambda: (
769
- serving_embedding if serving_embedding is not None else None
770
- ),
771
- "wrapper_fn": None,
772
- },
773
- "score": {
774
- "url_matcher": lambda url: url.endswith("/score"),
775
- "handler_getter": lambda: (
776
- serving_scores if serving_scores is not None else None
777
- ),
778
- "wrapper_fn": None,
779
- },
780
- "rerank": {
781
- "url_matcher": lambda url: url.endswith("/rerank"),
782
- "handler_getter": lambda: (
783
- serving_scores if serving_scores is not None else None
784
- ),
785
- "wrapper_fn": None,
786
- },
787
- "transcriptions": {
788
- "url_matcher": lambda url: url == "/v1/audio/transcriptions",
789
- "handler_getter": lambda: (
790
- openai_serving_transcription.create_transcription
791
- if openai_serving_transcription is not None
792
- else None
793
- ),
794
- "wrapper_fn": make_transcription_wrapper(
795
- is_translation=False,
796
- allowed_media_domains=allowed_media_domains,
797
- ),
798
- },
799
- "translations": {
800
- "url_matcher": lambda url: url == "/v1/audio/translations",
801
- "handler_getter": lambda: (
802
- openai_serving_translation.create_translation
803
- if openai_serving_translation is not None
804
- else None
805
- ),
806
- "wrapper_fn": make_transcription_wrapper(
807
- is_translation=True,
808
- allowed_media_domains=allowed_media_domains,
809
- ),
810
- },
811
- }
812
-
813
- return endpoint_registry
814
-
815
-
816
-def validate_run_batch_args(args):
817
- valid_reasoning_parsers = ReasoningParserManager.list_registered()
818
- if (
819
- reasoning_parser := args.structured_outputs_config.reasoning_parser
820
- ) and reasoning_parser not in valid_reasoning_parsers:
821
- raise KeyError(
822
- f"invalid reasoning parser: {reasoning_parser} "
823
- f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
824
- )
825
-
826
-
827
-async def run_batch(
828
- engine_client: EngineClient,
829
- args: Namespace,
830
-) -> None:
831
- endpoint_registry = await build_endpoint_registry(
832
- engine_client=engine_client,
833
- args=args,
834
- )
835
-
836
- tracker = BatchProgressTracker()
837
- logger.info("Reading batch from %s...", args.input_file)
838
-
839
- # Submit all requests in the file to the engine "concurrently".
840
- response_futures: list[Awaitable[BatchRequestOutput]] = []
841
- for request_json in (await read_file(args.input_file)).strip().split("\n"):
842
- # Skip empty lines.
843
- request_json = request_json.strip()
844
- if not request_json:
845
- continue
846
-
847
- request = BatchRequestInput.model_validate_json(request_json)
848
-
849
- # Use the last segment of the URL as the endpoint key.
850
- # More advanced URL matching is done in url_matcher of endpoint_registry.
851
- endpoint_key = request.url.split("/")[-1]
852
-
853
- result = None
854
- if endpoint_key in endpoint_registry:
855
- endpoint_config = endpoint_registry[endpoint_key]
856
- result = handle_endpoint_request(
857
- request,
858
- tracker,
859
- url_matcher=endpoint_config["url_matcher"],
860
- handler_getter=endpoint_config["handler_getter"],
861
- wrapper_fn=endpoint_config["wrapper_fn"],
862
- )
863
-
864
- if result is not None:
865
- response_futures.append(result)
866
- else:
867
- response_futures.append(
868
- make_async_error_request_output(
869
- request,
870
- error_msg=f"URL {request.url} was used. "
871
- "Supported endpoints: /v1/chat/completions, /v1/embeddings,"
872
- " /v1/audio/transcriptions, /v1/audio/translations, /score, "
873
- " /rerank.",
874
- )
875
- )
876
-
877
- with tracker.pbar():
878
- responses = await asyncio.gather(*response_futures)
879
-
880
- await write_file(args.output_file, responses, args.output_tmp_dir)
881
-
882
-
883
-async def main(args: Namespace):
884
- from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
885
- from vllm.usage.usage_lib import UsageContext
886
-
887
- validate_run_batch_args(args)
888
-
889
- async with build_async_engine_client(
890
- args,
891
- usage_context=UsageContext.OPENAI_BATCH_RUNNER,
892
- ) as engine_client:
893
- await run_batch(engine_client, args)
19
+if __name__ == "__main__":
20
+ from vllm.entrypoints.launchers.run_batch import main, parse_args
21
22
+ logger = init_logger(__name__)
23
896
-if __name__ == "__main__":
24
args = parse_args()
25
26
logger.info("vLLM batch processing API version %s", VLLM_VERSION)