A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Frontend] add run batch to CLI (#18804)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
May 28, 2025 at 22:08 UTC
435fa95444b2b0e408dc4b51ce7e759bbb3d0bcc
5 files changed
+110
-22
examples/offline_inference/openai_batch/README.md
+35
-2
@@ -48,7 +48,19 @@ The batch running tool is designed to be used from the command line.
48
You can run the batch with the following command, which will write its results to a file called `results.jsonl`
49
50
```console
51
-python -m vllm.entrypoints.openai.run_batch -i offline_inference/openai_batch/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
51
+python -m vllm.entrypoints.openai.run_batch \
52
+ -i offline_inference/openai_batch/openai_example_batch.jsonl \
53
+ -o results.jsonl \
54
+ --model meta-llama/Meta-Llama-3-8B-Instruct
55
+```
56
+
57
+or use command-line:
58
+
59
+```console
60
+vllm run-batch \
61
+ -i offline_inference/openai_batch/openai_example_batch.jsonl \
62
+ -o results.jsonl \
63
+ --model meta-llama/Meta-Llama-3-8B-Instruct
64
```
65
66
### Step 3: Check your results
@@ -68,7 +80,19 @@ The batch runner supports remote input and output urls that are accessible via h
80
For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl`, you can run
81
82
```console
71
-python -m vllm.entrypoints.openai.run_batch -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
83
+python -m vllm.entrypoints.openai.run_batch \
84
+ -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl \
85
+ -o results.jsonl \
86
+ --model meta-llama/Meta-Llama-3-8B-Instruct
87
+```
88
+
89
+or use command-line:
90
+
91
+```console
92
+vllm run-batch \
93
+ -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl \
94
+ -o results.jsonl \
95
+ --model meta-llama/Meta-Llama-3-8B-Instruct
96
```
97
98
## Example 3: Integrating with AWS S3
@@ -164,6 +188,15 @@ python -m vllm.entrypoints.openai.run_batch \
188
--model --model meta-llama/Meta-Llama-3-8B-Instruct
189
```
190
191
+or use command-line:
192
+
193
+```console
194
+vllm run-batch \
195
+ -i "https://s3.us-west-2.amazonaws.com/MY_BUCKET/MY_INPUT_FILE.jsonl?AWSAccessKeyId=ABCDEFGHIJKLMNOPQRST&Signature=abcdefghijklmnopqrstuvwxyz12345&Expires=1715800091" \
196
+ -o "https://s3.us-west-2.amazonaws.com/MY_BUCKET/MY_OUTPUT_FILE.jsonl?AWSAccessKeyId=ABCDEFGHIJKLMNOPQRST&Signature=abcdefghijklmnopqrstuvwxyz12345&Expires=1715800091" \
197
+ --model --model meta-llama/Meta-Llama-3-8B-Instruct
198
+```
199
+
200
### Step 4: View your results
201
202
Your results are now on S3. You can view them in your terminal by running
tests/entrypoints/openai/test_run_batch.py
+10
-16
@@ -2,7 +2,6 @@
2
3
import json
4
import subprocess
5
-import sys
5
import tempfile
6
7
from vllm.entrypoints.openai.protocol import BatchRequestOutput
@@ -35,9 +34,8 @@ def test_empty_file():
34
input_file.write("")
35
input_file.flush()
36
proc = subprocess.Popen([
38
- sys.executable, "-m", "vllm.entrypoints.openai.run_batch", "-i",
39
- input_file.name, "-o", output_file.name, "--model",
40
- "intfloat/multilingual-e5-small"
37
+ "vllm", "run-batch", "-i", input_file.name, "-o", output_file.name,
38
+ "--model", "intfloat/multilingual-e5-small"
39
], )
40
proc.communicate()
41
proc.wait()
@@ -54,9 +52,8 @@ def test_completions():
52
input_file.write(INPUT_BATCH)
53
input_file.flush()
54
proc = subprocess.Popen([
57
- sys.executable, "-m", "vllm.entrypoints.openai.run_batch", "-i",
58
- input_file.name, "-o", output_file.name, "--model",
59
- "NousResearch/Meta-Llama-3-8B-Instruct"
55
+ "vllm", "run-batch", "-i", input_file.name, "-o", output_file.name,
56
+ "--model", "NousResearch/Meta-Llama-3-8B-Instruct"
57
], )
58
proc.communicate()
59
proc.wait()
@@ -79,9 +76,8 @@ def test_completions_invalid_input():
76
input_file.write(INVALID_INPUT_BATCH)
77
input_file.flush()
78
proc = subprocess.Popen([
82
- sys.executable, "-m", "vllm.entrypoints.openai.run_batch", "-i",
83
- input_file.name, "-o", output_file.name, "--model",
84
- "NousResearch/Meta-Llama-3-8B-Instruct"
79
+ "vllm", "run-batch", "-i", input_file.name, "-o", output_file.name,
80
+ "--model", "NousResearch/Meta-Llama-3-8B-Instruct"
81
], )
82
proc.communicate()
83
proc.wait()
@@ -95,9 +91,8 @@ def test_embeddings():
91
input_file.write(INPUT_EMBEDDING_BATCH)
92
input_file.flush()
93
proc = subprocess.Popen([
98
- sys.executable, "-m", "vllm.entrypoints.openai.run_batch", "-i",
99
- input_file.name, "-o", output_file.name, "--model",
100
- "intfloat/multilingual-e5-small"
94
+ "vllm", "run-batch", "-i", input_file.name, "-o", output_file.name,
95
+ "--model", "intfloat/multilingual-e5-small"
96
], )
97
proc.communicate()
98
proc.wait()
@@ -117,9 +112,8 @@ def test_score():
112
input_file.write(INPUT_SCORE_BATCH)
113
input_file.flush()
114
proc = subprocess.Popen([
120
- sys.executable,
121
- "-m",
122
- "vllm.entrypoints.openai.run_batch",
115
+ "vllm",
116
+ "run-batch",
117
"-i",
118
input_file.name,
119
"-o",
vllm/entrypoints/cli/main.py
+2
@@ -7,6 +7,7 @@ import sys
7
import vllm.entrypoints.cli.benchmark.main
8
import vllm.entrypoints.cli.collect_env
9
import vllm.entrypoints.cli.openai
10
+import vllm.entrypoints.cli.run_batch
11
import vllm.entrypoints.cli.serve
12
import vllm.version
13
from vllm.entrypoints.utils import VLLM_SERVE_PARSER_EPILOG, cli_env_setup
@@ -17,6 +18,7 @@ CMD_MODULES = [
18
vllm.entrypoints.cli.serve,
19
vllm.entrypoints.cli.benchmark.main,
20
vllm.entrypoints.cli.collect_env,
21
+ vllm.entrypoints.cli.run_batch,
22
]
23
24
vllm/entrypoints/cli/run_batch.py
new
+55
@@ -0,0 +1,55 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import argparse
4
+import asyncio
5
+
6
+from prometheus_client import start_http_server
7
+
8
+from vllm.entrypoints.cli.types import CLISubcommand
9
+from vllm.entrypoints.logger import logger
10
+from vllm.entrypoints.openai.run_batch import main as run_batch_main
11
+from vllm.entrypoints.openai.run_batch import make_arg_parser
12
+from vllm.utils import FlexibleArgumentParser
13
+from vllm.version import __version__ as VLLM_VERSION
14
+
15
+
16
+class RunBatchSubcommand(CLISubcommand):
17
+ """The `run-batch` subcommand for vLLM CLI."""
18
+
19
+ def __init__(self):
20
+ self.name = "run-batch"
21
+ super().__init__()
22
+
23
+ @staticmethod
24
+ def cmd(args: argparse.Namespace) -> None:
25
+ logger.info("vLLM batch processing API version %s", VLLM_VERSION)
26
+ logger.info("args: %s", args)
27
+
28
+ # Start the Prometheus metrics server.
29
+ # LLMEngine uses the Prometheus client
30
+ # to publish metrics at the /metrics endpoint.
31
+ if args.enable_metrics:
32
+ logger.info("Prometheus metrics enabled")
33
+ start_http_server(port=args.port, addr=args.url)
34
+ else:
35
+ logger.info("Prometheus metrics disabled")
36
+
37
+ asyncio.run(run_batch_main(args))
38
+
39
+ def subparser_init(
40
+ self,
41
+ subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
42
+ run_batch_parser = subparsers.add_parser(
43
+ "run-batch",
44
+ help="Run batch prompts and write results to file.",
45
+ description=(
46
+ "Run batch prompts using vLLM's OpenAI-compatible API.\n"
47
+ "Supports local or HTTP input/output files."),
48
+ usage=
49
+ "vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
50
+ )
51
+ return make_arg_parser(run_batch_parser)
52
+
53
+
54
+def cmd_init() -> list[CLISubcommand]:
55
+ return [RunBatchSubcommand()]
vllm/entrypoints/openai/run_batch.py
+8
-4
@@ -33,9 +33,7 @@ from vllm.utils import FlexibleArgumentParser, random_uuid
33
from vllm.version import __version__ as VLLM_VERSION
34
35
36
-def parse_args():
37
- parser = FlexibleArgumentParser(
38
- description="vLLM OpenAI-Compatible batch runner.")
36
+def make_arg_parser(parser: FlexibleArgumentParser):
37
parser.add_argument(
38
"-i",
39
"--input-file",
@@ -98,7 +96,13 @@ def parse_args():
96
default=False,
97
help="If set to True, enable prompt_tokens_details in usage.")
98
101
- return parser.parse_args()
99
+ return parser
100
+
101
+
102
+def parse_args():
103
+ parser = FlexibleArgumentParser(
104
+ description="vLLM OpenAI-Compatible batch runner.")
105
+ return make_arg_parser(parser).parse_args()
106
107
108
# explicitly use pure text format, with a newline at the end