A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Clean up flags in `vllm bench serve` (#25138)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Sep 18, 2025 at 05:43 UTC
21da73343ad35f756e053ba4155dafb05229b0c5
4 files changed
+39
-23
docs/contributing/benchmarks.md
-3
@@ -156,7 +156,6 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct
156
```bash
157
vllm bench serve \
158
--backend openai-chat \
159
- --endpoint-type openai-chat \
159
--model Qwen/Qwen2-VL-7B-Instruct \
160
--endpoint /v1/chat/completions \
161
--dataset-name hf \
@@ -230,7 +229,6 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct
229
```bash
230
vllm bench serve \
231
--backend openai-chat \
233
- --endpoint-type openai-chat \
232
--model Qwen/Qwen2-VL-7B-Instruct \
233
--endpoint /v1/chat/completions \
234
--dataset-name hf \
@@ -245,7 +243,6 @@ vllm bench serve \
243
```bash
244
vllm bench serve \
245
--backend openai-chat \
248
- --endpoint-type openai-chat \
246
--model Qwen/Qwen2-VL-7B-Instruct \
247
--endpoint /v1/chat/completions \
248
--dataset-name hf \
tests/benchmarks/test_serve_cli.py
+1
-1
@@ -68,7 +68,7 @@ def test_bench_serve_chat(server):
68
"5",
69
"--endpoint",
70
"/v1/chat/completions",
71
- "--endpoint-type",
71
+ "--backend",
72
"openai-chat",
73
]
74
result = subprocess.run(command, capture_output=True, text=True)
vllm/benchmarks/datasets.py
+4
-4
@@ -1358,7 +1358,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1358
elif args.dataset_name == "sonnet":
1359
dataset = SonnetDataset(dataset_path=args.dataset_path)
1360
# For the "sonnet" dataset, formatting depends on the backend.
1361
- if args.endpoint_type == "openai-chat":
1361
+ if args.backend == "openai-chat":
1362
input_requests = dataset.sample(
1363
num_requests=args.num_prompts,
1364
input_len=args.sonnet_input_len,
@@ -1462,7 +1462,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1462
"Please consider contributing if you would "
1463
"like to add support for additional dataset formats.")
1464
1465
- if dataset_class.IS_MULTIMODAL and args.endpoint_type not in [
1465
+ if dataset_class.IS_MULTIMODAL and args.backend not in [
1466
"openai-chat",
1467
"openai-audio",
1468
]:
@@ -1470,7 +1470,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1470
# endpoint-type.
1471
raise ValueError(
1472
"Multi-modal content is only supported on 'openai-chat' and "
1473
- "'openai-audio' endpoint-type.")
1473
+ "'openai-audio' backends.")
1474
input_requests = dataset_class(
1475
dataset_path=args.dataset_path,
1476
dataset_subset=args.hf_subset,
@@ -1563,7 +1563,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1563
1564
try:
1565
# Enforce endpoint compatibility for multimodal datasets.
1566
- if args.dataset_name == "random-mm" and args.endpoint_type not in [
1566
+ if args.dataset_name == "random-mm" and args.backend not in [
1567
"openai-chat"]:
1568
raise ValueError(
1569
"Multi-modal content (images) is only supported on "
vllm/benchmarks/serve.py
+34
-15
@@ -8,8 +8,8 @@ to launch the vLLM OpenAI API server:
8
9
On the client side, run:
10
vllm bench serve \
11
- --endpoint-type <endpoint_type. Default 'openai'> \
12
- --label <benchmark result label. Default using endpoint_type> \
11
+ --backend <backend or endpoint type. Default 'openai'> \
12
+ --label <benchmark result label. Default using backend> \
13
--model <your_model> \
14
--dataset-name <dataset_name. Default 'random'> \
15
--request-rate <request_rate. Default inf> \
@@ -52,6 +52,21 @@ TERM_PLOTLIB_AVAILABLE = ((importlib.util.find_spec("termplotlib") is not None)
52
and (shutil.which("gnuplot") is not None))
53
54
55
+# TODO: Remove this in v0.11.0
56
+class DeprecatedEndpointTypeAction(argparse.Action):
57
+ """Argparse action for the deprecated --endpoint-type flag.
58
+ """
59
+
60
+ def __call__(self, _, namespace, values, option_string=None):
61
+ warnings.warn(
62
+ "'--endpoint-type' is deprecated and will be removed in v0.11.0. "
63
+ "Please use '--backend' instead or remove this argument if you "
64
+ "have already set it.",
65
+ stacklevel=1,
66
+ )
67
+ setattr(namespace, self.dest, values)
68
+
69
+
70
class TaskType(Enum):
71
GENERATION = "generation"
72
EMBEDDING = "embedding"
@@ -470,7 +485,7 @@ async def benchmark(
485
else:
486
request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
487
else:
473
- raise ValueError(f"Unknown endpoint_type: {endpoint_type}")
488
+ raise ValueError(f"Unknown backend: {endpoint_type}")
489
490
# Reuses connections across requests to reduce TLS handshake overhead.
491
connector = aiohttp.TCPConnector(
@@ -850,24 +865,28 @@ def save_to_pytorch_benchmark_format(args: argparse.Namespace,
865
866
def add_cli_args(parser: argparse.ArgumentParser):
867
add_dataset_parser(parser)
853
- parser.add_argument(
854
- "--endpoint-type",
855
- type=str,
856
- default="openai",
857
- choices=list(ASYNC_REQUEST_FUNCS.keys()),
858
- )
868
parser.add_argument(
869
"--label",
870
type=str,
871
default=None,
872
help="The label (prefix) of the benchmark results. If not specified, "
864
- "the endpoint type will be used as the label.",
873
+ "the value of '--backend' will be used as the label.",
874
)
875
parser.add_argument(
876
"--backend",
877
type=str,
869
- default="vllm",
878
+ default="openai",
879
+ choices=list(ASYNC_REQUEST_FUNCS.keys()),
880
+ help="The type of backend or endpoint to use for the benchmark."
881
+ )
882
+ parser.add_argument(
883
+ "--endpoint-type",
884
+ type=str,
885
+ default=None,
886
choices=list(ASYNC_REQUEST_FUNCS.keys()),
887
+ action=DeprecatedEndpointTypeAction,
888
+ help="'--endpoint-type' is deprecated and will be removed in v0.11.0. "
889
+ "Please use '--backend' instead.",
890
)
891
parser.add_argument(
892
"--base-url",
@@ -1165,7 +1184,6 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1184
raise ValueError(
1185
"For exponential ramp-up, the start RPS cannot be 0.")
1186
1168
- endpoint_type = args.endpoint_type
1187
label = args.label
1188
model_id = args.model
1189
model_name = args.served_model_name
@@ -1228,7 +1246,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1246
gc.freeze()
1247
1248
benchmark_result = await benchmark(
1231
- endpoint_type=args.endpoint_type,
1249
+ endpoint_type=args.backend,
1250
api_url=api_url,
1251
base_url=base_url,
1252
model_id=model_id,
@@ -1262,7 +1280,8 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1280
# Setup
1281
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
1282
result_json["date"] = current_dt
1265
- result_json["endpoint_type"] = args.endpoint_type
1283
+ result_json["endpoint_type"] = args.backend # for backward compatibility
1284
+ result_json["backend"] = args.backend
1285
result_json["label"] = label
1286
result_json["model_id"] = model_id
1287
result_json["tokenizer_id"] = tokenizer_id
@@ -1312,7 +1331,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1331
base_model_id = model_id.split("/")[-1]
1332
max_concurrency_str = (f"-concurrency{args.max_concurrency}"
1333
if args.max_concurrency is not None else "")
1315
- label = label or endpoint_type
1334
+ label = label or args.backend
1335
if args.ramp_up_strategy is not None:
1336
file_name = f"{label}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1337
else: