A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 14:13 UTC
Support benchmarking of Geospatial models (#33922)
Signed-off-by: Michele Gazzetti <michele.gazzetti1@ibm.com>
mgazz committed
Feb 10, 2026 at 15:04 UTC
599e4335a42bbb6f2cad75ac0b4be81272a77aa3
3 files changed
+110
-56
vllm/benchmarks/datasets.py
+30
-24
@@ -2072,32 +2072,38 @@ class CustomDataset(BenchmarkDataset):
2072
break
2073
prompt = item["prompt"]
2074
2075
- new_output_len = output_len
2076
- if output_len is None or output_len == -1:
2077
- # check that the request has an 'output_tokens' field
2078
- if "output_tokens" not in item:
2079
- raise ValueError(
2080
- "If no output length is provided the "
2081
- "custom dataset must contain an 'output_tokens' field."
2075
+ if tokenizer is None:
2076
+ new_output_len = 1
2077
+ else:
2078
+ new_output_len = output_len
2079
+ if output_len is None or output_len == -1:
2080
+ # check that the request has an 'output_tokens' field
2081
+ if "output_tokens" not in item:
2082
+ raise ValueError(
2083
+ "If no output length is provided the "
2084
+ "custom dataset must contain an 'output_tokens' field."
2085
+ )
2086
+ # Use number of output tokens from the request data
2087
+ try:
2088
+ new_output_len = int(item["output_tokens"])
2089
+ except (ValueError, TypeError) as e:
2090
+ raise ValueError(
2091
+ f"Invalid value for 'output_tokens' in custom dataset: "
2092
+ f"'{item['output_tokens']}'. Must be an integer."
2093
+ ) from e
2094
+
2095
+ if tokenizer is None:
2096
+ prompt_len = 1
2097
+ else:
2098
+ # apply template
2099
+ if not skip_chat_template:
2100
+ prompt = tokenizer.apply_chat_template(
2101
+ [{"role": "user", "content": prompt}],
2102
+ add_generation_prompt=True,
2103
+ tokenize=False,
2104
)
2083
- # Use number of output tokens from the request data
2084
- try:
2085
- new_output_len = int(item["output_tokens"])
2086
- except (ValueError, TypeError) as e:
2087
- raise ValueError(
2088
- f"Invalid value for 'output_tokens' in custom dataset: "
2089
- f"'{item['output_tokens']}'. Must be an integer."
2090
- ) from e
2105
2092
- # apply template
2093
- if not skip_chat_template:
2094
- prompt = tokenizer.apply_chat_template(
2095
- [{"role": "user", "content": prompt}],
2096
- add_generation_prompt=True,
2097
- tokenize=False,
2098
- )
2099
-
2100
- prompt_len = len(tokenizer(prompt).input_ids)
2106
+ prompt_len = len(tokenizer(prompt).input_ids)
2107
sampled_requests.append(
2108
SampleRequest(
2109
prompt=prompt,
vllm/benchmarks/lib/endpoint_request_func.py
+32
@@ -746,6 +746,37 @@ async def async_request_infinity_embeddings_clip(
746
)
747
748
749
+async def async_request_vllm_pooling(
750
+ request_func_input: RequestFuncInput,
751
+ session: aiohttp.ClientSession,
752
+ pbar: tqdm | None = None,
753
+) -> RequestFuncOutput:
754
+ api_url = request_func_input.api_url
755
+ _validate_api_url(api_url, "vLLM Pooling API", "pooling")
756
+
757
+ payload = {
758
+ "model": request_func_input.model_name
759
+ if request_func_input.model_name
760
+ else request_func_input.model,
761
+ "truncate_prompt_tokens": -1,
762
+ }
763
+
764
+ payload = payload | request_func_input.prompt
765
+
766
+ _update_payload_common(payload, request_func_input)
767
+
768
+ headers = _get_headers("application/json")
769
+ _update_headers_common(headers, request_func_input)
770
+
771
+ return await _run_pooling_request(
772
+ session,
773
+ api_url,
774
+ payload=payload,
775
+ headers=headers,
776
+ pbar=pbar,
777
+ )
778
+
779
+
780
# TODO: Add more request functions for different API protocols.
781
ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
782
"vllm": async_request_openai_completions,
@@ -760,6 +791,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
791
"infinity-embeddings": async_request_infinity_embeddings,
792
"infinity-embeddings-clip": async_request_infinity_embeddings_clip,
793
# (Infinity embedding server does not support vlm2vec)
794
+ "vllm-pooling": async_request_vllm_pooling,
795
"vllm-rerank": async_request_vllm_rerank,
796
}
797
vllm/benchmarks/serve.py
+48
-32
@@ -423,16 +423,19 @@ def calculate_metrics(
423
output_len = outputs[i].output_tokens
424
425
if not output_len:
426
- # We use the tokenizer to count the number of output tokens
427
- # for some serving backends instead of looking at
428
- # len(outputs[i].itl) since multiple output tokens may be
429
- # bundled together
430
- # Note : this may inflate the output token count slightly
431
- output_len = len(
432
- tokenizer(
433
- outputs[i].generated_text, add_special_tokens=False
434
- ).input_ids
435
- )
426
+ if tokenizer is None:
427
+ output_len = 1
428
+ else:
429
+ # We use the tokenizer to count the number of output tokens
430
+ # for some serving backends instead of looking at
431
+ # len(outputs[i].itl) since multiple output tokens may be
432
+ # bundled together
433
+ # Note : this may inflate the output token count slightly
434
+ output_len = len(
435
+ tokenizer(
436
+ outputs[i].generated_text, add_special_tokens=False
437
+ ).input_ids
438
+ )
439
actual_output_lens.append(output_len)
440
total_input += input_requests[i].prompt_len
441
tpot = 0
@@ -919,7 +922,7 @@ async def benchmark(
922
print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate))
923
print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))
924
print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
922
- if isinstance(metrics, BenchmarkMetrics):
925
+ if isinstance(metrics, BenchmarkMetrics) and tokenizer:
926
print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))
927
print(
928
"{:<40} {:<10.2f}".format(
@@ -933,16 +936,18 @@ async def benchmark(
936
)
937
)
938
if isinstance(metrics, BenchmarkMetrics):
936
- print(
937
- "{:<40} {:<10.2f}".format(
938
- "Output token throughput (tok/s):", metrics.output_throughput
939
+ if tokenizer:
940
+ print(
941
+ "{:<40} {:<10.2f}".format(
942
+ "Output token throughput (tok/s):", metrics.output_throughput
943
+ )
944
)
940
- )
941
- print(
942
- "{:<40} {:<10.2f}".format(
943
- "Peak output token throughput (tok/s):", metrics.max_output_tokens_per_s
945
+ print(
946
+ "{:<40} {:<10.2f}".format(
947
+ "Peak output token throughput (tok/s):",
948
+ metrics.max_output_tokens_per_s,
949
+ )
950
)
945
- )
951
print(
952
"{:<40} {:<10.2f}".format(
953
"Peak concurrent requests:", metrics.max_concurrent_requests
@@ -954,11 +959,12 @@ async def benchmark(
959
"RTFx (Inverse Real-Time Factor):", metrics.rtfx
960
)
961
)
957
- print(
958
- "{:<40} {:<10.2f}".format(
959
- "Total token throughput (tok/s):", metrics.total_token_throughput
962
+ if tokenizer:
963
+ print(
964
+ "{:<40} {:<10.2f}".format(
965
+ "Total token throughput (tok/s):", metrics.total_token_throughput
966
+ )
967
)
961
- )
968
969
if isinstance(metrics, BenchmarkMetrics):
970
result = {
@@ -1047,7 +1053,7 @@ async def benchmark(
1053
print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value))
1054
result[f"p{p_word}_{metric_attribute_name}_ms"] = value
1055
1050
- if task_type == TaskType.GENERATION:
1056
+ if task_type == TaskType.GENERATION and tokenizer:
1057
process_one_metric("ttft", "TTFT", "Time to First Token")
1058
process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)")
1059
process_one_metric("itl", "ITL", "Inter-token Latency")
@@ -1519,6 +1525,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
1525
type=json.loads,
1526
default=None,
1527
)
1528
+ parser.add_argument(
1529
+ "--skip-tokenizer-init",
1530
+ action="store_true",
1531
+ default=False,
1532
+ help="Skip initialization of tokenizer and detokenizer",
1533
+ )
1534
1535
parser.add_argument(
1536
"--insecure",
@@ -1599,14 +1611,18 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1611
model_name = args.served_model_name
1612
model_id = args.model
1613
1602
- tokenizer_id = args.tokenizer if args.tokenizer is not None else model_id
1603
- tokenizer_mode = args.tokenizer_mode
1604
-
1605
- tokenizer = get_tokenizer(
1606
- tokenizer_id,
1607
- tokenizer_mode=tokenizer_mode,
1608
- trust_remote_code=args.trust_remote_code,
1609
- )
1614
+ if args.skip_tokenizer_init:
1615
+ tokenizer_id = None
1616
+ tokenizer_mode = None
1617
+ tokenizer = None
1618
+ else:
1619
+ tokenizer_id = args.tokenizer if args.tokenizer is not None else model_id
1620
+ tokenizer_mode = args.tokenizer_mode
1621
+ tokenizer = get_tokenizer(
1622
+ tokenizer_id,
1623
+ tokenizer_mode=tokenizer_mode,
1624
+ trust_remote_code=args.trust_remote_code,
1625
+ )
1626
1627
if args.dataset_name is None:
1628
raise ValueError(