A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmark] Show E2EL by default for pooling models (#27014)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 16, 2025 at 20:47 UTC
334535b6fb54484d801e4ba4f91f3978cc5abbae
1 file changed
+16
-5
vllm/benchmarks/serve.py
+16
-5
@@ -58,7 +58,7 @@ TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) a
58
59
class TaskType(Enum):
60
GENERATION = "generation"
61
- EMBEDDING = "embedding"
61
+ POOLING = "pooling"
62
63
64
@dataclass
@@ -1084,10 +1084,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
1084
parser.add_argument(
1085
"--percentile-metrics",
1086
type=str,
1087
- default="ttft,tpot,itl",
1087
+ default=None,
1088
help="Comma-separated list of selected metrics to report percentils. "
1089
"This argument specifies the metrics to report percentiles. "
1090
- 'Allowed metric names are "ttft", "tpot", "itl", "e2el". ',
1090
+ 'Allowed metric names are "ttft", "tpot", "itl", "e2el". '
1091
+ 'If not specified, defaults to "ttft,tpot,itl" for generative models '
1092
+ 'and "e2el" for pooling models.',
1093
)
1094
parser.add_argument(
1095
"--metric-percentiles",
@@ -1310,7 +1312,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1312
goodput_config_dict = check_goodput_args(args)
1313
1314
backend = args.backend
1313
- task_type = TaskType.EMBEDDING if "embeddings" in backend else TaskType.GENERATION
1315
+ task_type = (
1316
+ TaskType.POOLING
1317
+ if "embeddings" in backend or "rerank" in backend
1318
+ else TaskType.GENERATION
1319
+ )
1320
1321
# Collect the sampling parameters.
1322
if task_type == TaskType.GENERATION:
@@ -1336,12 +1342,17 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1342
1343
if "temperature" not in sampling_params:
1344
sampling_params["temperature"] = 0.0 # Default to greedy decoding.
1345
+
1346
+ default_percentile_metrics = "ttft,tpot,itl"
1347
else:
1348
sampling_params = {}
1349
+ default_percentile_metrics = "e2el"
1350
1351
extra_body = args.extra_body or {}
1352
extra_body = {**sampling_params, **extra_body}
1353
1354
+ percentile_metrics: str = args.percentile_metrics or default_percentile_metrics
1355
+
1356
# Avoid GC processing "static" data - reduce pause times.
1357
gc.collect()
1358
gc.freeze()
@@ -1360,7 +1371,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1371
burstiness=args.burstiness,
1372
disable_tqdm=args.disable_tqdm,
1373
profile=args.profile,
1363
- selected_percentile_metrics=args.percentile_metrics.split(","),
1374
+ selected_percentile_metrics=percentile_metrics.split(","),
1375
selected_percentiles=[float(p) for p in args.metric_percentiles.split(",")],
1376
ignore_eos=args.ignore_eos,
1377
goodput_config_dict=goodput_config_dict,