A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 17:45 UTC
[UX] Add `--performance-mode {balanced,interactivity,throughput}` (#34936)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Feb 25, 2026 at 20:28 UTC
cbf8f7028cc0d80de4eeaf789b4bd56afbb5aafd
2 files changed
+37
-5
vllm/config/vllm.py
+26
-4
@@ -14,7 +14,7 @@ from datetime import datetime
14
from enum import IntEnum
15
from functools import lru_cache
16
from pathlib import Path
17
-from typing import TYPE_CHECKING, Any, TypeVar, get_args
17
+from typing import TYPE_CHECKING, Any, Literal, TypeVar, get_args
18
19
import torch
20
from pydantic import ConfigDict, Field, model_validator
@@ -76,6 +76,8 @@ class OptimizationLevel(IntEnum):
76
"""O3: Currently the same as -O2s."""
77
78
79
+PerformanceMode = Literal["balanced", "interactivity", "throughput"]
80
+
81
IS_QUANTIZED = False
82
IS_DENSE = False
83
# The optimizations that depend on these properties currently set to False
@@ -312,6 +314,13 @@ class VllmConfig:
314
performance. -O2 is used by default. See OptimizationLevel for full
315
description."""
316
317
+ performance_mode: PerformanceMode = "balanced"
318
+ """Performance mode for runtime behavior, 'balanced' is the default.
319
+ 'interactivity' favors low end-to-end per-request latency at small batch
320
+ sizes (fine-grained CUDA graphs, latency-oriented kernels).
321
+ 'throughput' favors aggregate tokens/sec at high concurrency (larger CUDA
322
+ graphs, more aggressive batching, throughput-oriented kernels)."""
323
+
324
weight_transfer_config: WeightTransferConfig | None = None
325
"""The configurations for weight transfer during RL training."""
326
@@ -643,6 +652,11 @@ class VllmConfig:
652
# To give each torch profile run a unique instance name.
653
self.instance_id = f"{time.time_ns()}"
654
655
+ if self.performance_mode != "balanced":
656
+ logger.info_once(
657
+ "Performance mode set to '%s'.", self.performance_mode, scope="local"
658
+ )
659
+
660
self.try_verify_and_update_config()
661
662
if self.model_config is not None:
@@ -1332,9 +1346,15 @@ class VllmConfig:
1346
# sort to make sure the sizes are in ascending order
1347
cudagraph_capture_sizes.sort()
1348
else:
1335
- cudagraph_capture_sizes = [
1336
- i for i in [1, 2, 4] if i <= max_cudagraph_capture_size
1337
- ]
1349
+ if self.performance_mode == "interactivity":
1350
+ # Fine-grained CUDA graphs at small batch sizes
1351
+ # for minimal padding overhead
1352
+ interactivity_max = min(max_cudagraph_capture_size, 32)
1353
+ cudagraph_capture_sizes = list(range(1, interactivity_max + 1))
1354
+ else:
1355
+ cudagraph_capture_sizes = [
1356
+ i for i in [1, 2, 4] if i <= max_cudagraph_capture_size
1357
+ ]
1358
if max_cudagraph_capture_size >= 8:
1359
# Step size 8 for small batch sizes, up to 256(not included)
1360
cudagraph_capture_sizes += list(
@@ -1345,6 +1365,8 @@ class VllmConfig:
1365
cudagraph_capture_sizes += list(
1366
range(256, max_cudagraph_capture_size + 1, 16)
1367
)
1368
+ # de-duplicate and sort the sizes
1369
+ cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
1370
1371
if (
1372
self.parallel_config.tensor_parallel_size > 1
vllm/engine/arg_utils.py
+11
-1
@@ -89,7 +89,7 @@ from vllm.config.parallel import (
89
)
90
from vllm.config.scheduler import SchedulerPolicy
91
from vllm.config.utils import get_field
92
-from vllm.config.vllm import OptimizationLevel
92
+from vllm.config.vllm import OptimizationLevel, PerformanceMode
93
from vllm.logger import init_logger, suppress_logging
94
from vllm.platforms import CpuArchEnum, current_platform
95
from vllm.plugins import load_general_plugins
@@ -596,6 +596,7 @@ class EngineArgs:
596
597
kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
598
optimization_level: OptimizationLevel = VllmConfig.optimization_level
599
+ performance_mode: PerformanceMode = VllmConfig.performance_mode
600
601
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
602
kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
@@ -1264,6 +1265,7 @@ class EngineArgs:
1265
vllm_group.add_argument(
1266
"--optimization-level", **vllm_kwargs["optimization_level"]
1267
)
1268
+ vllm_group.add_argument("--performance-mode", **vllm_kwargs["performance_mode"])
1269
vllm_group.add_argument(
1270
"--weight-transfer-config", **vllm_kwargs["weight_transfer_config"]
1271
)
@@ -1894,6 +1896,7 @@ class EngineArgs:
1896
profiler_config=self.profiler_config,
1897
additional_config=self.additional_config,
1898
optimization_level=self.optimization_level,
1899
+ performance_mode=self.performance_mode,
1900
weight_transfer_config=self.weight_transfer_config,
1901
)
1902
@@ -2110,6 +2113,13 @@ class EngineArgs:
2113
SchedulerConfig.DEFAULT_MAX_NUM_SEQS,
2114
)
2115
2116
+ # If throughput mode is set, double max_num_batched_tokens and max_num_seqs.
2117
+ if self.performance_mode == "throughput":
2118
+ if orig_max_num_batched_tokens is None:
2119
+ self.max_num_batched_tokens *= 2
2120
+ if orig_max_num_seqs is None:
2121
+ self.max_num_seqs *= 2
2122
+
2123
if orig_max_num_batched_tokens is None:
2124
assert model_config.max_model_len is not None, (
2125
"max_model_len must be set by this point"