A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 08:45 UTC
[DSv4] Tune default value of `VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD` (#41526)
Co-authored-by: Copilot <copilot@github.com>
Roger Wang committed
May 2, 2026 at 18:32 UTC
856ec4804a4e236bdd796e0954094a532c7d19bc
1 file changed
+4
-4
vllm/envs.py
+4
-4
@@ -245,7 +245,7 @@ if TYPE_CHECKING:
245
VLLM_DEBUG_WORKSPACE: bool = False
246
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
247
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
248
- VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 4096
248
+ VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
249
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
250
VLLM_USE_V2_MODEL_RUNNER: bool = False
251
VLLM_LOG_MODEL_INSPECTION: bool = False
@@ -1686,10 +1686,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
1686
# tokens the FP8 main GEMM has idle SMs to share with the bf16 aux GEMMs
1687
# and overlap is a 5-45% win; above it the FP8 GEMM saturates the device
1688
# and the cross-stream sync becomes pure overhead. Set to 0 to disable
1689
- # the multi-stream path entirely. Empirical crossover on B300 (148 SMs)
1690
- # is ~4096; B200 (132 SMs) is expected ~3072.
1689
+ # the multi-stream path entirely. See #PR 41526 for the empirical result
1690
+ # for the default value of 1024 tokens.
1691
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
1692
- os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "4096")
1692
+ os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
1693
),
1694
# Format for saving torch.compile cache artifacts
1695
# - "binary": saves as binary file