A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf] Increase default max splits for FA3 full cudagraphs (#25495)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Sep 23, 2025 at 19:53 UTC
e0b24ea0305e0ead2c1cb1e0488744b5a91d524e
1 file changed
+2
-2
vllm/envs.py
+2
-2
@@ -119,7 +119,7 @@ if TYPE_CHECKING:
119
VLLM_SERVER_DEV_MODE: bool = False
120
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
121
VLLM_MLA_DISABLE: bool = False
122
- VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH: int = 16
122
+ VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH: int = 32
123
VLLM_RAY_PER_WORKER_GPUS: float = 1.0
124
VLLM_RAY_BUNDLE_INDICES: str = ""
125
VLLM_CUDART_SO_PATH: Optional[str] = None
@@ -1017,7 +1017,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
1017
# max number splits for cuda graph decode
1018
"VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH":
1019
lambda: int(os.getenv("VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH",
1020
- "16")),
1020
+ "32")),
1021
1022
# Number of GPUs per worker in Ray, if it is set to be a fraction,
1023
# it allows ray to schedule multiple actors on a single GPU,