@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96a85c5750

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 21:15 UTC

[Startup][UX] Enable CUDAGraph memory profiling by default (#38284)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>

Matthew Bonanni committed Apr 21, 2026 at 18:16 UTC 96a85c57501fe12592efc1c601a8fa2fd8214b81

8 files changed +26 -27

tests/distributed/test_torchrun_example.py

+1 -1

@@ -29,7 +29,7 @@ llm = LLM(

29 tensor_parallel_size=2,

30 pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)),

31 distributed_executor_backend="external_launcher",

32 - gpu_memory_utilization=random.uniform(0.7, 0.9),

32 + gpu_memory_utilization=random.uniform(0.8, 0.92),

33 seed=0,

34 )

tests/distributed/test_torchrun_example_moe.py

+1 -1

@@ -36,7 +36,7 @@ llm = LLM(

36 pipeline_parallel_size=int(os.getenv("PP_SIZE", "1")),

37 enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1,

38 distributed_executor_backend="external_launcher",

39 - gpu_memory_utilization=random.uniform(0.7, 0.9),

39 + gpu_memory_utilization=random.uniform(0.8, 0.92),

40 seed=0,

41 max_model_len=1024,

42 max_num_seqs=16,

tests/v1/determinism/test_batch_invariance.py

+1 -1

@@ -65,7 +65,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(

65 assert max_batch_size >= 2, "Batch size should be >= 2 to mix needle."

67 # Keep GPU memory usage low to avoid startup allocation failures.

68 - gpu_mem_util = float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.4"))

68 + gpu_mem_util = float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.5"))

69 max_model_len = int(os.getenv("VLLM_MAX_MODEL_LEN", "5120"))

71 # Sampling parameters: longer outputs with a more random-sounding

tests/v1/e2e/spec_decode/test_spec_decode.py

+2 -2

@@ -321,7 +321,7 @@ def test_speculators_model_integration(

321 test_prompts = get_test_prompts(mm_enabled=False)

322

323 # First run: Direct speculator model (simplified integration)

324 - spec_llm = LLM(model=model_path, max_model_len=4096)

324 + spec_llm = LLM(model=model_path, max_model_len=4096, gpu_memory_utilization=0.92)

325 evaluate_llm_for_gsm8k(

326 spec_llm, expected_accuracy_threshold=expected_accuracy_threshold

327 )

@@ -351,7 +351,7 @@ def test_speculators_model_integration(

351 cleanup_dist_env_and_memory()

352

353 # Second run: Reference without speculative decoding

354 - ref_llm = LLM(model=verifier_model, max_model_len=4096)

354 + ref_llm = LLM(model=verifier_model, max_model_len=4096, gpu_memory_utilization=0.92)

355 ref_outputs = ref_llm.chat(test_prompts, sampling_config)

356 del ref_llm

357 torch.accelerator.empty_cache()

vllm/config/cache.py

+2 -2

@@ -51,10 +51,10 @@ class CacheConfig:

51 """Whether block_size was explicitly provided. Derived automatically."""

52 user_specified_mamba_block_size: bool = field(default=False, init=False)

53 """Whether mamba_block_size was explicitly provided. Derived automatically."""

54 - gpu_memory_utilization: float = Field(default=0.9, gt=0, le=1)

54 + gpu_memory_utilization: float = Field(default=0.92, gt=0, le=1)

55 """The fraction of GPU memory to be used for the model executor, which can

56 range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory

57 - utilization. If unspecified, will use the default value of 0.9. This is a

57 + utilization. If unspecified, will use the default value of 0.92. This is a

58 per-instance limit, and only applies to the current vLLM instance. It does

59 not matter if you have another vLLM instance running on the same GPU. For

60 example, if you have two vLLM instances running on the same GPU, you can

vllm/entrypoints/llm.py

+1 -1

@@ -228,7 +228,7 @@ class LLM:

228 tokenizer_revision: str | None = None,

229 chat_template: Path | str | None = None,

230 seed: int = 0,

231 - gpu_memory_utilization: float = 0.9,

231 + gpu_memory_utilization: float = 0.92,

232 cpu_offload_gb: float = 0,

233 offload_group_size: int = 0,

234 offload_num_in_group: int = 1,

vllm/envs.py

+3 -3

@@ -253,7 +253,7 @@ if TYPE_CHECKING:

253 VLLM_CUDA_COMPATIBILITY_PATH: str | None = None

254 VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False

255 VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False

256 - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = False

256 + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True

257 VLLM_NIXL_EP_MAX_NUM_RANKS: int = 32

258 VLLM_XPU_ENABLE_XPU_GRAPH: bool = False

259 VLLM_LORA_ENABLE_DUAL_STREAM: bool = False

@@ -1687,9 +1687,9 @@ environment_variables: dict[str, Callable[[], Any]] = {

1687 ),

1688 # If set to 1, enable CUDA graph memory estimation during memory profiling.

1689 # This profiles CUDA graph memory usage to provide more accurate KV cache

1690 - # memory allocation. Disabled by default to preserve existing behavior.

1690 + # memory allocation. Enabled by default as of v0.21.0

1691 "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool(

1692 - int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "0"))

1692 + int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1"))

1693 ),

1694 # NIXL EP environment variables

1695 "VLLM_NIXL_EP_MAX_NUM_RANKS": lambda: int(

vllm/v1/worker/gpu_worker.py

+15 -16

@@ -454,14 +454,13 @@ class Worker(WorkerBase):

454 1.0,

455 )

456 logger.info(

457 - "CUDA graph memory profiling is enabled "

458 - "(VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1). "

459 - "This will become the default in v0.21. "

460 - "The current --gpu-memory-utilization=%.4f is equivalent "

461 - "to --gpu-memory-utilization=%.4f without CUDA graph "

462 - "memory profiling. To maintain the same effective KV "

463 - "cache size as before, increase "

464 - "--gpu-memory-utilization to %.4f.",

457 + "CUDA graph memory profiling is enabled (default since "

458 + "v0.21.0). The current --gpu-memory-utilization=%.4f is "

459 + "equivalent to --gpu-memory-utilization=%.4f without "

460 + "CUDA graph memory profiling. To maintain the same "

461 + "effective KV cache size as before, increase "

462 + "--gpu-memory-utilization to %.4f. To disable, set "

463 + "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.",

464 current_util,

465 equiv_util,

466 suggested_util,

@@ -471,14 +470,14 @@ class Worker(WorkerBase):

470 round(current_util + cg_util_delta, 4),

471 1.0,

472 )

474 - logger.info(

475 - "In v0.21, CUDA graph memory profiling will be enabled "

476 - "by default (VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1), "

477 - "which more accurately accounts for CUDA graph memory "

478 - "during KV cache allocation. To try it now, set "

479 - "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 and increase "

480 - "--gpu-memory-utilization from %.4f to %.4f to maintain "

481 - "the same effective KV cache size.",

473 + logger.warning(

474 + "CUDA graph memory profiling is disabled "

475 + "(VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0). "

476 + "Without it, CUDA graph memory is not accounted for "

477 + "during KV cache allocation, which may require lowering "

478 + "--gpu-memory-utilization to avoid OOM. Consider "

479 + "re-enabling it (the default as of v0.21.0) and increasing "

480 + "--gpu-memory-utilization from %.4f to %.4f.",

481 current_util,

482 suggested_util,

483 )