A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 07:00 UTC
fix CUDAGraph memory being counted twice (#37426)
Signed-off-by: Peter Pan <Peter.Pan@daocloud.io> Signed-off-by: Peter Pan <peter.pan@daocloud.io> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Peter Pan committed
Mar 21, 2026 at 01:36 UTC
79eb9369c5baa83db934407b0a448a5005c3dd72
1 file changed
+2
-3
vllm/v1/worker/gpu_worker.py
+2
-3
@@ -417,9 +417,7 @@ class Worker(WorkerBase):
417
)
418
419
self.non_torch_memory = profile_result.non_torch_increase
420
- self.peak_activation_memory = (
421
- profile_result.torch_peak_increase + cudagraph_memory_estimate_applied
422
- )
420
+ self.peak_activation_memory = profile_result.torch_peak_increase
421
self.cudagraph_memory_estimate = cudagraph_memory_estimate
422
423
free_gpu_memory = profile_result.after_profile.free_memory
@@ -638,6 +636,7 @@ class Worker(WorkerBase):
636
# slightly underestimate the memory consumption.
637
# So leave a small buffer (=150MiB) to avoid OOM.
638
redundancy_buffer_memory = 150 * (1 << 20)
639
+
640
non_kv_cache_memory = (
641
self.model_runner.model_memory_usage
642
+ self.peak_activation_memory