A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Fix Negative Cuda Memory Usage (#25683)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Oct 1, 2025 at 18:16 UTC
da554f932ee20990cc74c9f8c80debb5c0f1605b
1 file changed
+4
-2
vllm/v1/worker/gpu_model_runner.py
+4
-2
@@ -3517,7 +3517,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3517
compilation_counter.num_gpu_runner_capture_triggers += 1
3518
3519
start_time = time.perf_counter()
3520
- start_free_gpu_memory = torch.cuda.mem_get_info()[0]
3520
3521
@contextmanager
3522
def freeze_gc():
@@ -3540,6 +3539,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3539
# can reuse the memory pool allocated for the large shapes.
3540
set_cudagraph_capturing_enabled(True)
3541
with freeze_gc(), graph_capture(device=self.device):
3542
+ start_free_gpu_memory = torch.cuda.mem_get_info()[0]
3543
cudagraph_mode = self.compilation_config.cudagraph_mode
3544
assert cudagraph_mode is not None
3545
if cudagraph_mode.mixed_mode() != CUDAGraphMode.NONE:
@@ -3568,6 +3568,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3568
cudagraph_runtime_mode=CUDAGraphMode.FULL,
3569
uniform_decode=True)
3570
3571
+ torch.cuda.synchronize()
3572
+ end_free_gpu_memory = torch.cuda.mem_get_info()[0]
3573
+
3574
# Disable cudagraph capturing globally, so any unexpected cudagraph
3575
# capturing will be detected and raise an error after here.
3576
# Note: We don't put it into graph_capture context manager because
@@ -3576,7 +3579,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3579
set_cudagraph_capturing_enabled(False)
3580
3581
end_time = time.perf_counter()
3579
- end_free_gpu_memory = torch.cuda.mem_get_info()[0]
3582
elapsed_time = end_time - start_time
3583
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
3584
# This usually takes 5~20 seconds.