@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09d3550372
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Add logging for CUDA memory (#10027)

Signed-off-by: Chenghao Yang <yangalan1996@gmail.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Chenghao Yang <yangalan1996@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Chenghao (Alan) Yang committed Nov 5, 2024 at 11:50 UTC 09d3550372db10f8c75fddd437325a863265fd82
1 file changed +14 -10
vllm/worker/model_runner.py
+14 -10
@@ -48,9 +48,10 @@ from vllm.prompt_adapter.worker_manager import (
48 from vllm.sampling_params import SamplingParams
49 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50 from vllm.transformers_utils.config import uses_mrope
51 -from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52 - flatten_2d_lists, is_pin_memory_available,
53 - supports_dynamo, weak_ref_tensor)
51 +from vllm.utils import (DeviceMemoryProfiler, GiB_bytes, PyObjectCache,
52 + async_tensor_h2d, flatten_2d_lists,
53 + is_pin_memory_available, supports_dynamo,
54 + weak_ref_tensor)
55 from vllm.worker.model_runner_base import (
56 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
57 _add_attn_metadata_broadcastable_dict,
@@ -1383,16 +1384,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1384 per sequence in the batch.
1385 """
1386 assert not self.model_config.enforce_eager
1386 - logger.info("Capturing the model for CUDA graphs. This may lead to "
1387 + logger.info("Capturing cudagraphs for decoding. This may lead to "
1388 "unexpected consequences if the model is not static. To "
1389 "run the model in eager mode, set 'enforce_eager=True' or "
1390 "use '--enforce-eager' in the CLI.")
1390 - logger.info("CUDA graphs can take additional 1~3 GiB memory per GPU. "
1391 - "If you are running out of memory, consider decreasing "
1392 - "`gpu_memory_utilization` or enforcing eager mode. "
1393 - "You can also reduce the `max_num_seqs` as needed "
1394 - "to decrease memory usage.")
1391 + logger.info("If out-of-memory error occurs during cudagraph capture,"
1392 + " consider decreasing `gpu_memory_utilization` or "
1393 + "switching to eager mode. You can also reduce the "
1394 + "`max_num_seqs` as needed to decrease memory usage.")
1395 start_time = time.perf_counter()
1396 + start_free_gpu_memory = torch.cuda.mem_get_info()[0]
1397
1398 # Prepare dummy inputs. These will be reused for all batch sizes.
1399 max_batch_size = self.max_batchsize_to_capture
@@ -1497,9 +1498,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1498 graph_runner)
1499
1500 end_time = time.perf_counter()
1501 + end_free_gpu_memory = torch.cuda.mem_get_info()[0]
1502 elapsed_time = end_time - start_time
1503 + cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
1504 # This usually takes < 10 seconds.
1502 - logger.info("Graph capturing finished in %.0f secs.", elapsed_time)
1505 + logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
1506 + elapsed_time, cuda_graph_size / GiB_bytes)
1507
1508 def _update_inputs_to_capture_for_enc_dec_model(self,
1509 capture_inputs: Dict[str,