A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Add logging for CUDA memory (#10027)
Signed-off-by: Chenghao Yang <yangalan1996@gmail.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Chenghao Yang <yangalan1996@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Chenghao (Alan) Yang committed
Nov 5, 2024 at 11:50 UTC
09d3550372db10f8c75fddd437325a863265fd82
1 file changed
+14
-10
vllm/worker/model_runner.py
+14
-10
@@ -48,9 +48,10 @@ from vllm.prompt_adapter.worker_manager import (
48
from vllm.sampling_params import SamplingParams
49
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50
from vllm.transformers_utils.config import uses_mrope
51
-from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52
- flatten_2d_lists, is_pin_memory_available,
53
- supports_dynamo, weak_ref_tensor)
51
+from vllm.utils import (DeviceMemoryProfiler, GiB_bytes, PyObjectCache,
52
+ async_tensor_h2d, flatten_2d_lists,
53
+ is_pin_memory_available, supports_dynamo,
54
+ weak_ref_tensor)
55
from vllm.worker.model_runner_base import (
56
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
57
_add_attn_metadata_broadcastable_dict,
@@ -1383,16 +1384,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1384
per sequence in the batch.
1385
"""
1386
assert not self.model_config.enforce_eager
1386
- logger.info("Capturing the model for CUDA graphs. This may lead to "
1387
+ logger.info("Capturing cudagraphs for decoding. This may lead to "
1388
"unexpected consequences if the model is not static. To "
1389
"run the model in eager mode, set 'enforce_eager=True' or "
1390
"use '--enforce-eager' in the CLI.")
1390
- logger.info("CUDA graphs can take additional 1~3 GiB memory per GPU. "
1391
- "If you are running out of memory, consider decreasing "
1392
- "`gpu_memory_utilization` or enforcing eager mode. "
1393
- "You can also reduce the `max_num_seqs` as needed "
1394
- "to decrease memory usage.")
1391
+ logger.info("If out-of-memory error occurs during cudagraph capture,"
1392
+ " consider decreasing `gpu_memory_utilization` or "
1393
+ "switching to eager mode. You can also reduce the "
1394
+ "`max_num_seqs` as needed to decrease memory usage.")
1395
start_time = time.perf_counter()
1396
+ start_free_gpu_memory = torch.cuda.mem_get_info()[0]
1397
1398
# Prepare dummy inputs. These will be reused for all batch sizes.
1399
max_batch_size = self.max_batchsize_to_capture
@@ -1497,9 +1498,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1498
graph_runner)
1499
1500
end_time = time.perf_counter()
1501
+ end_free_gpu_memory = torch.cuda.mem_get_info()[0]
1502
elapsed_time = end_time - start_time
1503
+ cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
1504
# This usually takes < 10 seconds.
1502
- logger.info("Graph capturing finished in %.0f secs.", elapsed_time)
1505
+ logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
1506
+ elapsed_time, cuda_graph_size / GiB_bytes)
1507
1508
def _update_inputs_to_capture_for_enc_dec_model(self,
1509
capture_inputs: Dict[str,