@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f659bf07f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Optimize cuda graph memory usage (#2437)

Roy committed Jan 15, 2024 at 01:40 UTC 9f659bf07fa3aa0183f2d36b09abaa5ef7cbd77a
1 file changed +10 -2
vllm/worker/model_runner.py
+10 -2
@@ -506,7 +506,9 @@ class ModelRunner:
506 "use '--enforce-eager' in the CLI.")
507 logger.info("CUDA graphs can take additional 1~3 GiB memory per GPU. "
508 "If you are running out of memory, consider decreasing "
509 - "`gpu_memory_utilization` or enforcing eager mode.")
509 + "`gpu_memory_utilization` or enforcing eager mode. "
510 + "You can also reduce the `max_num_seqs` as needed "
511 + "to decrease memory usage.")
512 start_time = time.perf_counter()
513
514 # Prepare dummy inputs. These will be reused for all batch sizes.
@@ -519,9 +521,15 @@ class ModelRunner:
521 context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
522 block_tables = torch.from_numpy(self.graph_block_tables).cuda()
523
524 + graph_batch_size = _get_graph_batch_size(
525 + self.scheduler_config.max_num_seqs)
526 + batch_size_capture_list = [
527 + bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
528 + ]
529 +
530 # NOTE: Capturing the largest batch size first may help reduce the
531 # memory usage of CUDA graph.
524 - for batch_size in reversed(_BATCH_SIZES_TO_CAPTURE):
532 + for batch_size in reversed(batch_size_capture_list):
533 # Create dummy input_metadata.
534 input_metadata = InputMetadata(
535 is_prompt=False,