A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] Optimize cuda graph memory usage (#2437)
Roy committed
Jan 15, 2024 at 01:40 UTC
9f659bf07fa3aa0183f2d36b09abaa5ef7cbd77a
1 file changed
+10
-2
vllm/worker/model_runner.py
+10
-2
@@ -506,7 +506,9 @@ class ModelRunner:
506
"use '--enforce-eager' in the CLI.")
507
logger.info("CUDA graphs can take additional 1~3 GiB memory per GPU. "
508
"If you are running out of memory, consider decreasing "
509
- "`gpu_memory_utilization` or enforcing eager mode.")
509
+ "`gpu_memory_utilization` or enforcing eager mode. "
510
+ "You can also reduce the `max_num_seqs` as needed "
511
+ "to decrease memory usage.")
512
start_time = time.perf_counter()
513
514
# Prepare dummy inputs. These will be reused for all batch sizes.
@@ -519,9 +521,15 @@ class ModelRunner:
521
context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
522
block_tables = torch.from_numpy(self.graph_block_tables).cuda()
523
524
+ graph_batch_size = _get_graph_batch_size(
525
+ self.scheduler_config.max_num_seqs)
526
+ batch_size_capture_list = [
527
+ bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
528
+ ]
529
+
530
# NOTE: Capturing the largest batch size first may help reduce the
531
# memory usage of CUDA graph.
524
- for batch_size in reversed(_BATCH_SIZES_TO_CAPTURE):
532
+ for batch_size in reversed(batch_size_capture_list):
533
# Create dummy input_metadata.
534
input_metadata = InputMetadata(
535
is_prompt=False,