A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] Fix Compilation config & Enable CUDA graph by default (#10528)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 21, 2024 at 12:53 UTC
f9310cbd0c1109c4f22cf9f1dc615b2d08f06408
3 files changed
+62
-42
vllm/config.py
+2
-1
@@ -2370,7 +2370,7 @@ class VllmConfig:
2370
2371
if self.compilation_config is None:
2372
self.compilation_config = CompilationConfig()
2373
- if envs.VLLM_USE_V1:
2373
+ if envs.VLLM_USE_V1 and not self.model_config.enforce_eager:
2374
# NOTE(woosuk): Currently, we use inductor because the piecewise
2375
# CUDA graphs do not work properly with the custom CUDA kernels.
2376
# FIXME(woosuk): Disable inductor to reduce the compilation time
@@ -2380,6 +2380,7 @@ class VllmConfig:
2380
self.compilation_config.use_inductor = True
2381
self.compilation_config.pass_config.enable_fusion = False
2382
self.compilation_config.pass_config.enable_reshape = False
2383
+ self.compilation_config.level = CompilationLevel.PIECEWISE
2384
2385
current_platform.check_and_update_config(self)
2386
vllm/v1/worker/gpu_model_runner.py
+34
-28
@@ -1,3 +1,4 @@
1
+import gc
2
import time
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
@@ -515,7 +516,25 @@ class GPUModelRunner:
516
logger.info("Loading model weights took %.4f GB",
517
self.model_memory_usage / float(2**30))
518
518
- def _dummy_run(self, model: nn.Module, num_tokens: int) -> None:
519
+ @torch.inference_mode()
520
+ def _dummy_run(
521
+ self,
522
+ model: nn.Module,
523
+ num_tokens: int,
524
+ kv_caches: List[torch.Tensor],
525
+ ) -> torch.Tensor:
526
+ with set_forward_context(None):
527
+ hidden_states = model(
528
+ input_ids=None,
529
+ positions=self.positions[:num_tokens],
530
+ kv_caches=kv_caches,
531
+ attn_metadata=None,
532
+ inputs_embeds=self.inputs_embeds[:num_tokens])
533
+ return hidden_states
534
+
535
+ def profile_run(self) -> None:
536
+ # TODO(woosuk): Profile the max memory usage of the encoder and
537
+ # the encoder cache.
538
# use an empty tensor instead of `None`` to force Dynamo to pass
539
# it by reference, rather by specializing on the value `None`.
540
# the `dtype` argument does not matter, and we use `float32` as
@@ -527,23 +546,17 @@ class GPUModelRunner:
546
torch.tensor([], dtype=torch.float32, device=self.device)
547
for _ in range(self.num_attn_layers)
548
]
530
- with set_forward_context(None): # noqa: SIM117
531
- with set_compile_context(self.cudagraph_batch_sizes):
532
- # Trigger compilation for general shape.
533
- model(input_ids=None,
534
- positions=self.positions,
535
- kv_caches=dummy_kv_caches,
536
- attn_metadata=None,
537
- inputs_embeds=self.inputs_embeds)
538
-
539
- @torch.inference_mode()
540
- def profile_run(self) -> None:
541
- # TODO(woosuk): Profile the max memory usage of the encoder and
542
- # the encoder cache.
543
- self._dummy_run(self.model, self.max_num_tokens)
549
+ with set_compile_context(self.cudagraph_batch_sizes):
550
+ # Trigger compilation for general shape.
551
+ hidden_states = self._dummy_run(self.model, self.max_num_tokens,
552
+ dummy_kv_caches)
553
+ logits = self.model.compute_logits(hidden_states, None)
554
+ logits = logits[:self.max_num_tokens]
555
+ # TODO(woosuk): Consider the memory usage of the sampler.
556
torch.cuda.synchronize()
557
+ del hidden_states, logits
558
+ gc.collect()
559
546
- @torch.inference_mode()
560
def capture_model(self) -> None:
561
if not self.use_cuda_graph:
562
logger.warning(
@@ -554,18 +567,11 @@ class GPUModelRunner:
567
start_time = time.perf_counter()
568
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
569
557
- with set_forward_context(None):
558
- # Trigger CUDA graph capture for specific shapes.
559
- # Capture the large shapes first so that the smaller shapes
560
- # can reuse the memory pool allocated for the large shapes.
561
- for num_tokens in reversed(self.cudagraph_batch_sizes):
562
- self.model(
563
- input_ids=None,
564
- positions=self.positions[:num_tokens],
565
- kv_caches=self.kv_caches,
566
- attn_metadata=None,
567
- inputs_embeds=self.inputs_embeds[:num_tokens],
568
- )
570
+ # Trigger CUDA graph capture for specific shapes.
571
+ # Capture the large shapes first so that the smaller shapes
572
+ # can reuse the memory pool allocated for the large shapes.
573
+ for num_tokens in reversed(self.cudagraph_batch_sizes):
574
+ self._dummy_run(self.model, num_tokens, self.kv_caches)
575
576
end_time = time.perf_counter()
577
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
vllm/v1/worker/gpu_worker.py
+26
-13
@@ -105,35 +105,48 @@ class Worker:
105
# Profile the memory usage of the model and get the maximum number of
106
# cache blocks that can be allocated with the remaining free memory.
107
torch.cuda.empty_cache()
108
+ torch.cuda.reset_peak_memory_stats()
109
110
+ _, total_gpu_memory = torch.cuda.mem_get_info()
111
# Execute a forward pass with dummy inputs to profile the memory usage
112
# of the model.
113
self.model_runner.profile_run()
112
-
113
- # Calculate the number of blocks that can be allocated with the
114
- # profiled peak memory.
114
torch.cuda.synchronize()
116
- free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
115
+
116
+ free_gpu_memory, _ = torch.cuda.mem_get_info()
117
# NOTE(woosuk): Here we assume that the other processes using the same
118
# GPU did not change their memory usage during the profiling.
119
- peak_memory = self.init_gpu_memory - free_gpu_memory
120
- assert peak_memory > 0, (
119
+ assert self.init_gpu_memory > free_gpu_memory, (
120
"Error in memory profiling. "
121
f"Initial free memory {self.init_gpu_memory}, current free memory"
122
f" {free_gpu_memory}. This happens when the GPU memory was "
123
"not properly cleaned up before initializing the vLLM instance.")
124
125
+ # Get the peak memory allocation recorded by torch
126
+ peak_memory = torch.cuda.memory_stats()["allocated_bytes.all.peak"]
127
+
128
+ # Check for any memory left around that may have been allocated on the
129
+ # gpu outside of `torch`. NCCL operations, for example, can use a few
130
+ # GB during a forward pass
131
+ torch.cuda.empty_cache()
132
+ torch_allocated_bytes = torch.cuda.memory_stats(
133
+ )["allocated_bytes.all.current"]
134
+ total_allocated_bytes = torch.cuda.mem_get_info(
135
+ )[1] - torch.cuda.mem_get_info()[0]
136
+ non_torch_allocations = total_allocated_bytes - torch_allocated_bytes
137
+ if non_torch_allocations > 0:
138
+ peak_memory += non_torch_allocations
139
+ available_kv_cache_memory = (
140
+ total_gpu_memory * self.cache_config.gpu_memory_utilization -
141
+ peak_memory)
142
+
143
+ # Calculate the number of blocks that can be allocated with the
144
+ # profiled peak memory.
145
cache_block_size = _get_cache_block_size(self.cache_config,
146
self.model_config,
147
self.parallel_config)
129
- num_gpu_blocks = int(
130
- (total_gpu_memory * self.cache_config.gpu_memory_utilization -
131
- peak_memory) // cache_block_size)
148
+ num_gpu_blocks = int(available_kv_cache_memory // cache_block_size)
149
num_gpu_blocks = max(num_gpu_blocks, 0)
133
- # if self.model_runner.lora_manager:
134
- # self.model_runner.remove_all_loras()
135
- gc.collect()
136
- torch.cuda.empty_cache()
150
return num_gpu_blocks, 0
151
152
def initialize_cache(self, num_gpu_blocks: int) -> None: