@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f9310cbd0c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] Fix Compilation config & Enable CUDA graph by default (#10528)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 21, 2024 at 12:53 UTC f9310cbd0c1109c4f22cf9f1dc615b2d08f06408
3 files changed +62 -42
vllm/config.py
+2 -1
@@ -2370,7 +2370,7 @@ class VllmConfig:
2370
2371 if self.compilation_config is None:
2372 self.compilation_config = CompilationConfig()
2373 - if envs.VLLM_USE_V1:
2373 + if envs.VLLM_USE_V1 and not self.model_config.enforce_eager:
2374 # NOTE(woosuk): Currently, we use inductor because the piecewise
2375 # CUDA graphs do not work properly with the custom CUDA kernels.
2376 # FIXME(woosuk): Disable inductor to reduce the compilation time
@@ -2380,6 +2380,7 @@ class VllmConfig:
2380 self.compilation_config.use_inductor = True
2381 self.compilation_config.pass_config.enable_fusion = False
2382 self.compilation_config.pass_config.enable_reshape = False
2383 + self.compilation_config.level = CompilationLevel.PIECEWISE
2384
2385 current_platform.check_and_update_config(self)
2386
vllm/v1/worker/gpu_model_runner.py
+34 -28
@@ -1,3 +1,4 @@
1 +import gc
2 import time
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
@@ -515,7 +516,25 @@ class GPUModelRunner:
516 logger.info("Loading model weights took %.4f GB",
517 self.model_memory_usage / float(2**30))
518
518 - def _dummy_run(self, model: nn.Module, num_tokens: int) -> None:
519 + @torch.inference_mode()
520 + def _dummy_run(
521 + self,
522 + model: nn.Module,
523 + num_tokens: int,
524 + kv_caches: List[torch.Tensor],
525 + ) -> torch.Tensor:
526 + with set_forward_context(None):
527 + hidden_states = model(
528 + input_ids=None,
529 + positions=self.positions[:num_tokens],
530 + kv_caches=kv_caches,
531 + attn_metadata=None,
532 + inputs_embeds=self.inputs_embeds[:num_tokens])
533 + return hidden_states
534 +
535 + def profile_run(self) -> None:
536 + # TODO(woosuk): Profile the max memory usage of the encoder and
537 + # the encoder cache.
538 # use an empty tensor instead of `None`` to force Dynamo to pass
539 # it by reference, rather by specializing on the value `None`.
540 # the `dtype` argument does not matter, and we use `float32` as
@@ -527,23 +546,17 @@ class GPUModelRunner:
546 torch.tensor([], dtype=torch.float32, device=self.device)
547 for _ in range(self.num_attn_layers)
548 ]
530 - with set_forward_context(None): # noqa: SIM117
531 - with set_compile_context(self.cudagraph_batch_sizes):
532 - # Trigger compilation for general shape.
533 - model(input_ids=None,
534 - positions=self.positions,
535 - kv_caches=dummy_kv_caches,
536 - attn_metadata=None,
537 - inputs_embeds=self.inputs_embeds)
538 -
539 - @torch.inference_mode()
540 - def profile_run(self) -> None:
541 - # TODO(woosuk): Profile the max memory usage of the encoder and
542 - # the encoder cache.
543 - self._dummy_run(self.model, self.max_num_tokens)
549 + with set_compile_context(self.cudagraph_batch_sizes):
550 + # Trigger compilation for general shape.
551 + hidden_states = self._dummy_run(self.model, self.max_num_tokens,
552 + dummy_kv_caches)
553 + logits = self.model.compute_logits(hidden_states, None)
554 + logits = logits[:self.max_num_tokens]
555 + # TODO(woosuk): Consider the memory usage of the sampler.
556 torch.cuda.synchronize()
557 + del hidden_states, logits
558 + gc.collect()
559
546 - @torch.inference_mode()
560 def capture_model(self) -> None:
561 if not self.use_cuda_graph:
562 logger.warning(
@@ -554,18 +567,11 @@ class GPUModelRunner:
567 start_time = time.perf_counter()
568 start_free_gpu_memory = torch.cuda.mem_get_info()[0]
569
557 - with set_forward_context(None):
558 - # Trigger CUDA graph capture for specific shapes.
559 - # Capture the large shapes first so that the smaller shapes
560 - # can reuse the memory pool allocated for the large shapes.
561 - for num_tokens in reversed(self.cudagraph_batch_sizes):
562 - self.model(
563 - input_ids=None,
564 - positions=self.positions[:num_tokens],
565 - kv_caches=self.kv_caches,
566 - attn_metadata=None,
567 - inputs_embeds=self.inputs_embeds[:num_tokens],
568 - )
570 + # Trigger CUDA graph capture for specific shapes.
571 + # Capture the large shapes first so that the smaller shapes
572 + # can reuse the memory pool allocated for the large shapes.
573 + for num_tokens in reversed(self.cudagraph_batch_sizes):
574 + self._dummy_run(self.model, num_tokens, self.kv_caches)
575
576 end_time = time.perf_counter()
577 end_free_gpu_memory = torch.cuda.mem_get_info()[0]
vllm/v1/worker/gpu_worker.py
+26 -13
@@ -105,35 +105,48 @@ class Worker:
105 # Profile the memory usage of the model and get the maximum number of
106 # cache blocks that can be allocated with the remaining free memory.
107 torch.cuda.empty_cache()
108 + torch.cuda.reset_peak_memory_stats()
109
110 + _, total_gpu_memory = torch.cuda.mem_get_info()
111 # Execute a forward pass with dummy inputs to profile the memory usage
112 # of the model.
113 self.model_runner.profile_run()
112 -
113 - # Calculate the number of blocks that can be allocated with the
114 - # profiled peak memory.
114 torch.cuda.synchronize()
116 - free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
115 +
116 + free_gpu_memory, _ = torch.cuda.mem_get_info()
117 # NOTE(woosuk): Here we assume that the other processes using the same
118 # GPU did not change their memory usage during the profiling.
119 - peak_memory = self.init_gpu_memory - free_gpu_memory
120 - assert peak_memory > 0, (
119 + assert self.init_gpu_memory > free_gpu_memory, (
120 "Error in memory profiling. "
121 f"Initial free memory {self.init_gpu_memory}, current free memory"
122 f" {free_gpu_memory}. This happens when the GPU memory was "
123 "not properly cleaned up before initializing the vLLM instance.")
124
125 + # Get the peak memory allocation recorded by torch
126 + peak_memory = torch.cuda.memory_stats()["allocated_bytes.all.peak"]
127 +
128 + # Check for any memory left around that may have been allocated on the
129 + # gpu outside of `torch`. NCCL operations, for example, can use a few
130 + # GB during a forward pass
131 + torch.cuda.empty_cache()
132 + torch_allocated_bytes = torch.cuda.memory_stats(
133 + )["allocated_bytes.all.current"]
134 + total_allocated_bytes = torch.cuda.mem_get_info(
135 + )[1] - torch.cuda.mem_get_info()[0]
136 + non_torch_allocations = total_allocated_bytes - torch_allocated_bytes
137 + if non_torch_allocations > 0:
138 + peak_memory += non_torch_allocations
139 + available_kv_cache_memory = (
140 + total_gpu_memory * self.cache_config.gpu_memory_utilization -
141 + peak_memory)
142 +
143 + # Calculate the number of blocks that can be allocated with the
144 + # profiled peak memory.
145 cache_block_size = _get_cache_block_size(self.cache_config,
146 self.model_config,
147 self.parallel_config)
129 - num_gpu_blocks = int(
130 - (total_gpu_memory * self.cache_config.gpu_memory_utilization -
131 - peak_memory) // cache_block_size)
148 + num_gpu_blocks = int(available_kv_cache_memory // cache_block_size)
149 num_gpu_blocks = max(num_gpu_blocks, 0)
133 - # if self.model_runner.lora_manager:
134 - # self.model_runner.remove_all_loras()
135 - gc.collect()
136 - torch.cuda.empty_cache()
150 return num_gpu_blocks, 0
151
152 def initialize_cache(self, num_gpu_blocks: int) -> None: