@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e650f56a1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[torch.compile] decouple compile sizes and cudagraph sizes (#12243)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 24, 2025 at 02:01 UTC 6e650f56a16618db87147d97f699fa407ed1205d
7 files changed +95 -58
vllm/compilation/backends.py
+5 -5
@@ -680,7 +680,7 @@ class VllmBackend:
680 class ConcreteSizeEntry:
681 runtime_shape: int
682 need_to_compile: bool # the size is in compile_sizes
683 - use_cudagraph: bool # the size is in capture_sizes
683 + use_cudagraph: bool # the size is in cudagraph_capture_sizes
684
685 compiled: bool = False
686 runnable: Callable = None # type: ignore
@@ -727,8 +727,8 @@ class PiecewiseBackend:
727
728 self.compile_sizes: Set[int] = set(
729 self.compilation_config.compile_sizes)
730 - self.capture_sizes: Set[int] = set(
731 - self.compilation_config.capture_sizes
730 + self.cudagraph_capture_sizes: Set[int] = set(
731 + self.compilation_config.cudagraph_capture_sizes
732 ) if self.compilation_config.use_cudagraph else set()
733
734 self.first_run_finished = False
@@ -746,11 +746,11 @@ class PiecewiseBackend:
746 # to_be_compiled_sizes tracks the remaining sizes to compile,
747 # and updates during the compilation process, so we need to copy it
748 self.to_be_compiled_sizes: Set[int] = self.compile_sizes.copy()
749 - for shape in self.compile_sizes.union(self.capture_sizes):
749 + for shape in self.compile_sizes.union(self.cudagraph_capture_sizes):
750 self.concrete_size_entries[shape] = ConcreteSizeEntry(
751 runtime_shape=shape,
752 need_to_compile=shape in self.compile_sizes,
753 - use_cudagraph=shape in self.capture_sizes,
753 + use_cudagraph=shape in self.cudagraph_capture_sizes,
754 )
755
756 def check_for_ending_compilation(self):
vllm/config.py
+37 -34
@@ -2711,10 +2711,11 @@ class CompilationConfig(BaseModel):
2711 - use_inductor: whether to use inductor compilation.
2712 - False: inductor compilation is not used. graph runs in eager.
2713 - True: inductor compilation is used. one graph for symbolic shape
2714 - is compiled. In addition, compile for cudagraph sizes that are
2715 - in candidate_compile_sizes, using configurations
2716 - in inductor_compile_config.
2717 - - candidate_compile_sizes: sizes to compile for inductor.
2714 + is compiled. In addition, compile for compile_sizes,
2715 + using configurations in inductor_compile_config.
2716 + - compile_sizes: sizes to compile for inductor. In addition
2717 + to integers, it also supports "cudagraph_capture_sizes" to
2718 + specify the sizes for cudagraph capture.
2719 - inductor_compile_config: additional configurations for inductor.
2720 - None: use default configurations.
2721 - inductor_passes: additional passes for inductor. It is a dictionary
@@ -2742,7 +2743,7 @@ class CompilationConfig(BaseModel):
2743 splitting_ops: List[str] = Field(default=None) # type: ignore
2744
2745 use_inductor: bool = True
2745 - candidate_compile_sizes: Optional[List[int]] = Field(default=None)
2746 + compile_sizes: Optional[List[Union[int, str]]] = Field(default=None)
2747 inductor_compile_config: Dict = Field(default_factory=dict)
2748 inductor_passes: Dict[str, str] = Field(default_factory=dict)
2749
@@ -2790,8 +2791,6 @@ class CompilationConfig(BaseModel):
2791 pass_config: PassConfig = Field(default_factory=PassConfig)
2792
2793 # not configurable, computed after init
2793 - compile_sizes: List[int] = PrivateAttr
2794 - capture_sizes: List[int] = PrivateAttr
2794 max_capture_size: int = PrivateAttr
2795 local_cache_dir: str = PrivateAttr # local cache dir for each rank
2796 # optimization:
@@ -2918,43 +2917,47 @@ class CompilationConfig(BaseModel):
2917 from vllm.compilation.backends import VllmBackend
2918 return VllmBackend(vllm_config)
2919
2921 - def init_with_cudagraph_sizes(self, sizes_to_specialize: List[int]):
2920 + def init_with_cudagraph_sizes(self,
2921 + cudagraph_capture_sizes: List[int]) -> None:
2922 """To complete the initialization of config,
2923 we need to know the cudagraph sizes."""
2924
2925 if self.cudagraph_capture_sizes is None:
2926 - self.capture_sizes = sizes_to_specialize
2926 + self.cudagraph_capture_sizes = cudagraph_capture_sizes
2927 else:
2928 - self.capture_sizes = self.cudagraph_capture_sizes
2928 + # de-duplicate the sizes provided by the config
2929 + self.cudagraph_capture_sizes = list(
2930 + set(self.cudagraph_capture_sizes))
2931 logger.info(("cudagraph sizes specified by model runner"
2932 " %s is overridden by config %s"),
2931 - sizes_to_specialize, self.cudagraph_capture_sizes)
2932 -
2933 - if self.candidate_compile_sizes is None:
2934 - self.candidate_compile_sizes = []
2935 - self.compile_sizes = [
2936 - x for x in self.candidate_compile_sizes if x in self.capture_sizes
2937 - ]
2938 - ignored_sizes = [
2939 - x for x in self.candidate_compile_sizes
2940 - if x not in self.capture_sizes
2941 - ]
2942 - if ignored_sizes:
2943 - logger.warning(("candidate_compile_sizes %s are ignored "
2944 - "because they are not cudagraph capture sizes."),
2945 - ignored_sizes)
2933 + cudagraph_capture_sizes, self.cudagraph_capture_sizes)
2934 +
2935 + computed_compile_sizes = []
2936 + if self.compile_sizes is not None:
2937 + # de-duplicate the sizes provided by the config
2938 + self.compile_sizes = list(set(self.compile_sizes))
2939 + for x in self.compile_sizes:
2940 + if isinstance(x, str):
2941 + assert x == "cudagraph_capture_sizes", \
2942 + "Unrecognized size type in compile_sizes, " \
2943 + f"expect 'cudagraph_capture_sizes', got {x}"
2944 + computed_compile_sizes.extend(self.cudagraph_capture_sizes)
2945 + else:
2946 + assert isinstance(x, int)
2947 + computed_compile_sizes.append(x)
2948 + self.compile_sizes = computed_compile_sizes # type: ignore
2949
2950 # sort to make sure cudagraph capture sizes are in descending order
2948 - self.capture_sizes.sort(reverse=True)
2949 - self.max_capture_size = self.capture_sizes[
2950 - 0] if self.capture_sizes else 0
2951 + self.cudagraph_capture_sizes.sort(reverse=True)
2952 + self.max_capture_size = self.cudagraph_capture_sizes[
2953 + 0] if self.cudagraph_capture_sizes else 0
2954
2955 # pre-compute the mapping from batch size to padded graph size
2956 self.bs_to_padded_graph_size = [
2957 0 for i in range(self.max_capture_size + 1)
2958 ]
2956 - for end, start in zip(self.capture_sizes,
2957 - self.capture_sizes[1:] + [0]):
2959 + for end, start in zip(self.cudagraph_capture_sizes,
2960 + self.cudagraph_capture_sizes[1:] + [0]):
2961 for bs in range(start, end):
2962 if bs == start:
2963 self.bs_to_padded_graph_size[bs] = start
@@ -3225,14 +3228,14 @@ class VllmConfig:
3228 However, if users specify the cudagraph capture sizes through
3229 compilation config, we will use the specified sizes instead.
3230
3228 - In the end, `vllm_config.compilation_config.capture_sizes` will be the
3229 - final sizes to capture cudagraph (in descending order).
3231 + In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`
3232 + will be the final sizes to capture cudagraph (in descending order).
3233
3234 During runtime, if batchsize is larger than
3232 - `vllm_config.compilation_config.capture_sizes`,
3235 + `vllm_config.compilation_config.cudagraph_capture_sizes`,
3236 no cudagraph will be used.
3237 If the batch size is no larger than
3235 - `vllm_config.compilation_config.capture_sizes`,
3238 + `vllm_config.compilation_config.cudagraph_capture_sizes`,
3239 we can quickly find the padded graph size for a given batch size by
3240 looking up `vllm_config.compilation_config.bs_to_padded_graph_size`.
3241 """
vllm/engine/metrics.py
+2 -1
@@ -120,7 +120,8 @@ class Metrics:
120 labelnames=labelnames)
121 buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
122 if not vllm_config.model_config.enforce_eager:
123 - buckets = vllm_config.compilation_config.capture_sizes.copy()
123 + buckets = vllm_config.compilation_config.\
124 + cudagraph_capture_sizes.copy()
125 buckets.sort()
126 self.histogram_iteration_tokens = self._histogram_cls(
127 name="vllm:iteration_tokens_total",
vllm/v1/worker/gpu_model_runner.py
+10 -8
@@ -1,6 +1,6 @@
1 import gc
2 import time
3 -from typing import TYPE_CHECKING, Dict, List, Tuple, cast
3 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast
4
5 import numpy as np
6 import torch
@@ -128,7 +128,8 @@ class GPUModelRunner:
128 # self.cudagraph_batch_sizes sorts in ascending order.
129 # The batch sizes in the config are in descending order.
130 self.cudagraph_batch_sizes = list(
131 - reversed(self.vllm_config.compilation_config.capture_sizes))
131 + reversed(
132 + self.vllm_config.compilation_config.cudagraph_capture_sizes))
133
134 # Cache the device properties.
135 self.device_properties = torch.cuda.get_device_properties(self.device)
@@ -834,10 +835,12 @@ class GPUModelRunner:
835 @torch.inference_mode()
836 def _dummy_run(
837 self,
837 - model: nn.Module,
838 num_tokens: int,
839 - kv_caches: List[torch.Tensor],
839 + kv_caches: Optional[List[torch.Tensor]] = None,
840 ) -> torch.Tensor:
841 + model = self.model
842 + if kv_caches is None:
843 + kv_caches = self.kv_caches
844 if self.is_multimodal_model:
845 input_ids = None
846 inputs_embeds = self.inputs_embeds[:num_tokens]
@@ -963,8 +966,7 @@ class GPUModelRunner:
966 self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))
967
968 # Trigger compilation for general shape.
966 - hidden_states = self._dummy_run(self.model, self.max_num_tokens,
967 - dummy_kv_caches)
969 + hidden_states = self._dummy_run(self.max_num_tokens, dummy_kv_caches)
970 logits = self.model.compute_logits(hidden_states, None)
971 logits = logits[:self.max_num_tokens]
972 # TODO(woosuk): Consider the memory usage of the sampler.
@@ -990,8 +992,8 @@ class GPUModelRunner:
992 for num_tokens in reversed(self.cudagraph_batch_sizes):
993 for _ in range(self.vllm_config.compilation_config.
994 cudagraph_num_of_warmups):
993 - self._dummy_run(self.model, num_tokens, self.kv_caches)
994 - self._dummy_run(self.model, num_tokens, self.kv_caches)
995 + self._dummy_run(num_tokens)
996 + self._dummy_run(num_tokens)
997
998 end_time = time.perf_counter()
999 end_free_gpu_memory = torch.cuda.mem_get_info()[0]
vllm/v1/worker/gpu_worker.py
+12
@@ -206,6 +206,18 @@ class Worker:
206 self.model_runner.initialize_kv_cache(kv_cache_config)
207
208 def compile_or_warm_up_model(self) -> None:
209 + # warm up sizes that are not in cudagraph capture sizes,
210 + # but users still want to compile for better performance,
211 + # e.g. for the max-num-batched token size in chunked prefill.
212 + warmup_sizes = self.vllm_config.compilation_config.compile_sizes.copy()
213 + if not self.model_config.enforce_eager:
214 + warmup_sizes = [
215 + x for x in warmup_sizes if x not in
216 + self.vllm_config.compilation_config.cudagraph_capture_sizes
217 + ]
218 + for size in sorted(warmup_sizes, reverse=True):
219 + logger.info("Compile and warming up model for size %d", size)
220 + self.model_runner._dummy_run(size)
221 if not self.model_config.enforce_eager:
222 self.model_runner.capture_model()
223 # Reset the seed to ensure that the random state is not affected by
vllm/worker/model_runner.py
+17 -10
@@ -1256,13 +1256,19 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1256
1257 @torch.inference_mode()
1258 def profile_run(self) -> None:
1259 + max_num_batched_tokens = \
1260 + self.scheduler_config.max_num_batched_tokens
1261 + max_num_seqs = self.scheduler_config.max_num_seqs
1262 + self._dummy_run(max_num_batched_tokens, max_num_seqs)
1263 +
1264 + def _dummy_run(self,
1265 + max_num_batched_tokens: int,
1266 + max_num_seqs: int = 1) -> None:
1267 with self.set_in_profile_run():
1268 # Enable top-k sampling to reflect the accurate memory usage.
1269 sampling_params = \
1270 SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
1263 - max_num_batched_tokens = \
1264 - self.scheduler_config.max_num_batched_tokens
1265 - max_num_seqs = self.scheduler_config.max_num_seqs
1271 +
1272 # This represents the maximum number of different requests
1273 # that will have unique loras, an therefore the max amount of memory
1274 # consumption create dummy lora request copies from the lora request
@@ -1491,13 +1497,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1497 for virtual_engine in range(
1498 self.parallel_config.pipeline_parallel_size):
1499 # Only rank 0 should print progress bar during capture
1494 - capture_sizes = (
1495 - tqdm(
1496 - self.vllm_config.compilation_config.capture_sizes,
1497 - desc="Capturing CUDA graph shapes",
1498 - ) if get_tensor_model_parallel_rank() == 0 else
1499 - self.vllm_config.compilation_config.capture_sizes)
1500 - for batch_size in capture_sizes:
1500 + cudagraph_capture_sizes = (tqdm(
1501 + self.vllm_config.compilation_config.
1502 + cudagraph_capture_sizes,
1503 + desc="Capturing CUDA graph shapes",
1504 + ) if get_tensor_model_parallel_rank() == 0 else
1505 + self.vllm_config.compilation_config.
1506 + cudagraph_capture_sizes)
1507 + for batch_size in cudagraph_capture_sizes:
1508 attn_metadata = (
1509 self.attn_state.graph_capture_get_metadata_for_batch(
1510 batch_size,
vllm/worker/worker.py
+12
@@ -323,6 +323,18 @@ class Worker(LocalOrDistributedWorkerBase):
323 self.gpu_cache)
324
325 def _warm_up_model(self) -> None:
326 + # warm up sizes that are not in cudagraph capture sizes,
327 + # but users still want to compile for better performance,
328 + # e.g. for the max-num-batched token size in chunked prefill.
329 + warmup_sizes = self.vllm_config.compilation_config.compile_sizes.copy()
330 + if not self.model_config.enforce_eager:
331 + warmup_sizes = [
332 + x for x in warmup_sizes if x not in
333 + self.vllm_config.compilation_config.cudagraph_capture_sizes
334 + ]
335 + for size in sorted(warmup_sizes, reverse=True):
336 + logger.info("Compile and warming up model for size %d", size)
337 + self.model_runner._dummy_run(size)
338 if not self.model_config.enforce_eager:
339 self.model_runner.capture_model(self.gpu_cache)
340 # Reset the seed to ensure that the random state is not affected by