@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0cd3d9717e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[7/N] torch.compile, reduce compilation time (#10460)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 20, 2024 at 11:20 UTC 0cd3d9717e38c7a122ed01fe2a8fddd8b37dff4b
5 files changed +27 -16
tests/compile/piecewise/test_simple.py
+1 -1
@@ -79,7 +79,7 @@ def test_simple_piecewise_compile():
79 vllm_config = VllmConfig(compilation_config=CompilationConfig(
80 level=CompilationLevel.PIECEWISE,
81 use_cudagraph=True,
82 - non_cudagraph_ops=["silly.attention"],
82 + splitting_ops=["silly.attention"],
83 cudagraph_copy_inputs=True,
84 ))
85 with set_current_vllm_config(vllm_config):
tests/compile/piecewise/test_toy_llama.py
+2 -2
@@ -258,7 +258,7 @@ def run_model(llama_config,
258 use_cudagraph=True,
259 )
260 if split_attn:
261 - compilation_config.non_cudagraph_ops = ["silly.attention"]
261 + compilation_config.splitting_ops = ["silly.attention"]
262 else:
263 compilation_config = CompilationConfig(
264 level=CompilationLevel.NO_COMPILATION, )
@@ -378,7 +378,7 @@ def benchmark():
378 compilation_config = CompilationConfig(
379 level=CompilationLevel.PIECEWISE,
380 use_cudagraph=True,
381 - non_cudagraph_ops=["silly.attention"],
381 + splitting_ops=["silly.attention"],
382 )
383 else:
384 compilation_config = CompilationConfig(
vllm/compilation/backends.py
+1 -1
@@ -447,7 +447,7 @@ class VllmBackend:
447 self.add_passes_to_config()
448
449 self.split_gm, self.piecewise_graphs = split_graph(
450 - graph, self.compilation_configs.non_cudagraph_ops)
450 + graph, self.compilation_configs.splitting_ops)
451
452 from torch._dynamo.utils import lazy_format_graph_code
453 logger.debug("%s", lazy_format_graph_code("before split", self.graph))
vllm/config.py
+10 -7
@@ -2089,13 +2089,15 @@ class CompilationConfig(BaseModel):
2089 - 'none,+op1,+op2' to enable only op1 and op2
2090 By default, all custom ops are enabled when running without Inductor
2091 and disabled when running with Inductor (compile_level >= Inductor).
2092 + - splitting_ops: a list of ops to split the full graph into subgraphs, used in piecewise compilation.
2093 - CudaGraph capture:
2094 - use_cudagraph: whether to use cudagraph inside compilation.
2095 - False: cudagraph inside compilation is not used.
2096 - True: cudagraph inside compilation is used. It requires
2096 - that all input buffers have fixed addresses.
2097 - Note that this is orthogonal to the cudagraph capture out
2098 - side of compilation.
2097 + that all input buffers have fixed addresses, and all
2098 + splitting ops write their outputs to input buffers.
2099 + Note that this is orthogonal to the cudagraph capture logic
2100 + outside of compilation.
2101 TODO: move outside cudagraph logic into compilation.
2102 torch.compile will handle cudagraph capture logic in the future.
2103 - cudagraph_capture_sizes: sizes to capture cudagraph.
@@ -2149,6 +2151,11 @@ class CompilationConfig(BaseModel):
2151 level: int = 0
2152 backend: str = ""
2153 custom_ops: List[str] = Field(default_factory=list)
2154 + splitting_ops: List[str] = Field(default_factory=lambda: [
2155 + "vllm.unified_flash_attention",
2156 + "vllm.unified_flash_infer",
2157 + "vllm.unified_v1_flash_attention",
2158 + ])
2159
2160 use_inductor: bool = True
2161 inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
@@ -2157,7 +2164,6 @@ class CompilationConfig(BaseModel):
2164 inductor_passes: Dict[str, str] = Field(default_factory=dict)
2165
2166 use_cudagraph: bool = False
2160 - non_cudagraph_ops: List[str] = Field(default_factory=list)
2167 cudagraph_num_of_warmups: int = 0
2168 cudagraph_capture_sizes: Optional[List[int]] = None
2169 cudagraph_copy_inputs: bool = False
@@ -2348,9 +2354,6 @@ class VllmConfig:
2354 # and avoid any potential issues with the inductor.
2355 self.compilation_config.custom_ops = ["none"]
2356 self.compilation_config.use_cudagraph = True
2351 - self.compilation_config.non_cudagraph_ops = [
2352 - "vllm.unified_v1_flash_attention"
2353 - ]
2357 self.compilation_config.use_inductor = True
2358 self.compilation_config.enable_fusion = False
2359
vllm/worker/worker.py
+13 -5
@@ -1,6 +1,7 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 +import time
5 from typing import Dict, List, Optional, Set, Tuple, Type, Union
6
7 import torch
@@ -189,6 +190,7 @@ class Worker(LocalOrDistributedWorkerBase):
190 torch.cuda.reset_peak_memory_stats()
191
192 free_memory_pre_profile, total_gpu_memory = torch.cuda.mem_get_info()
193 + start_time = time.time()
194
195 # Execute a forward pass with dummy inputs to profile the memory usage
196 # of the model.
@@ -229,12 +231,18 @@ class Worker(LocalOrDistributedWorkerBase):
231 num_gpu_blocks = max(num_gpu_blocks, 0)
232 num_cpu_blocks = max(num_cpu_blocks, 0)
233
234 + end_time = time.time()
235 logger.info(
233 - "Memory profiling results: total_gpu_memory=%.2fGiB"
234 - " initial_memory_usage=%.2fGiB peak_torch_memory=%.2fGiB"
235 - " memory_usage_post_profile=%.2fGiB"
236 - " non_torch_memory=%.2fGiB kv_cache_size=%.2fGiB"
237 - " gpu_memory_utilization=%.2f", total_gpu_memory / (1024**3),
236 + "Memory profiling results: "
237 + "duration=%.2f seconds, "
238 + "total_gpu_memory=%.2fGiB, "
239 + "initial_memory_usage=%.2fGiB, "
240 + "peak_torch_memory=%.2fGiB, "
241 + "memory_usage_post_profile=%.2fGiB, "
242 + "non_torch_memory=%.2fGiB, "
243 + "kv_cache_size=%.2fGiB, "
244 + "gpu_memory_utilization=%.2f.", end_time - start_time,
245 + total_gpu_memory / (1024**3),
246 (total_gpu_memory - free_memory_pre_profile) / (1024**3),
247 (peak_memory - non_torch_allocations) / (1024**3),
248 total_allocated_bytes / (1024**3),