A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[7/N] torch.compile, reduce compilation time (#10460)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 20, 2024 at 11:20 UTC
0cd3d9717e38c7a122ed01fe2a8fddd8b37dff4b
5 files changed
+27
-16
tests/compile/piecewise/test_simple.py
+1
-1
@@ -79,7 +79,7 @@ def test_simple_piecewise_compile():
79
vllm_config = VllmConfig(compilation_config=CompilationConfig(
80
level=CompilationLevel.PIECEWISE,
81
use_cudagraph=True,
82
- non_cudagraph_ops=["silly.attention"],
82
+ splitting_ops=["silly.attention"],
83
cudagraph_copy_inputs=True,
84
))
85
with set_current_vllm_config(vllm_config):
tests/compile/piecewise/test_toy_llama.py
+2
-2
@@ -258,7 +258,7 @@ def run_model(llama_config,
258
use_cudagraph=True,
259
)
260
if split_attn:
261
- compilation_config.non_cudagraph_ops = ["silly.attention"]
261
+ compilation_config.splitting_ops = ["silly.attention"]
262
else:
263
compilation_config = CompilationConfig(
264
level=CompilationLevel.NO_COMPILATION, )
@@ -378,7 +378,7 @@ def benchmark():
378
compilation_config = CompilationConfig(
379
level=CompilationLevel.PIECEWISE,
380
use_cudagraph=True,
381
- non_cudagraph_ops=["silly.attention"],
381
+ splitting_ops=["silly.attention"],
382
)
383
else:
384
compilation_config = CompilationConfig(
vllm/compilation/backends.py
+1
-1
@@ -447,7 +447,7 @@ class VllmBackend:
447
self.add_passes_to_config()
448
449
self.split_gm, self.piecewise_graphs = split_graph(
450
- graph, self.compilation_configs.non_cudagraph_ops)
450
+ graph, self.compilation_configs.splitting_ops)
451
452
from torch._dynamo.utils import lazy_format_graph_code
453
logger.debug("%s", lazy_format_graph_code("before split", self.graph))
vllm/config.py
+10
-7
@@ -2089,13 +2089,15 @@ class CompilationConfig(BaseModel):
2089
- 'none,+op1,+op2' to enable only op1 and op2
2090
By default, all custom ops are enabled when running without Inductor
2091
and disabled when running with Inductor (compile_level >= Inductor).
2092
+ - splitting_ops: a list of ops to split the full graph into subgraphs, used in piecewise compilation.
2093
- CudaGraph capture:
2094
- use_cudagraph: whether to use cudagraph inside compilation.
2095
- False: cudagraph inside compilation is not used.
2096
- True: cudagraph inside compilation is used. It requires
2096
- that all input buffers have fixed addresses.
2097
- Note that this is orthogonal to the cudagraph capture out
2098
- side of compilation.
2097
+ that all input buffers have fixed addresses, and all
2098
+ splitting ops write their outputs to input buffers.
2099
+ Note that this is orthogonal to the cudagraph capture logic
2100
+ outside of compilation.
2101
TODO: move outside cudagraph logic into compilation.
2102
torch.compile will handle cudagraph capture logic in the future.
2103
- cudagraph_capture_sizes: sizes to capture cudagraph.
@@ -2149,6 +2151,11 @@ class CompilationConfig(BaseModel):
2151
level: int = 0
2152
backend: str = ""
2153
custom_ops: List[str] = Field(default_factory=list)
2154
+ splitting_ops: List[str] = Field(default_factory=lambda: [
2155
+ "vllm.unified_flash_attention",
2156
+ "vllm.unified_flash_infer",
2157
+ "vllm.unified_v1_flash_attention",
2158
+ ])
2159
2160
use_inductor: bool = True
2161
inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
@@ -2157,7 +2164,6 @@ class CompilationConfig(BaseModel):
2164
inductor_passes: Dict[str, str] = Field(default_factory=dict)
2165
2166
use_cudagraph: bool = False
2160
- non_cudagraph_ops: List[str] = Field(default_factory=list)
2167
cudagraph_num_of_warmups: int = 0
2168
cudagraph_capture_sizes: Optional[List[int]] = None
2169
cudagraph_copy_inputs: bool = False
@@ -2348,9 +2354,6 @@ class VllmConfig:
2354
# and avoid any potential issues with the inductor.
2355
self.compilation_config.custom_ops = ["none"]
2356
self.compilation_config.use_cudagraph = True
2351
- self.compilation_config.non_cudagraph_ops = [
2352
- "vllm.unified_v1_flash_attention"
2353
- ]
2357
self.compilation_config.use_inductor = True
2358
self.compilation_config.enable_fusion = False
2359
vllm/worker/worker.py
+13
-5
@@ -1,6 +1,7 @@
1
"""A GPU worker class."""
2
import gc
3
import os
4
+import time
5
from typing import Dict, List, Optional, Set, Tuple, Type, Union
6
7
import torch
@@ -189,6 +190,7 @@ class Worker(LocalOrDistributedWorkerBase):
190
torch.cuda.reset_peak_memory_stats()
191
192
free_memory_pre_profile, total_gpu_memory = torch.cuda.mem_get_info()
193
+ start_time = time.time()
194
195
# Execute a forward pass with dummy inputs to profile the memory usage
196
# of the model.
@@ -229,12 +231,18 @@ class Worker(LocalOrDistributedWorkerBase):
231
num_gpu_blocks = max(num_gpu_blocks, 0)
232
num_cpu_blocks = max(num_cpu_blocks, 0)
233
234
+ end_time = time.time()
235
logger.info(
233
- "Memory profiling results: total_gpu_memory=%.2fGiB"
234
- " initial_memory_usage=%.2fGiB peak_torch_memory=%.2fGiB"
235
- " memory_usage_post_profile=%.2fGiB"
236
- " non_torch_memory=%.2fGiB kv_cache_size=%.2fGiB"
237
- " gpu_memory_utilization=%.2f", total_gpu_memory / (1024**3),
236
+ "Memory profiling results: "
237
+ "duration=%.2f seconds, "
238
+ "total_gpu_memory=%.2fGiB, "
239
+ "initial_memory_usage=%.2fGiB, "
240
+ "peak_torch_memory=%.2fGiB, "
241
+ "memory_usage_post_profile=%.2fGiB, "
242
+ "non_torch_memory=%.2fGiB, "
243
+ "kv_cache_size=%.2fGiB, "
244
+ "gpu_memory_utilization=%.2f.", end_time - start_time,
245
+ total_gpu_memory / (1024**3),
246
(total_gpu_memory - free_memory_pre_profile) / (1024**3),
247
(peak_memory - non_torch_allocations) / (1024**3),
248
total_allocated_bytes / (1024**3),