A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[v1] reduce graph capture time for piecewise cudagraph (#10059)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 5, 2024 at 18:19 UTC
c4cacbaa7faf9d0d3b2aa26e5df496724e80cb05
1 file changed
+25
-11
vllm/compilation/backends.py
+25
-11
@@ -1,7 +1,9 @@
1
import copy
2
import dataclasses
3
import operator
4
+from contextlib import ExitStack
5
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
6
+from unittest.mock import patch
7
8
import torch
9
import torch.fx as fx
@@ -503,17 +505,29 @@ class PiecewiseBackend:
505
entry.input_addresses = input_addresses
506
cudagraph = torch.cuda.CUDAGraph()
507
506
- # mind-exploding: carefully manage the reference and memory.
507
- with torch.cuda.graph(cudagraph, pool=self.graph_pool):
508
- # `output` is managed by pytorch's cudagraph pool
509
- output = entry.runnable(*args)
510
- if self.is_last_graph:
511
- # by converting it to weak ref,
512
- # the original `output` will immediately be released
513
- # to save memory. It is only safe to do this for
514
- # the last graph, because the output of the last graph
515
- # will not be used by any other cuda graph.
516
- output = weak_ref_tensors(output)
508
+ with ExitStack() as stack:
509
+ if not self.is_first_graph:
510
+ # during every model forward, we will capture
511
+ # many pieces of cudagraphs (roughly one per layer).
512
+ # running gc again and again across layers will
513
+ # make the cudagraph capture very slow.
514
+ # therefore, we only run gc for the first graph,
515
+ # and disable gc for the rest of the graphs.
516
+ stack.enter_context(patch("gc.collect", lambda: None))
517
+ stack.enter_context(
518
+ patch("torch.cuda.empty_cache", lambda: None))
519
+
520
+ # mind-exploding: carefully manage the reference and memory.
521
+ with torch.cuda.graph(cudagraph, pool=self.graph_pool):
522
+ # `output` is managed by pytorch's cudagraph pool
523
+ output = entry.runnable(*args)
524
+ if self.is_last_graph:
525
+ # by converting it to weak ref,
526
+ # the original `output` will immediately be released
527
+ # to save memory. It is only safe to do this for
528
+ # the last graph, because the output of the last graph
529
+ # will not be used by any other cuda graph.
530
+ output = weak_ref_tensors(output)
531
532
# here we always use weak ref for the output
533
# to save memory