@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4cacbaa7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[v1] reduce graph capture time for piecewise cudagraph (#10059)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 5, 2024 at 18:19 UTC c4cacbaa7faf9d0d3b2aa26e5df496724e80cb05
1 file changed +25 -11
vllm/compilation/backends.py
+25 -11
@@ -1,7 +1,9 @@
1 import copy
2 import dataclasses
3 import operator
4 +from contextlib import ExitStack
5 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
6 +from unittest.mock import patch
7
8 import torch
9 import torch.fx as fx
@@ -503,17 +505,29 @@ class PiecewiseBackend:
505 entry.input_addresses = input_addresses
506 cudagraph = torch.cuda.CUDAGraph()
507
506 - # mind-exploding: carefully manage the reference and memory.
507 - with torch.cuda.graph(cudagraph, pool=self.graph_pool):
508 - # `output` is managed by pytorch's cudagraph pool
509 - output = entry.runnable(*args)
510 - if self.is_last_graph:
511 - # by converting it to weak ref,
512 - # the original `output` will immediately be released
513 - # to save memory. It is only safe to do this for
514 - # the last graph, because the output of the last graph
515 - # will not be used by any other cuda graph.
516 - output = weak_ref_tensors(output)
508 + with ExitStack() as stack:
509 + if not self.is_first_graph:
510 + # during every model forward, we will capture
511 + # many pieces of cudagraphs (roughly one per layer).
512 + # running gc again and again across layers will
513 + # make the cudagraph capture very slow.
514 + # therefore, we only run gc for the first graph,
515 + # and disable gc for the rest of the graphs.
516 + stack.enter_context(patch("gc.collect", lambda: None))
517 + stack.enter_context(
518 + patch("torch.cuda.empty_cache", lambda: None))
519 +
520 + # mind-exploding: carefully manage the reference and memory.
521 + with torch.cuda.graph(cudagraph, pool=self.graph_pool):
522 + # `output` is managed by pytorch's cudagraph pool
523 + output = entry.runnable(*args)
524 + if self.is_last_graph:
525 + # by converting it to weak ref,
526 + # the original `output` will immediately be released
527 + # to save memory. It is only safe to do this for
528 + # the last graph, because the output of the last graph
529 + # will not be used by any other cuda graph.
530 + output = weak_ref_tensors(output)
531
532 # here we always use weak ref for the output
533 # to save memory