A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Enable custom ops with piecewise CUDA graphs (#10228)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 11, 2024 at 11:58 UTC
9d5b4e4deaa3318df49419d325490730391efd75
1 file changed
+2
vllm/v1/worker/gpu_model_runner.py
+2
@@ -1,3 +1,4 @@
1
+import os
2
import time
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Dict, List, Optional, Set
@@ -405,6 +406,7 @@ class GPUModelRunner:
406
if self.use_cuda_graph:
407
# FIXME(woosuk): Currently, we do not use inductor to reduce the
408
# compilation time and any potential issues with the inductor.
409
+ os.environ["VLLM_CUSTOM_OPS"] = "all"
410
set_compilation_config(
411
CompilationConfig(
412
use_cudagraph=True,