@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fe15729a2b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Use custom ops for piecewise CUDA graphs (#10227)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 11, 2024 at 11:26 UTC fe15729a2b77d760fcf99da76f15806c5eab33df
1 file changed -2
vllm/v1/worker/gpu_model_runner.py
-2
@@ -1,4 +1,3 @@
1 -import os
1 import time
2 from dataclasses import dataclass
3 from typing import TYPE_CHECKING, Dict, List, Optional, Set
@@ -406,7 +405,6 @@ class GPUModelRunner:
405 if self.use_cuda_graph:
406 # FIXME(woosuk): Currently, we do not use inductor to reduce the
407 # compilation time and any potential issues with the inductor.
409 - os.environ["VLLM_CUSTOM_OPS"] = "none"
408 set_compilation_config(
409 CompilationConfig(
410 use_cudagraph=True,