@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d5b4e4dea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Enable custom ops with piecewise CUDA graphs (#10228)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 11, 2024 at 11:58 UTC 9d5b4e4deaa3318df49419d325490730391efd75
1 file changed +2
vllm/v1/worker/gpu_model_runner.py
+2
@@ -1,3 +1,4 @@
1 +import os
2 import time
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING, Dict, List, Optional, Set
@@ -405,6 +406,7 @@ class GPUModelRunner:
406 if self.use_cuda_graph:
407 # FIXME(woosuk): Currently, we do not use inductor to reduce the
408 # compilation time and any potential issues with the inductor.
409 + os.environ["VLLM_CUSTOM_OPS"] = "all"
410 set_compilation_config(
411 CompilationConfig(
412 use_cudagraph=True,