A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] Enable Inductor when using piecewise CUDA graphs (#10268)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 12, 2024 at 13:39 UTC
1f55e0571350f3dd2c04638e13e52d8ed557d93e
1 file changed
+7
-4
vllm/v1/worker/gpu_model_runner.py
+7
-4
@@ -404,14 +404,17 @@ class GPUModelRunner:
404
405
def load_model(self) -> None:
406
if self.use_cuda_graph:
407
- # FIXME(woosuk): Currently, we do not use inductor to reduce the
408
- # compilation time and any potential issues with the inductor.
409
- os.environ["VLLM_CUSTOM_OPS"] = "all"
407
+ # NOTE(woosuk): Currently, we use inductor because the piecewise
408
+ # CUDA graphs do not work properly with the custom CUDA kernels.
409
+ # FIXME(woosuk): Disable inductor to reduce the compilation time
410
+ # and avoid any potential issues with the inductor.
411
+ os.environ["VLLM_CUSTOM_OPS"] = "none"
412
set_compilation_config(
413
CompilationConfig(
414
use_cudagraph=True,
415
non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
414
- use_inductor=False,
416
+ use_inductor=True,
417
+ enable_fusion=False,
418
))
419
420
logger.info("Starting to load model %s...", self.model_config.model)