@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f55e05713
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] Enable Inductor when using piecewise CUDA graphs (#10268)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 12, 2024 at 13:39 UTC 1f55e0571350f3dd2c04638e13e52d8ed557d93e
1 file changed +7 -4
vllm/v1/worker/gpu_model_runner.py
+7 -4
@@ -404,14 +404,17 @@ class GPUModelRunner:
404
405 def load_model(self) -> None:
406 if self.use_cuda_graph:
407 - # FIXME(woosuk): Currently, we do not use inductor to reduce the
408 - # compilation time and any potential issues with the inductor.
409 - os.environ["VLLM_CUSTOM_OPS"] = "all"
407 + # NOTE(woosuk): Currently, we use inductor because the piecewise
408 + # CUDA graphs do not work properly with the custom CUDA kernels.
409 + # FIXME(woosuk): Disable inductor to reduce the compilation time
410 + # and avoid any potential issues with the inductor.
411 + os.environ["VLLM_CUSTOM_OPS"] = "none"
412 set_compilation_config(
413 CompilationConfig(
414 use_cudagraph=True,
415 non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
414 - use_inductor=False,
416 + use_inductor=True,
417 + enable_fusion=False,
418 ))
419
420 logger.info("Starting to load model %s...", self.model_config.model)