@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d7a4f2207b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Do not use inductor for piecewise CUDA graphs (#10225)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 11, 2024 at 11:05 UTC d7a4f2207bd0ff31cacf311a05266557d66e474e
1 file changed +3 -4
vllm/v1/worker/gpu_model_runner.py
+3 -4
@@ -404,15 +404,14 @@ class GPUModelRunner:
404
405 def load_model(self) -> None:
406 if self.use_cuda_graph:
407 - # FIXME(woosuk): Currently, the custom ops are not supported
408 - # in the piecewise compilation mode. We rely on TorchInductor
409 - # to optimize the model.
407 + # FIXME(woosuk): Currently, we do not use inductor to reduce the
408 + # compilation time and any potential issues with the inductor.
409 os.environ["VLLM_CUSTOM_OPS"] = "none"
410 set_compilation_config(
411 CompilationConfig(
412 use_cudagraph=True,
413 non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
415 - use_inductor=True,
414 + use_inductor=False,
415 ))
416
417 logger.info("Starting to load model %s...", self.model_config.model)