A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Do not use inductor for piecewise CUDA graphs (#10225)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 11, 2024 at 11:05 UTC
d7a4f2207bd0ff31cacf311a05266557d66e474e
1 file changed
+3
-4
vllm/v1/worker/gpu_model_runner.py
+3
-4
@@ -404,15 +404,14 @@ class GPUModelRunner:
404
405
def load_model(self) -> None:
406
if self.use_cuda_graph:
407
- # FIXME(woosuk): Currently, the custom ops are not supported
408
- # in the piecewise compilation mode. We rely on TorchInductor
409
- # to optimize the model.
407
+ # FIXME(woosuk): Currently, we do not use inductor to reduce the
408
+ # compilation time and any potential issues with the inductor.
409
os.environ["VLLM_CUSTOM_OPS"] = "none"
410
set_compilation_config(
411
CompilationConfig(
412
use_cudagraph=True,
413
non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
415
- use_inductor=True,
414
+ use_inductor=False,
415
))
416
417
logger.info("Starting to load model %s...", self.model_config.model)