@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5815c8413
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Fix non-cudagraph op name (#10166)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 8, 2024 at 10:23 UTC b5815c8413b4e09ba6ccd9c41ea3f9fb2d057aa8
1 file changed +1 -1
vllm/v1/worker/gpu_model_runner.py
+1 -1
@@ -411,7 +411,7 @@ class GPUModelRunner:
411 set_compilation_config(
412 CompilationConfig(
413 use_cudagraph=True,
414 - non_cudagraph_ops=["vllm.unified_flash_attention"],
414 + non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
415 use_inductor=True,
416 ))
417