A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Fix non-cudagraph op name (#10166)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 8, 2024 at 10:23 UTC
b5815c8413b4e09ba6ccd9c41ea3f9fb2d057aa8
1 file changed
+1
-1
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -411,7 +411,7 @@ class GPUModelRunner:
411
set_compilation_config(
412
CompilationConfig(
413
use_cudagraph=True,
414
- non_cudagraph_ops=["vllm.unified_flash_attention"],
414
+ non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
415
use_inductor=True,
416
))
417