A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] ChunkedLocalAttention is currently not CG compatible (#26034)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Oct 1, 2025 at 19:28 UTC
4134312b35465748828c29ee845a3812b7310dfd
1 file changed
+5
-3
vllm/attention/layers/chunked_local_attention.py
+5
-3
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import functools
4
-from typing import List, Optional
4
+from typing import ClassVar, List, Optional
5
6
import torch
7
@@ -12,8 +12,8 @@ from vllm.attention.selector import get_attn_backend
12
from vllm.config import CacheConfig
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.v1.attention.backends.utils import (
15
- CommonAttentionMetadata, make_local_attention_virtual_batches,
16
- subclass_attention_backend)
15
+ AttentionCGSupport, CommonAttentionMetadata,
16
+ make_local_attention_virtual_batches, subclass_attention_backend)
17
18
from ..layer import Attention
19
@@ -29,6 +29,8 @@ def create_chunked_local_attention_backend(
29
underlying_builder = underlying_attn_backend.get_builder_cls()
30
31
class ChunkedLocalAttentionBuilder(underlying_builder): # type: ignore
32
+ cudagraph_support: ClassVar[AttentionCGSupport] = \
33
+ AttentionCGSupport.NEVER
34
35
def build(self,
36
common_prefix_len: int,