A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bug] Fix Attention when ignored in by quant_method (#14313)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Mar 6, 2025 at 17:18 UTC
6b2ef5cd17c5889557711af6f873282a363df042
1 file changed
+3
-1
vllm/attention/layer.py
+3
-1
@@ -11,6 +11,7 @@ from vllm.attention import AttentionType
11
from vllm.attention.selector import backend_name_to_enum, get_attn_backend
12
from vllm.config import CacheConfig, get_current_vllm_config
13
from vllm.forward_context import ForwardContext, get_forward_context
14
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig)
17
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
@@ -97,7 +98,8 @@ class Attention(nn.Module):
98
99
quant_method = quant_config.get_quant_method(
100
self, prefix=prefix) if quant_config else None
100
- if quant_method is not None:
101
+ if quant_method is not None and not isinstance(
102
+ quant_method, UnquantizedLinearMethod):
103
assert isinstance(quant_method, BaseKVCacheMethod)
104
# TODO (mgoin): kv cache dtype should be specified in the FP8
105
# checkpoint config and become the "auto" behavior