A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Only add `Attention.kv_scale` if kv cache quantization is enabled (#5936)
Michael Goin committed
Jun 28, 2024 at 17:12 UTC
4bf35ed9ae3cb2ee2efd8f5b9ced620ca9836240
1 file changed
+14
-9
vllm/attention/layer.py
+14
-9
@@ -9,6 +9,7 @@ from vllm.attention.selector import get_attn_backend
9
from vllm.config import CacheConfig
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig)
12
+from vllm.model_executor.layers.quantization.fp8 import Fp8KVCacheMethod
13
14
15
class Attention(nn.Module):
@@ -56,15 +57,19 @@ class Attention(nn.Module):
57
quant_method = quant_config.get_quant_method(
58
self) if quant_config else None
59
if quant_method is not None:
59
- if self.kv_cache_dtype == "fp8_e5m2":
60
- raise ValueError("fp8_e5m2 kv-cache is not supported with "
61
- "fp8 checkpoints.")
62
- # When FP8 quantization is enabled, we make a parameter
63
- # "kv_scale" so that it can be loaded from FP8 checkpoint.
64
- # The kv_scale will then be converted back
65
- # to self._kv_scale in a native float32 value after weight loading.
66
- self.quant_method = quant_method
67
- self.quant_method.create_weights(self)
60
+ assert isinstance(quant_method, Fp8KVCacheMethod)
61
+ # TODO (mgoin): kv cache dtype should be specified in the FP8
62
+ # checkpoint config and become the "auto" behavior
63
+ if "fp8" in self.kv_cache_dtype:
64
+ if self.kv_cache_dtype == "fp8_e5m2":
65
+ raise ValueError("fp8_e5m2 kv-cache is not supported with "
66
+ "fp8 checkpoints.")
67
+ # When FP8 quantization is enabled, we make a parameter
68
+ # "kv_scale" so that it can be loaded from FP8 checkpoint.
69
+ # The kv_scale will then be converted back to self._kv_scale
70
+ # in a native float32 value after weight loading.
71
+ self.quant_method = quant_method
72
+ self.quant_method.create_weights(self)
73
74
# During model initialization, the default dtype is set as the model
75
# weight and activation dtype.