A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Load FP8 kv-cache scaling factors from checkpoints (#4893)
The 2nd PR for #4532. This PR supports loading FP8 kv-cache scaling factors from a FP8 checkpoint (with .kv_scale parameter).
Cody Yu committed
May 22, 2024 at 13:28 UTC
a3a73ab0696b6692f3eecf80271a01fa97bd001d
40 files changed
+284
-158
benchmarks/benchmark_latency.py
+6
-8
@@ -153,15 +153,13 @@ if __name__ == '__main__':
153
action='store_true',
154
help='enforce eager mode and disable CUDA graph')
155
parser.add_argument(
156
- "--kv-cache-dtype",
156
+ '--kv-cache-dtype',
157
type=str,
158
- choices=['auto', 'fp8'],
159
- default='auto',
160
- help=
161
- 'Data type for kv cache storage. If "auto", will use model data type. '
162
- 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
163
- 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
164
- 'instead supported for common inference criteria.')
158
+ choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
159
+ default="auto",
160
+ help='Data type for kv cache storage. If "auto", will use model '
161
+ 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
162
+ 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
163
parser.add_argument(
164
'--quantization-param-path',
165
type=str,
benchmarks/benchmark_throughput.py
+5
-7
@@ -323,15 +323,13 @@ if __name__ == "__main__":
323
action="store_true",
324
help="enforce eager execution")
325
parser.add_argument(
326
- "--kv-cache-dtype",
326
+ '--kv-cache-dtype',
327
type=str,
328
- choices=["auto", "fp8"],
328
+ choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
329
default="auto",
330
- help=
331
- 'Data type for kv cache storage. If "auto", will use model data type. '
332
- 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
333
- 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
334
- 'common inference criteria.')
330
+ help='Data type for kv cache storage. If "auto", will use model '
331
+ 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
332
+ 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
333
parser.add_argument(
334
'--quantization-param-path',
335
type=str,
benchmarks/kernels/benchmark_paged_attention.py
+4
-6
@@ -183,13 +183,11 @@ if __name__ == '__main__':
183
parser.add_argument(
184
"--kv-cache-dtype",
185
type=str,
186
- choices=["auto", "fp8"],
186
+ choices=["auto", "fp8", "fp8_e5m2", "fp8_e4m3"],
187
default="auto",
188
- help=
189
- 'Data type for kv cache storage. If "auto", will use model data type. '
190
- 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
191
- 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
192
- 'common inference criteria.')
188
+ help="Data type for kv cache storage. If 'auto', will use model "
189
+ "data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. "
190
+ "ROCm (AMD GPU) supports fp8 (=fp8_e4m3)")
191
args = parser.parse_args()
192
print(args)
193
tests/models/test_fp8.py
+52
-28
@@ -16,31 +16,55 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
MAX_MODEL_LEN = 1024
17
18
MODELS = [
19
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8",
19
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV",
20
"meta-llama/Meta-Llama-3-8B-Instruct",
21
]
22
23
EXPECTED_STRS_MAP = {
24
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8": [
25
- 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
26
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
27
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
28
- 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
29
- 'Zeta-5, a highly advanced robot designed for menial labor, whirred to a',
30
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
31
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
32
- 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, guri o',
33
- ],
34
- "meta-llama/Meta-Llama-3-8B-Instruct": [
35
- 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
36
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
37
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
38
- 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
39
- 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
40
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
41
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
42
- 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
43
- ],
24
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV": {
25
+ "auto": [
26
+ 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
27
+ 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
28
+ 'Artificial intelligence (AI) and human intelligence (HI) process information in distinct ways, with both',
29
+ 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
30
+ 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
31
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
32
+ 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
33
+ 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, nemuri no'
34
+ ],
35
+ "fp8": [
36
+ 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
37
+ 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
38
+ 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
39
+ 'A neural network is a complex system made up of several basic components that work together to enable it to',
40
+ 'Zeta-5, a highly advanced robot designed for menial labor, had never experienced anything like',
41
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
42
+ 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
43
+ 'Here are the translations:\n\n**Japanese:** (Haya kotori wa mushi o tsuk'
44
+ ]
45
+ },
46
+ "meta-llama/Meta-Llama-3-8B-Instruct": {
47
+ "auto": [
48
+ 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
49
+ 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
50
+ 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
51
+ 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
52
+ 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
53
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
54
+ 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
55
+ 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
56
+ ],
57
+ "fp8": [
58
+ 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
59
+ 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
60
+ 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
61
+ 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
62
+ 'In the year 2154, robotics engineer Dr. Rachel Kim had spent years perfecting her latest',
63
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
64
+ 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
65
+ 'Here are the translations:\n\n**Japanese:** (Haya tori, mushi o tsukamu'
66
+ ]
67
+ },
68
}
69
70
capability = torch.cuda.get_device_capability()
@@ -52,14 +76,14 @@ fp8_not_supported = (capability <
76
@pytest.mark.skipif(fp8_not_supported,
77
reason="fp8 is not supported on this GPU type.")
78
@pytest.mark.parametrize("model_name", MODELS)
55
-def test_models(
56
- example_prompts,
57
- model_name,
58
-) -> None:
79
+@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
80
+def test_models(example_prompts, model_name, kv_cache_dtype) -> None:
81
model = LLM(model=model_name,
82
max_model_len=MAX_MODEL_LEN,
83
+ trust_remote_code=True,
84
enforce_eager=True,
62
- quantization="fp8")
85
+ quantization="fp8",
86
+ kv_cache_dtype=kv_cache_dtype)
87
88
tokenizer = AutoTokenizer.from_pretrained(model_name)
89
formatted_prompts = [
@@ -81,8 +105,8 @@ def test_models(
105
generations.append(outputs[0].outputs[0].text)
106
del model
107
84
- print(generations)
85
- expected_strs = EXPECTED_STRS_MAP[model_name]
108
+ print(model_name, kv_cache_dtype, generations)
109
+ expected_strs = EXPECTED_STRS_MAP[model_name][kv_cache_dtype]
110
for i in range(len(example_prompts)):
111
generated_str = generations[i]
112
expected_str = expected_strs[i]
vllm/attention/layer.py
+25
-2
@@ -7,6 +7,8 @@ import torch.nn as nn
7
from vllm.attention.backends.abstract import AttentionMetadata
8
from vllm.attention.selector import get_attn_backend
9
from vllm.config import CacheConfig
10
+from vllm.model_executor.layers.quantization.base_config import (
11
+ QuantizationConfig)
12
13
14
class Attention(nn.Module):
@@ -30,6 +32,7 @@ class Attention(nn.Module):
32
alibi_slopes: Optional[List[float]] = None,
33
sliding_window: Optional[int] = None,
34
cache_config: Optional[CacheConfig] = None,
35
+ quant_config: Optional[QuantizationConfig] = None,
36
) -> None:
37
super().__init__()
38
if cache_config is not None:
@@ -40,6 +43,27 @@ class Attention(nn.Module):
43
block_size = 16
44
if num_kv_heads is None:
45
num_kv_heads = num_heads
46
+
47
+ # The default kv_scale is set to 1.0. This is ignored
48
+ # when kv-cache is not fp8, and should be used with
49
+ # kv-cache in fp8_e5m2. For kv-cache in fp8_e4m3, we
50
+ # expect the pre-quantized kv_scale to be loaded along
51
+ # with the model weights.
52
+ self.kv_cache_dtype = kv_cache_dtype
53
+ self._kv_scale = 1.0
54
+ quant_method = quant_config.get_quant_method(
55
+ self) if quant_config else None
56
+ if quant_method is not None:
57
+ if self.kv_cache_dtype == "fp8_e5m2":
58
+ raise ValueError("fp8_e5m2 kv-cache is not supported with "
59
+ "fp8 checkpoints.")
60
+ # When FP8 quantization is enabled, we make a parameter
61
+ # "kv_scale" so that it can be loaded from FP8 checkpoint.
62
+ # The kv_scale will then be converted back
63
+ # to self._kv_scale in a native float32 value after weight loading.
64
+ self.quant_method = quant_method
65
+ self.quant_method.create_weights(self)
66
+
67
# During model initialization, the default dtype is set as the model
68
# weight and activation dtype.
69
dtype = torch.get_default_dtype()
@@ -57,10 +81,9 @@ class Attention(nn.Module):
81
value: torch.Tensor,
82
kv_cache: Optional[torch.Tensor],
83
attn_metadata: AttentionMetadata,
60
- kv_scale: float = 1.0,
84
) -> torch.Tensor:
85
return self.impl.forward(query, key, value, kv_cache, attn_metadata,
63
- kv_scale)
86
+ self._kv_scale)
87
88
def extra_repr(self) -> str:
89
s = f"head_size={self.impl.head_size}" # type: ignore
vllm/config.py
+3
-5
@@ -355,14 +355,12 @@ class CacheConfig:
355
def _verify_cache_dtype(self) -> None:
356
if self.cache_dtype == "auto":
357
pass
358
- elif self.cache_dtype == "fp8":
358
+ elif self.cache_dtype in ("fp8", "fp8_e4m3", "fp8_e5m2"):
359
logger.info(
360
"Using fp8 data type to store kv cache. It reduces the GPU "
361
"memory footprint and boosts the performance. "
362
- "But it may cause slight accuracy drop without scaling "
363
- "factors. FP8_E5M2 (without scaling) is only supported on "
364
- "cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 "
365
- "is instead supported for common inference criteria.")
362
+ "Meanwhile, it may cause accuracy drop without a proper "
363
+ "scaling factor")
364
else:
365
raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
366
vllm/engine/arg_utils.py
+3
-4
@@ -191,12 +191,11 @@ class EngineArgs:
191
parser.add_argument(
192
'--kv-cache-dtype',
193
type=str,
194
- choices=['auto', 'fp8'],
194
+ choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
195
default=EngineArgs.kv_cache_dtype,
196
help='Data type for kv cache storage. If "auto", will use model '
197
- 'data type. FP8_E5M2 (without scaling) is only supported on cuda '
198
- 'version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
199
- 'supported for common inference criteria.')
197
+ 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
198
+ 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
199
parser.add_argument(
200
'--quantization-param-path',
201
type=nullable_str,
vllm/model_executor/layers/quantization/fp8.py
+45
-2
@@ -8,8 +8,9 @@ from vllm import _custom_ops as ops
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10
from vllm.model_executor.layers.quantization.base_config import (
11
- QuantizationConfig)
11
+ QuantizationConfig, QuantizeMethodBase)
12
from vllm.model_executor.utils import set_weight_attrs
13
+from vllm.utils import print_warning_once
14
15
ACTIVATION_SCHEMES = ["static", "dynamic"]
16
@@ -58,9 +59,13 @@ class Fp8Config(QuantizationConfig):
59
activation_scheme=activation_scheme)
60
61
def get_quant_method(
61
- self, layer: torch.nn.Module) -> Optional["Fp8LinearMethod"]:
62
+ self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
63
+ from vllm.attention.layer import Attention # Avoid circular import
64
+
65
if isinstance(layer, LinearBase):
66
return Fp8LinearMethod(self)
67
+ if isinstance(layer, Attention):
68
+ return Fp8KVCacheMethod(self)
69
return None
70
71
def get_scaled_act_names(self) -> List[str]:
@@ -251,6 +256,44 @@ class Fp8LinearMethod(LinearMethodBase):
256
return torch.narrow(output, 0, 0, x.shape[0])
257
258
259
+class Fp8KVCacheMethod(QuantizeMethodBase):
260
+ """Supports loading kv-cache scaling factors from FP8 checkpoints.
261
+ """
262
+
263
+ def __init__(self, quant_config: Fp8Config):
264
+ self.quant_config = quant_config
265
+
266
+ def create_weights(self, layer: torch.nn.Module):
267
+ """Create "weight" (aka kv_scale) for an attention layer.
268
+
269
+ Args:
270
+ layer: The layer that is using the QuantizeMethodBase factory.
271
+ """
272
+ # Initialize the KV cache scale to 1.0 as the default value.
273
+ # If the kv_scale appears in the checkpoint, it will be
274
+ # overwritten when loading weights.
275
+ layer.kv_scale = Parameter(torch.tensor(1.0), requires_grad=False)
276
+
277
+ def apply(self, layer: torch.nn.Module) -> torch.Tensor:
278
+ raise RuntimeError("Fp8KVCacheMethod.apply should not be called.")
279
+
280
+ def process_weights_after_loading(self, layer: Module) -> None:
281
+ # If the kv-cache dtype is auto, we enforce the kv-scale to be 1.0
282
+ # regardless whether the kv-scale is available in the checkpoint.
283
+ if layer.kv_cache_dtype != "auto":
284
+ kv_scale = layer.kv_scale.to("cpu").tolist()
285
+ if not isinstance(kv_scale, float):
286
+ raise ValueError("Only support per-tensor scaling factor "
287
+ "for fp8 KV cache")
288
+ layer._kv_scale = kv_scale
289
+ if layer._kv_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
290
+ print_warning_once(
291
+ "Using KV cache scaling factor 1.0 for fp8_e4m3. This may "
292
+ "cause accuracy issues. Please make sure kv-cache scaling "
293
+ "factor is available in the fp8 checkpoint.")
294
+ del layer.kv_scale
295
+
296
+
297
def all_close_1d(x: torch.Tensor) -> bool:
298
assert len(x.shape) == 1
299
return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/models/arctic.py
+2
-1
@@ -268,7 +268,8 @@ class ArcticAttention(nn.Module):
268
self.head_dim,
269
self.scaling,
270
num_kv_heads=self.num_kv_heads,
271
- cache_config=cache_config)
271
+ cache_config=cache_config,
272
+ quant_config=quant_config)
273
274
def forward(
275
self,
vllm/model_executor/models/baichuan.py
+4
-2
@@ -154,7 +154,8 @@ class BaiChuanAttention(nn.Module):
154
self.attn = Attention(self.num_heads,
155
self.head_dim,
156
scaling,
157
- alibi_slopes=alibi_slopes)
157
+ alibi_slopes=alibi_slopes,
158
+ quant_config=quant_config)
159
else:
160
self.rotary_emb = get_rope(
161
self.head_dim,
@@ -166,7 +167,8 @@ class BaiChuanAttention(nn.Module):
167
self.attn = Attention(self.num_heads,
168
self.head_dim,
169
self.scaling,
169
- cache_config=cache_config)
170
+ cache_config=cache_config,
171
+ quant_config=quant_config)
172
173
def forward(
174
self,
vllm/model_executor/models/bloom.py
+2
-1
@@ -111,7 +111,8 @@ class BloomAttention(nn.Module):
111
self.head_dim,
112
scaling,
113
alibi_slopes=alibi_slopes,
114
- cache_config=cache_config)
114
+ cache_config=cache_config,
115
+ quant_config=quant_config)
116
117
def forward(
118
self,
vllm/model_executor/models/chatglm.py
+6
-7
@@ -86,13 +86,12 @@ class GLMAttention(nn.Module):
86
base=10000 * rope_ratio,
87
is_neox_style=False,
88
)
89
- self.attn = Attention(
90
- self.num_heads,
91
- self.head_dim,
92
- self.scaling,
93
- num_kv_heads=self.num_kv_heads,
94
- cache_config=cache_config,
95
- )
89
+ self.attn = Attention(self.num_heads,
90
+ self.head_dim,
91
+ self.scaling,
92
+ num_kv_heads=self.num_kv_heads,
93
+ cache_config=cache_config,
94
+ quant_config=quant_config)
95
96
def forward(
97
self,
vllm/model_executor/models/commandr.py
+6
-7
@@ -177,13 +177,12 @@ class CohereAttention(nn.Module):
177
rope_scaling=self.rope_scaling,
178
is_neox_style=False,
179
)
180
- self.attn = Attention(
181
- self.num_heads,
182
- self.head_dim,
183
- self.scaling,
184
- num_kv_heads=self.num_kv_heads,
185
- cache_config=cache_config,
186
- )
180
+ self.attn = Attention(self.num_heads,
181
+ self.head_dim,
182
+ self.scaling,
183
+ num_kv_heads=self.num_kv_heads,
184
+ cache_config=cache_config,
185
+ quant_config=quant_config)
186
if self.use_qk_norm:
187
self.q_norm = LayerNorm(param_shape=(self.num_heads,
188
self.head_dim),
vllm/model_executor/models/dbrx.py
+6
-7
@@ -218,13 +218,12 @@ class DbrxAttention(nn.Module):
218
self.q_size = self.num_heads * self.head_dim
219
self.kv_size = self.num_kv_heads * self.head_dim
220
self.scaling = self.head_dim**-0.5
221
- self.attn = Attention(
222
- self.num_heads,
223
- self.head_dim,
224
- self.scaling,
225
- num_kv_heads=self.num_kv_heads,
226
- cache_config=cache_config,
227
- )
221
+ self.attn = Attention(self.num_heads,
222
+ self.head_dim,
223
+ self.scaling,
224
+ num_kv_heads=self.num_kv_heads,
225
+ cache_config=cache_config,
226
+ quant_config=quant_config)
227
228
def forward(
229
self,
vllm/model_executor/models/deepseek.py
+2
-1
@@ -232,7 +232,8 @@ class DeepseekAttention(nn.Module):
232
self.head_dim,
233
self.scaling,
234
num_kv_heads=self.num_kv_heads,
235
- cache_config=cache_config)
235
+ cache_config=cache_config,
236
+ quant_config=quant_config)
237
238
def forward(
239
self,
vllm/model_executor/models/falcon.py
+6
-3
@@ -153,7 +153,8 @@ class FalconAttention(nn.Module):
153
self.attn = Attention(self.num_heads,
154
self.head_dim,
155
self.inv_norm_factor,
156
- num_kv_heads=self.num_kv_heads)
156
+ num_kv_heads=self.num_kv_heads,
157
+ quant_config=quant_config)
158
elif self.use_alibi:
159
tp_rank = get_tensor_model_parallel_rank()
160
head_start = tp_rank * self.num_heads
@@ -165,13 +166,15 @@ class FalconAttention(nn.Module):
166
self.head_dim,
167
self.inv_norm_factor,
168
num_kv_heads=self.num_kv_heads,
168
- alibi_slopes=alibi_slopes)
169
+ alibi_slopes=alibi_slopes,
170
+ quant_config=quant_config)
171
else:
172
self.attn = Attention(self.num_heads,
173
self.head_dim,
174
scale=self.inv_norm_factor,
175
num_kv_heads=self.num_kv_heads,
174
- cache_config=cache_config)
176
+ cache_config=cache_config,
177
+ quant_config=quant_config)
178
179
def forward(
180
self,
vllm/model_executor/models/gemma.py
+2
-1
@@ -157,7 +157,8 @@ class GemmaAttention(nn.Module):
157
self.head_dim,
158
self.scaling,
159
num_kv_heads=self.num_kv_heads,
160
- cache_config=cache_config)
160
+ cache_config=cache_config,
161
+ quant_config=quant_config)
162
163
def forward(
164
self,
vllm/model_executor/models/gpt2.py
+2
-1
@@ -75,7 +75,8 @@ class GPT2Attention(nn.Module):
75
self.attn = Attention(self.num_heads,
76
self.head_dim,
77
scale=self.scale,
78
- cache_config=cache_config)
78
+ cache_config=cache_config,
79
+ quant_config=quant_config)
80
81
def forward(
82
self,
vllm/model_executor/models/gpt_bigcode.py
+2
-1
@@ -88,7 +88,8 @@ class GPTBigCodeAttention(nn.Module):
88
self.head_dim,
89
scale=self.scale,
90
num_kv_heads=self.num_kv_heads,
91
- cache_config=cache_config)
91
+ cache_config=cache_config,
92
+ quant_config=quant_config)
93
94
def forward(
95
self,
vllm/model_executor/models/gpt_j.py
+2
-1
@@ -88,7 +88,8 @@ class GPTJAttention(nn.Module):
88
self.attn = Attention(self.num_heads,
89
self.head_size,
90
scaling,
91
- cache_config=cache_config)
91
+ cache_config=cache_config,
92
+ quant_config=quant_config)
93
94
def forward(
95
self,
vllm/model_executor/models/gpt_neox.py
+2
-1
@@ -89,7 +89,8 @@ class GPTNeoXAttention(nn.Module):
89
self.attn = Attention(self.num_heads,
90
self.head_size,
91
scaling,
92
- cache_config=cache_config)
92
+ cache_config=cache_config,
93
+ quant_config=quant_config)
94
95
def forward(
96
self,
vllm/model_executor/models/internlm2.py
+2
-1
@@ -117,7 +117,8 @@ class InternLM2Attention(nn.Module):
117
self.head_dim,
118
self.scaling,
119
num_kv_heads=self.num_kv_heads,
120
- cache_config=cache_config)
120
+ cache_config=cache_config,
121
+ quant_config=quant_config)
122
123
def forward(
124
self,
vllm/model_executor/models/jais.py
+6
-7
@@ -105,13 +105,12 @@ class JAISAttention(nn.Module):
105
head_end = (tp_rank + 1) * self.num_heads
106
alibi_slopes = _get_alibi_slopes(total_num_heads)
107
alibi_slopes = alibi_slopes[head_start:head_end]
108
- self.attn = Attention(
109
- self.num_heads,
110
- self.head_dim,
111
- scale=self.scale,
112
- alibi_slopes=alibi_slopes,
113
- cache_config=cache_config,
114
- )
108
+ self.attn = Attention(self.num_heads,
109
+ self.head_dim,
110
+ scale=self.scale,
111
+ alibi_slopes=alibi_slopes,
112
+ cache_config=cache_config,
113
+ quant_config=quant_config)
114
115
def forward(
116
self,
vllm/model_executor/models/llama.py
+18
-14
@@ -47,7 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import (
47
default_weight_loader, kv_cache_scales_loader)
48
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import SamplerOutput
50
-from vllm.utils import is_hip
50
+from vllm.utils import is_hip, print_warning_once
51
52
53
class LlamaMLP(nn.Module):
@@ -119,15 +119,6 @@ class LlamaAttention(nn.Module):
119
self.rope_theta = rope_theta
120
self.max_position_embeddings = max_position_embeddings
121
122
- # This will be overwritten by model initialization if we are using it.
123
- # N.B. currently we only support per tensor scalar scaling factors
124
- # & only applicable to ROCm (AMD GPU).
125
- # The scaling factor convention we are assuming is
126
- # quantized_value * scaling_factor ~= true_value
127
- # which is consistent with the practice of setting
128
- # scaling_factor = tensor_amax / FPtype_max
129
- self.kv_scale = 1.0
130
-
122
self.qkv_proj = QKVParallelLinear(
123
hidden_size,
124
self.head_dim,
@@ -155,7 +146,8 @@ class LlamaAttention(nn.Module):
146
self.scaling,
147
num_kv_heads=self.num_kv_heads,
148
sliding_window=sliding_window,
158
- cache_config=cache_config)
149
+ cache_config=cache_config,
150
+ quant_config=quant_config)
151
152
def forward(
153
self,
@@ -167,8 +159,7 @@ class LlamaAttention(nn.Module):
159
qkv, _ = self.qkv_proj(hidden_states)
160
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
161
q, k = self.rotary_emb(positions, q, k)
170
- attn_output = self.attn(q, k, v, kv_cache, attn_metadata,
171
- self.kv_scale)
162
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
163
output, _ = self.o_proj(attn_output)
164
return output
165
@@ -421,6 +412,19 @@ class LlamaForCausalLM(nn.Module):
412
# Skip loading extra bias for GPTQ models.
413
if name.endswith(".bias") and name not in params_dict:
414
continue
415
+ # Remapping the name of FP8 kv-scale.
416
+ if name.endswith("kv_scale"):
417
+ remapped_kv_scale_name = name.replace(
418
+ ".kv_scale", ".attn.kv_scale")
419
+ if remapped_kv_scale_name not in params_dict:
420
+ print_warning_once(
421
+ f"Found kv scale in the checkpoint (e.g. {name}), "
422
+ "but not found the expected name in the model "
423
+ f"(e.g. {remapped_kv_scale_name}). kv-scale is "
424
+ "not loaded.")
425
+ continue
426
+ else:
427
+ name = remapped_kv_scale_name
428
param = params_dict[name]
429
weight_loader = getattr(param, "weight_loader",
430
default_weight_loader)
@@ -445,7 +449,7 @@ class LlamaForCausalLM(nn.Module):
449
# scaling_factor = tensor_amax / FPtype_max
450
scaling_factor *= 2
451
if hasattr(layer_self_attn, "kv_scale"):
448
- layer_self_attn.kv_scale = scaling_factor
452
+ layer_self_attn.attn._kv_scale = scaling_factor
453
else:
454
raise RuntimeError("Self attention has no KV cache scaling "
455
"factor attribute!")
vllm/model_executor/models/minicpm.py
+2
-1
@@ -236,7 +236,8 @@ class MiniCPMAttention(nn.Module):
236
self.head_dim,
237
self.scaling,
238
num_kv_heads=self.num_kv_heads,
239
- cache_config=cache_config)
239
+ cache_config=cache_config,
240
+ quant_config=quant_config)
241
242
def forward(
243
self,
vllm/model_executor/models/mixtral.py
+21
-8
@@ -308,14 +308,13 @@ class MixtralAttention(nn.Module):
308
base=int(self.rope_theta),
309
is_neox_style=True,
310
)
311
- self.attn = Attention(
312
- self.num_heads,
313
- self.head_dim,
314
- self.scaling,
315
- num_kv_heads=self.num_kv_heads,
316
- sliding_window=self.sliding_window,
317
- cache_config=cache_config,
318
- )
311
+ self.attn = Attention(self.num_heads,
312
+ self.head_dim,
313
+ self.scaling,
314
+ num_kv_heads=self.num_kv_heads,
315
+ sliding_window=self.sliding_window,
316
+ cache_config=cache_config,
317
+ quant_config=quant_config)
318
319
def forward(
320
self,
@@ -581,6 +580,20 @@ class MixtralForCausalLM(nn.Module):
580
# Skip loading extra bias for GPTQ models.
581
if name.endswith(".bias") and name not in params_dict:
582
continue
583
+ # Remapping the name of FP8 kv-scale.
584
+ if name.endswith("kv_scale"):
585
+ remapped_kv_scale_name = name.replace(
586
+ ".kv_scale", ".attn.kv_scale")
587
+ if remapped_kv_scale_name not in params_dict:
588
+ print_warning_once(
589
+ "Found kv scale in the checkpoint "
590
+ f"(e.g. {name}), but not found the expected "
591
+ f"name in the model "
592
+ f"(e.g. {remapped_kv_scale_name}). "
593
+ "kv-scale is not loaded.")
594
+ continue
595
+ else:
596
+ name = remapped_kv_scale_name
597
param = params_dict[name]
598
weight_loader = getattr(param, "weight_loader",
599
default_weight_loader)
vllm/model_executor/models/mixtral_quant.py
+7
-8
@@ -213,14 +213,13 @@ class MixtralAttention(nn.Module):
213
base=int(self.rope_theta),
214
is_neox_style=True,
215
)
216
- self.attn = Attention(
217
- self.num_heads,
218
- self.head_dim,
219
- self.scaling,
220
- num_kv_heads=self.num_kv_heads,
221
- sliding_window=self.sliding_window,
222
- cache_config=cache_config,
223
- )
216
+ self.attn = Attention(self.num_heads,
217
+ self.head_dim,
218
+ self.scaling,
219
+ num_kv_heads=self.num_kv_heads,
220
+ sliding_window=self.sliding_window,
221
+ cache_config=cache_config,
222
+ quant_config=quant_config)
223
224
def forward(
225
self,
vllm/model_executor/models/mpt.py
+2
-1
@@ -110,7 +110,8 @@ class MPTAttention(nn.Module):
110
scaling,
111
alibi_slopes=alibi_slopes,
112
num_kv_heads=self.num_kv_heads,
113
- cache_config=cache_config)
113
+ cache_config=cache_config,
114
+ quant_config=quant_config)
115
116
def forward(
117
self,
vllm/model_executor/models/olmo.py
+2
-1
@@ -96,7 +96,8 @@ class OlmoAttention(nn.Module):
96
self.attn = Attention(self.num_heads,
97
self.head_dim,
98
scale=self.scaling,
99
- cache_config=cache_config)
99
+ cache_config=cache_config,
100
+ quant_config=quant_config)
101
102
# Attention output projection.
103
self.o_proj = RowParallelLinear(
vllm/model_executor/models/opt.py
+2
-1
@@ -91,7 +91,8 @@ class OPTAttention(nn.Module):
91
self.attn = Attention(self.num_heads,
92
self.head_dim,
93
scale=self.scaling,
94
- cache_config=cache_config)
94
+ cache_config=cache_config,
95
+ quant_config=quant_config)
96
97
def forward(
98
self,
vllm/model_executor/models/orion.py
+2
-1
@@ -121,7 +121,8 @@ class OrionAttention(nn.Module):
121
self.head_dim,
122
self.scaling,
123
num_kv_heads=self.num_kv_heads,
124
- cache_config=cache_config)
124
+ cache_config=cache_config,
125
+ quant_config=quant_config)
126
127
def forward(
128
self,
vllm/model_executor/models/phi.py
+2
-1
@@ -110,7 +110,8 @@ class PhiAttention(nn.Module):
110
self.attn = Attention(self.num_heads,
111
self.head_size,
112
scaling,
113
- cache_config=cache_config)
113
+ cache_config=cache_config,
114
+ quant_config=quant_config)
115
116
def forward(
117
self,
vllm/model_executor/models/qwen.py
+2
-1
@@ -106,7 +106,8 @@ class QWenAttention(nn.Module):
106
self.attn = Attention(self.num_heads,
107
self.head_dim,
108
self.scaling,
109
- cache_config=cache_config)
109
+ cache_config=cache_config,
110
+ quant_config=quant_config)
111
112
def forward(
113
self,
vllm/model_executor/models/qwen2.py
+2
-1
@@ -141,7 +141,8 @@ class Qwen2Attention(nn.Module):
141
self.scaling,
142
num_kv_heads=self.num_kv_heads,
143
sliding_window=self.sliding_window,
144
- cache_config=cache_config)
144
+ cache_config=cache_config,
145
+ quant_config=quant_config)
146
147
def forward(
148
self,
vllm/model_executor/models/qwen2_moe.py
+2
-1
@@ -241,7 +241,8 @@ class Qwen2MoeAttention(nn.Module):
241
self.head_dim,
242
self.scaling,
243
num_kv_heads=self.num_kv_heads,
244
- cache_config=cache_config)
244
+ cache_config=cache_config,
245
+ quant_config=quant_config)
246
247
def forward(
248
self,
vllm/model_executor/models/stablelm.py
+2
-1
@@ -127,7 +127,8 @@ class StablelmAttention(nn.Module):
127
self.head_dim,
128
self.scaling,
129
num_kv_heads=self.num_key_value_heads,
130
- cache_config=cache_config)
130
+ cache_config=cache_config,
131
+ quant_config=quant_config)
132
133
def forward(
134
self,
vllm/model_executor/models/starcoder2.py
+7
-8
@@ -97,14 +97,13 @@ class Starcoder2Attention(nn.Module):
97
base=int(self.rope_theta),
98
is_neox_style=True,
99
)
100
- self.attn = Attention(
101
- self.num_heads,
102
- self.head_dim,
103
- self.scaling,
104
- num_kv_heads=self.num_kv_heads,
105
- sliding_window=self.sliding_window,
106
- cache_config=cache_config,
107
- )
100
+ self.attn = Attention(self.num_heads,
101
+ self.head_dim,
102
+ self.scaling,
103
+ num_kv_heads=self.num_kv_heads,
104
+ sliding_window=self.sliding_window,
105
+ cache_config=cache_config,
106
+ quant_config=quant_config)
107
108
def forward(
109
self,
vllm/model_executor/models/xverse.py
+2
-1
@@ -135,7 +135,8 @@ class XverseAttention(nn.Module):
135
self.scaling,
136
num_kv_heads=self.num_kv_heads,
137
sliding_window=sliding_window,
138
- cache_config=cache_config)
138
+ cache_config=cache_config,
139
+ quant_config=quant_config)
140
141
def forward(
142
self,
vllm/utils.py
+2
@@ -31,6 +31,8 @@ STR_DTYPE_TO_TORCH_DTYPE = {
31
"bfloat16": torch.bfloat16,
32
"float": torch.float,
33
"fp8": torch.uint8,
34
+ "fp8_e4m3": torch.uint8,
35
+ "fp8_e5m2": torch.uint8,
36
}
37
38
vllm/worker/model_runner.py
+12
-5
@@ -1,4 +1,5 @@
1
import time
2
+import warnings
3
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
4
5
import numpy as np
@@ -168,11 +169,21 @@ class ModelRunner:
169
self.model = self.lora_manager.create_lora_manager(self.model)
170
171
if self.kv_cache_dtype == "fp8" and is_hip():
171
- # Currently scaled KV cache is only enabled on ROCm
172
+ # Currently only ROCm accepts kv-cache scaling factors
173
+ # via quantization_param_path and this will be deprecated
174
+ # in the future.
175
if self.model_config.quantization_param_path is not None:
176
if callable(getattr(self.model, "load_kv_cache_scales", None)):
177
+ warnings.warn(
178
+ "Loading kv cache scaling factor from JSON is "
179
+ "deprecated and will be removed. Please include "
180
+ "kv cache scaling factors in the model checkpoint.",
181
+ FutureWarning,
182
+ stacklevel=2)
183
self.model.load_kv_cache_scales(
184
self.model_config.quantization_param_path)
185
+ logger.info("Loaded KV cache scaling factors from %s",
186
+ self.model_config.quantization_param_path)
187
else:
188
raise RuntimeError(
189
"Using FP8 KV cache and scaling factors provided but "
@@ -183,10 +194,6 @@ class ModelRunner:
194
"Using FP8 KV cache but no scaling factors "
195
"provided. Defaulting to scaling factors of 1.0. "
196
"This may lead to less accurate results!")
186
- elif self.model_config.quantization_param_path is not None:
187
- logger.warning("KV cache scaling factors provided, "
188
- "but the KV cache data type is not FP8. "
189
- "KV cache scaling factors will not be used.")
197
198
def save_sharded_state(
199
self,