@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a3a73ab069
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Load FP8 kv-cache scaling factors from checkpoints (#4893)

The 2nd PR for #4532. This PR supports loading FP8 kv-cache scaling factors from a FP8 checkpoint (with .kv_scale parameter).

Cody Yu committed May 22, 2024 at 13:28 UTC a3a73ab0696b6692f3eecf80271a01fa97bd001d
40 files changed +284 -158
benchmarks/benchmark_latency.py
+6 -8
@@ -153,15 +153,13 @@ if __name__ == '__main__':
153 action='store_true',
154 help='enforce eager mode and disable CUDA graph')
155 parser.add_argument(
156 - "--kv-cache-dtype",
156 + '--kv-cache-dtype',
157 type=str,
158 - choices=['auto', 'fp8'],
159 - default='auto',
160 - help=
161 - 'Data type for kv cache storage. If "auto", will use model data type. '
162 - 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
163 - 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
164 - 'instead supported for common inference criteria.')
158 + choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
159 + default="auto",
160 + help='Data type for kv cache storage. If "auto", will use model '
161 + 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
162 + 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
163 parser.add_argument(
164 '--quantization-param-path',
165 type=str,
benchmarks/benchmark_throughput.py
+5 -7
@@ -323,15 +323,13 @@ if __name__ == "__main__":
323 action="store_true",
324 help="enforce eager execution")
325 parser.add_argument(
326 - "--kv-cache-dtype",
326 + '--kv-cache-dtype',
327 type=str,
328 - choices=["auto", "fp8"],
328 + choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
329 default="auto",
330 - help=
331 - 'Data type for kv cache storage. If "auto", will use model data type. '
332 - 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
333 - 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
334 - 'common inference criteria.')
330 + help='Data type for kv cache storage. If "auto", will use model '
331 + 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
332 + 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
333 parser.add_argument(
334 '--quantization-param-path',
335 type=str,
benchmarks/kernels/benchmark_paged_attention.py
+4 -6
@@ -183,13 +183,11 @@ if __name__ == '__main__':
183 parser.add_argument(
184 "--kv-cache-dtype",
185 type=str,
186 - choices=["auto", "fp8"],
186 + choices=["auto", "fp8", "fp8_e5m2", "fp8_e4m3"],
187 default="auto",
188 - help=
189 - 'Data type for kv cache storage. If "auto", will use model data type. '
190 - 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
191 - 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
192 - 'common inference criteria.')
188 + help="Data type for kv cache storage. If 'auto', will use model "
189 + "data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. "
190 + "ROCm (AMD GPU) supports fp8 (=fp8_e4m3)")
191 args = parser.parse_args()
192 print(args)
193
tests/models/test_fp8.py
+52 -28
@@ -16,31 +16,55 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
16 MAX_MODEL_LEN = 1024
17
18 MODELS = [
19 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8",
19 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV",
20 "meta-llama/Meta-Llama-3-8B-Instruct",
21 ]
22
23 EXPECTED_STRS_MAP = {
24 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8": [
25 - 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
26 - 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
27 - 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
28 - 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
29 - 'Zeta-5, a highly advanced robot designed for menial labor, whirred to a',
30 - 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
31 - 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
32 - 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, guri o',
33 - ],
34 - "meta-llama/Meta-Llama-3-8B-Instruct": [
35 - 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
36 - 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
37 - 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
38 - 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
39 - 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
40 - 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
41 - 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
42 - 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
43 - ],
24 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV": {
25 + "auto": [
26 + 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
27 + 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
28 + 'Artificial intelligence (AI) and human intelligence (HI) process information in distinct ways, with both',
29 + 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
30 + 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
31 + 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
32 + 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
33 + 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, nemuri no'
34 + ],
35 + "fp8": [
36 + 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
37 + 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
38 + 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
39 + 'A neural network is a complex system made up of several basic components that work together to enable it to',
40 + 'Zeta-5, a highly advanced robot designed for menial labor, had never experienced anything like',
41 + 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
42 + 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
43 + 'Here are the translations:\n\n**Japanese:** (Haya kotori wa mushi o tsuk'
44 + ]
45 + },
46 + "meta-llama/Meta-Llama-3-8B-Instruct": {
47 + "auto": [
48 + 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
49 + 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
50 + 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
51 + 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
52 + 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
53 + 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
54 + 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
55 + 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
56 + ],
57 + "fp8": [
58 + 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
59 + 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
60 + 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
61 + 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
62 + 'In the year 2154, robotics engineer Dr. Rachel Kim had spent years perfecting her latest',
63 + 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
64 + 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
65 + 'Here are the translations:\n\n**Japanese:** (Haya tori, mushi o tsukamu'
66 + ]
67 + },
68 }
69
70 capability = torch.cuda.get_device_capability()
@@ -52,14 +76,14 @@ fp8_not_supported = (capability <
76 @pytest.mark.skipif(fp8_not_supported,
77 reason="fp8 is not supported on this GPU type.")
78 @pytest.mark.parametrize("model_name", MODELS)
55 -def test_models(
56 - example_prompts,
57 - model_name,
58 -) -> None:
79 +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
80 +def test_models(example_prompts, model_name, kv_cache_dtype) -> None:
81 model = LLM(model=model_name,
82 max_model_len=MAX_MODEL_LEN,
83 + trust_remote_code=True,
84 enforce_eager=True,
62 - quantization="fp8")
85 + quantization="fp8",
86 + kv_cache_dtype=kv_cache_dtype)
87
88 tokenizer = AutoTokenizer.from_pretrained(model_name)
89 formatted_prompts = [
@@ -81,8 +105,8 @@ def test_models(
105 generations.append(outputs[0].outputs[0].text)
106 del model
107
84 - print(generations)
85 - expected_strs = EXPECTED_STRS_MAP[model_name]
108 + print(model_name, kv_cache_dtype, generations)
109 + expected_strs = EXPECTED_STRS_MAP[model_name][kv_cache_dtype]
110 for i in range(len(example_prompts)):
111 generated_str = generations[i]
112 expected_str = expected_strs[i]
vllm/attention/layer.py
+25 -2
@@ -7,6 +7,8 @@ import torch.nn as nn
7 from vllm.attention.backends.abstract import AttentionMetadata
8 from vllm.attention.selector import get_attn_backend
9 from vllm.config import CacheConfig
10 +from vllm.model_executor.layers.quantization.base_config import (
11 + QuantizationConfig)
12
13
14 class Attention(nn.Module):
@@ -30,6 +32,7 @@ class Attention(nn.Module):
32 alibi_slopes: Optional[List[float]] = None,
33 sliding_window: Optional[int] = None,
34 cache_config: Optional[CacheConfig] = None,
35 + quant_config: Optional[QuantizationConfig] = None,
36 ) -> None:
37 super().__init__()
38 if cache_config is not None:
@@ -40,6 +43,27 @@ class Attention(nn.Module):
43 block_size = 16
44 if num_kv_heads is None:
45 num_kv_heads = num_heads
46 +
47 + # The default kv_scale is set to 1.0. This is ignored
48 + # when kv-cache is not fp8, and should be used with
49 + # kv-cache in fp8_e5m2. For kv-cache in fp8_e4m3, we
50 + # expect the pre-quantized kv_scale to be loaded along
51 + # with the model weights.
52 + self.kv_cache_dtype = kv_cache_dtype
53 + self._kv_scale = 1.0
54 + quant_method = quant_config.get_quant_method(
55 + self) if quant_config else None
56 + if quant_method is not None:
57 + if self.kv_cache_dtype == "fp8_e5m2":
58 + raise ValueError("fp8_e5m2 kv-cache is not supported with "
59 + "fp8 checkpoints.")
60 + # When FP8 quantization is enabled, we make a parameter
61 + # "kv_scale" so that it can be loaded from FP8 checkpoint.
62 + # The kv_scale will then be converted back
63 + # to self._kv_scale in a native float32 value after weight loading.
64 + self.quant_method = quant_method
65 + self.quant_method.create_weights(self)
66 +
67 # During model initialization, the default dtype is set as the model
68 # weight and activation dtype.
69 dtype = torch.get_default_dtype()
@@ -57,10 +81,9 @@ class Attention(nn.Module):
81 value: torch.Tensor,
82 kv_cache: Optional[torch.Tensor],
83 attn_metadata: AttentionMetadata,
60 - kv_scale: float = 1.0,
84 ) -> torch.Tensor:
85 return self.impl.forward(query, key, value, kv_cache, attn_metadata,
63 - kv_scale)
86 + self._kv_scale)
87
88 def extra_repr(self) -> str:
89 s = f"head_size={self.impl.head_size}" # type: ignore
vllm/config.py
+3 -5
@@ -355,14 +355,12 @@ class CacheConfig:
355 def _verify_cache_dtype(self) -> None:
356 if self.cache_dtype == "auto":
357 pass
358 - elif self.cache_dtype == "fp8":
358 + elif self.cache_dtype in ("fp8", "fp8_e4m3", "fp8_e5m2"):
359 logger.info(
360 "Using fp8 data type to store kv cache. It reduces the GPU "
361 "memory footprint and boosts the performance. "
362 - "But it may cause slight accuracy drop without scaling "
363 - "factors. FP8_E5M2 (without scaling) is only supported on "
364 - "cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 "
365 - "is instead supported for common inference criteria.")
362 + "Meanwhile, it may cause accuracy drop without a proper "
363 + "scaling factor")
364 else:
365 raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
366
vllm/engine/arg_utils.py
+3 -4
@@ -191,12 +191,11 @@ class EngineArgs:
191 parser.add_argument(
192 '--kv-cache-dtype',
193 type=str,
194 - choices=['auto', 'fp8'],
194 + choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
195 default=EngineArgs.kv_cache_dtype,
196 help='Data type for kv cache storage. If "auto", will use model '
197 - 'data type. FP8_E5M2 (without scaling) is only supported on cuda '
198 - 'version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
199 - 'supported for common inference criteria.')
197 + 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
198 + 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
199 parser.add_argument(
200 '--quantization-param-path',
201 type=nullable_str,
vllm/model_executor/layers/quantization/fp8.py
+45 -2
@@ -8,8 +8,9 @@ from vllm import _custom_ops as ops
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 from vllm.model_executor.layers.quantization.base_config import (
11 - QuantizationConfig)
11 + QuantizationConfig, QuantizeMethodBase)
12 from vllm.model_executor.utils import set_weight_attrs
13 +from vllm.utils import print_warning_once
14
15 ACTIVATION_SCHEMES = ["static", "dynamic"]
16
@@ -58,9 +59,13 @@ class Fp8Config(QuantizationConfig):
59 activation_scheme=activation_scheme)
60
61 def get_quant_method(
61 - self, layer: torch.nn.Module) -> Optional["Fp8LinearMethod"]:
62 + self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
63 + from vllm.attention.layer import Attention # Avoid circular import
64 +
65 if isinstance(layer, LinearBase):
66 return Fp8LinearMethod(self)
67 + if isinstance(layer, Attention):
68 + return Fp8KVCacheMethod(self)
69 return None
70
71 def get_scaled_act_names(self) -> List[str]:
@@ -251,6 +256,44 @@ class Fp8LinearMethod(LinearMethodBase):
256 return torch.narrow(output, 0, 0, x.shape[0])
257
258
259 +class Fp8KVCacheMethod(QuantizeMethodBase):
260 + """Supports loading kv-cache scaling factors from FP8 checkpoints.
261 + """
262 +
263 + def __init__(self, quant_config: Fp8Config):
264 + self.quant_config = quant_config
265 +
266 + def create_weights(self, layer: torch.nn.Module):
267 + """Create "weight" (aka kv_scale) for an attention layer.
268 +
269 + Args:
270 + layer: The layer that is using the QuantizeMethodBase factory.
271 + """
272 + # Initialize the KV cache scale to 1.0 as the default value.
273 + # If the kv_scale appears in the checkpoint, it will be
274 + # overwritten when loading weights.
275 + layer.kv_scale = Parameter(torch.tensor(1.0), requires_grad=False)
276 +
277 + def apply(self, layer: torch.nn.Module) -> torch.Tensor:
278 + raise RuntimeError("Fp8KVCacheMethod.apply should not be called.")
279 +
280 + def process_weights_after_loading(self, layer: Module) -> None:
281 + # If the kv-cache dtype is auto, we enforce the kv-scale to be 1.0
282 + # regardless whether the kv-scale is available in the checkpoint.
283 + if layer.kv_cache_dtype != "auto":
284 + kv_scale = layer.kv_scale.to("cpu").tolist()
285 + if not isinstance(kv_scale, float):
286 + raise ValueError("Only support per-tensor scaling factor "
287 + "for fp8 KV cache")
288 + layer._kv_scale = kv_scale
289 + if layer._kv_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
290 + print_warning_once(
291 + "Using KV cache scaling factor 1.0 for fp8_e4m3. This may "
292 + "cause accuracy issues. Please make sure kv-cache scaling "
293 + "factor is available in the fp8 checkpoint.")
294 + del layer.kv_scale
295 +
296 +
297 def all_close_1d(x: torch.Tensor) -> bool:
298 assert len(x.shape) == 1
299 return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/models/arctic.py
+2 -1
@@ -268,7 +268,8 @@ class ArcticAttention(nn.Module):
268 self.head_dim,
269 self.scaling,
270 num_kv_heads=self.num_kv_heads,
271 - cache_config=cache_config)
271 + cache_config=cache_config,
272 + quant_config=quant_config)
273
274 def forward(
275 self,
vllm/model_executor/models/baichuan.py
+4 -2
@@ -154,7 +154,8 @@ class BaiChuanAttention(nn.Module):
154 self.attn = Attention(self.num_heads,
155 self.head_dim,
156 scaling,
157 - alibi_slopes=alibi_slopes)
157 + alibi_slopes=alibi_slopes,
158 + quant_config=quant_config)
159 else:
160 self.rotary_emb = get_rope(
161 self.head_dim,
@@ -166,7 +167,8 @@ class BaiChuanAttention(nn.Module):
167 self.attn = Attention(self.num_heads,
168 self.head_dim,
169 self.scaling,
169 - cache_config=cache_config)
170 + cache_config=cache_config,
171 + quant_config=quant_config)
172
173 def forward(
174 self,
vllm/model_executor/models/bloom.py
+2 -1
@@ -111,7 +111,8 @@ class BloomAttention(nn.Module):
111 self.head_dim,
112 scaling,
113 alibi_slopes=alibi_slopes,
114 - cache_config=cache_config)
114 + cache_config=cache_config,
115 + quant_config=quant_config)
116
117 def forward(
118 self,
vllm/model_executor/models/chatglm.py
+6 -7
@@ -86,13 +86,12 @@ class GLMAttention(nn.Module):
86 base=10000 * rope_ratio,
87 is_neox_style=False,
88 )
89 - self.attn = Attention(
90 - self.num_heads,
91 - self.head_dim,
92 - self.scaling,
93 - num_kv_heads=self.num_kv_heads,
94 - cache_config=cache_config,
95 - )
89 + self.attn = Attention(self.num_heads,
90 + self.head_dim,
91 + self.scaling,
92 + num_kv_heads=self.num_kv_heads,
93 + cache_config=cache_config,
94 + quant_config=quant_config)
95
96 def forward(
97 self,
vllm/model_executor/models/commandr.py
+6 -7
@@ -177,13 +177,12 @@ class CohereAttention(nn.Module):
177 rope_scaling=self.rope_scaling,
178 is_neox_style=False,
179 )
180 - self.attn = Attention(
181 - self.num_heads,
182 - self.head_dim,
183 - self.scaling,
184 - num_kv_heads=self.num_kv_heads,
185 - cache_config=cache_config,
186 - )
180 + self.attn = Attention(self.num_heads,
181 + self.head_dim,
182 + self.scaling,
183 + num_kv_heads=self.num_kv_heads,
184 + cache_config=cache_config,
185 + quant_config=quant_config)
186 if self.use_qk_norm:
187 self.q_norm = LayerNorm(param_shape=(self.num_heads,
188 self.head_dim),
vllm/model_executor/models/dbrx.py
+6 -7
@@ -218,13 +218,12 @@ class DbrxAttention(nn.Module):
218 self.q_size = self.num_heads * self.head_dim
219 self.kv_size = self.num_kv_heads * self.head_dim
220 self.scaling = self.head_dim**-0.5
221 - self.attn = Attention(
222 - self.num_heads,
223 - self.head_dim,
224 - self.scaling,
225 - num_kv_heads=self.num_kv_heads,
226 - cache_config=cache_config,
227 - )
221 + self.attn = Attention(self.num_heads,
222 + self.head_dim,
223 + self.scaling,
224 + num_kv_heads=self.num_kv_heads,
225 + cache_config=cache_config,
226 + quant_config=quant_config)
227
228 def forward(
229 self,
vllm/model_executor/models/deepseek.py
+2 -1
@@ -232,7 +232,8 @@ class DeepseekAttention(nn.Module):
232 self.head_dim,
233 self.scaling,
234 num_kv_heads=self.num_kv_heads,
235 - cache_config=cache_config)
235 + cache_config=cache_config,
236 + quant_config=quant_config)
237
238 def forward(
239 self,
vllm/model_executor/models/falcon.py
+6 -3
@@ -153,7 +153,8 @@ class FalconAttention(nn.Module):
153 self.attn = Attention(self.num_heads,
154 self.head_dim,
155 self.inv_norm_factor,
156 - num_kv_heads=self.num_kv_heads)
156 + num_kv_heads=self.num_kv_heads,
157 + quant_config=quant_config)
158 elif self.use_alibi:
159 tp_rank = get_tensor_model_parallel_rank()
160 head_start = tp_rank * self.num_heads
@@ -165,13 +166,15 @@ class FalconAttention(nn.Module):
166 self.head_dim,
167 self.inv_norm_factor,
168 num_kv_heads=self.num_kv_heads,
168 - alibi_slopes=alibi_slopes)
169 + alibi_slopes=alibi_slopes,
170 + quant_config=quant_config)
171 else:
172 self.attn = Attention(self.num_heads,
173 self.head_dim,
174 scale=self.inv_norm_factor,
175 num_kv_heads=self.num_kv_heads,
174 - cache_config=cache_config)
176 + cache_config=cache_config,
177 + quant_config=quant_config)
178
179 def forward(
180 self,
vllm/model_executor/models/gemma.py
+2 -1
@@ -157,7 +157,8 @@ class GemmaAttention(nn.Module):
157 self.head_dim,
158 self.scaling,
159 num_kv_heads=self.num_kv_heads,
160 - cache_config=cache_config)
160 + cache_config=cache_config,
161 + quant_config=quant_config)
162
163 def forward(
164 self,
vllm/model_executor/models/gpt2.py
+2 -1
@@ -75,7 +75,8 @@ class GPT2Attention(nn.Module):
75 self.attn = Attention(self.num_heads,
76 self.head_dim,
77 scale=self.scale,
78 - cache_config=cache_config)
78 + cache_config=cache_config,
79 + quant_config=quant_config)
80
81 def forward(
82 self,
vllm/model_executor/models/gpt_bigcode.py
+2 -1
@@ -88,7 +88,8 @@ class GPTBigCodeAttention(nn.Module):
88 self.head_dim,
89 scale=self.scale,
90 num_kv_heads=self.num_kv_heads,
91 - cache_config=cache_config)
91 + cache_config=cache_config,
92 + quant_config=quant_config)
93
94 def forward(
95 self,
vllm/model_executor/models/gpt_j.py
+2 -1
@@ -88,7 +88,8 @@ class GPTJAttention(nn.Module):
88 self.attn = Attention(self.num_heads,
89 self.head_size,
90 scaling,
91 - cache_config=cache_config)
91 + cache_config=cache_config,
92 + quant_config=quant_config)
93
94 def forward(
95 self,
vllm/model_executor/models/gpt_neox.py
+2 -1
@@ -89,7 +89,8 @@ class GPTNeoXAttention(nn.Module):
89 self.attn = Attention(self.num_heads,
90 self.head_size,
91 scaling,
92 - cache_config=cache_config)
92 + cache_config=cache_config,
93 + quant_config=quant_config)
94
95 def forward(
96 self,
vllm/model_executor/models/internlm2.py
+2 -1
@@ -117,7 +117,8 @@ class InternLM2Attention(nn.Module):
117 self.head_dim,
118 self.scaling,
119 num_kv_heads=self.num_kv_heads,
120 - cache_config=cache_config)
120 + cache_config=cache_config,
121 + quant_config=quant_config)
122
123 def forward(
124 self,
vllm/model_executor/models/jais.py
+6 -7
@@ -105,13 +105,12 @@ class JAISAttention(nn.Module):
105 head_end = (tp_rank + 1) * self.num_heads
106 alibi_slopes = _get_alibi_slopes(total_num_heads)
107 alibi_slopes = alibi_slopes[head_start:head_end]
108 - self.attn = Attention(
109 - self.num_heads,
110 - self.head_dim,
111 - scale=self.scale,
112 - alibi_slopes=alibi_slopes,
113 - cache_config=cache_config,
114 - )
108 + self.attn = Attention(self.num_heads,
109 + self.head_dim,
110 + scale=self.scale,
111 + alibi_slopes=alibi_slopes,
112 + cache_config=cache_config,
113 + quant_config=quant_config)
114
115 def forward(
116 self,
vllm/model_executor/models/llama.py
+18 -14
@@ -47,7 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import (
47 default_weight_loader, kv_cache_scales_loader)
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import SamplerOutput
50 -from vllm.utils import is_hip
50 +from vllm.utils import is_hip, print_warning_once
51
52
53 class LlamaMLP(nn.Module):
@@ -119,15 +119,6 @@ class LlamaAttention(nn.Module):
119 self.rope_theta = rope_theta
120 self.max_position_embeddings = max_position_embeddings
121
122 - # This will be overwritten by model initialization if we are using it.
123 - # N.B. currently we only support per tensor scalar scaling factors
124 - # & only applicable to ROCm (AMD GPU).
125 - # The scaling factor convention we are assuming is
126 - # quantized_value * scaling_factor ~= true_value
127 - # which is consistent with the practice of setting
128 - # scaling_factor = tensor_amax / FPtype_max
129 - self.kv_scale = 1.0
130 -
122 self.qkv_proj = QKVParallelLinear(
123 hidden_size,
124 self.head_dim,
@@ -155,7 +146,8 @@ class LlamaAttention(nn.Module):
146 self.scaling,
147 num_kv_heads=self.num_kv_heads,
148 sliding_window=sliding_window,
158 - cache_config=cache_config)
149 + cache_config=cache_config,
150 + quant_config=quant_config)
151
152 def forward(
153 self,
@@ -167,8 +159,7 @@ class LlamaAttention(nn.Module):
159 qkv, _ = self.qkv_proj(hidden_states)
160 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
161 q, k = self.rotary_emb(positions, q, k)
170 - attn_output = self.attn(q, k, v, kv_cache, attn_metadata,
171 - self.kv_scale)
162 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
163 output, _ = self.o_proj(attn_output)
164 return output
165
@@ -421,6 +412,19 @@ class LlamaForCausalLM(nn.Module):
412 # Skip loading extra bias for GPTQ models.
413 if name.endswith(".bias") and name not in params_dict:
414 continue
415 + # Remapping the name of FP8 kv-scale.
416 + if name.endswith("kv_scale"):
417 + remapped_kv_scale_name = name.replace(
418 + ".kv_scale", ".attn.kv_scale")
419 + if remapped_kv_scale_name not in params_dict:
420 + print_warning_once(
421 + f"Found kv scale in the checkpoint (e.g. {name}), "
422 + "but not found the expected name in the model "
423 + f"(e.g. {remapped_kv_scale_name}). kv-scale is "
424 + "not loaded.")
425 + continue
426 + else:
427 + name = remapped_kv_scale_name
428 param = params_dict[name]
429 weight_loader = getattr(param, "weight_loader",
430 default_weight_loader)
@@ -445,7 +449,7 @@ class LlamaForCausalLM(nn.Module):
449 # scaling_factor = tensor_amax / FPtype_max
450 scaling_factor *= 2
451 if hasattr(layer_self_attn, "kv_scale"):
448 - layer_self_attn.kv_scale = scaling_factor
452 + layer_self_attn.attn._kv_scale = scaling_factor
453 else:
454 raise RuntimeError("Self attention has no KV cache scaling "
455 "factor attribute!")
vllm/model_executor/models/minicpm.py
+2 -1
@@ -236,7 +236,8 @@ class MiniCPMAttention(nn.Module):
236 self.head_dim,
237 self.scaling,
238 num_kv_heads=self.num_kv_heads,
239 - cache_config=cache_config)
239 + cache_config=cache_config,
240 + quant_config=quant_config)
241
242 def forward(
243 self,
vllm/model_executor/models/mixtral.py
+21 -8
@@ -308,14 +308,13 @@ class MixtralAttention(nn.Module):
308 base=int(self.rope_theta),
309 is_neox_style=True,
310 )
311 - self.attn = Attention(
312 - self.num_heads,
313 - self.head_dim,
314 - self.scaling,
315 - num_kv_heads=self.num_kv_heads,
316 - sliding_window=self.sliding_window,
317 - cache_config=cache_config,
318 - )
311 + self.attn = Attention(self.num_heads,
312 + self.head_dim,
313 + self.scaling,
314 + num_kv_heads=self.num_kv_heads,
315 + sliding_window=self.sliding_window,
316 + cache_config=cache_config,
317 + quant_config=quant_config)
318
319 def forward(
320 self,
@@ -581,6 +580,20 @@ class MixtralForCausalLM(nn.Module):
580 # Skip loading extra bias for GPTQ models.
581 if name.endswith(".bias") and name not in params_dict:
582 continue
583 + # Remapping the name of FP8 kv-scale.
584 + if name.endswith("kv_scale"):
585 + remapped_kv_scale_name = name.replace(
586 + ".kv_scale", ".attn.kv_scale")
587 + if remapped_kv_scale_name not in params_dict:
588 + print_warning_once(
589 + "Found kv scale in the checkpoint "
590 + f"(e.g. {name}), but not found the expected "
591 + f"name in the model "
592 + f"(e.g. {remapped_kv_scale_name}). "
593 + "kv-scale is not loaded.")
594 + continue
595 + else:
596 + name = remapped_kv_scale_name
597 param = params_dict[name]
598 weight_loader = getattr(param, "weight_loader",
599 default_weight_loader)
vllm/model_executor/models/mixtral_quant.py
+7 -8
@@ -213,14 +213,13 @@ class MixtralAttention(nn.Module):
213 base=int(self.rope_theta),
214 is_neox_style=True,
215 )
216 - self.attn = Attention(
217 - self.num_heads,
218 - self.head_dim,
219 - self.scaling,
220 - num_kv_heads=self.num_kv_heads,
221 - sliding_window=self.sliding_window,
222 - cache_config=cache_config,
223 - )
216 + self.attn = Attention(self.num_heads,
217 + self.head_dim,
218 + self.scaling,
219 + num_kv_heads=self.num_kv_heads,
220 + sliding_window=self.sliding_window,
221 + cache_config=cache_config,
222 + quant_config=quant_config)
223
224 def forward(
225 self,
vllm/model_executor/models/mpt.py
+2 -1
@@ -110,7 +110,8 @@ class MPTAttention(nn.Module):
110 scaling,
111 alibi_slopes=alibi_slopes,
112 num_kv_heads=self.num_kv_heads,
113 - cache_config=cache_config)
113 + cache_config=cache_config,
114 + quant_config=quant_config)
115
116 def forward(
117 self,
vllm/model_executor/models/olmo.py
+2 -1
@@ -96,7 +96,8 @@ class OlmoAttention(nn.Module):
96 self.attn = Attention(self.num_heads,
97 self.head_dim,
98 scale=self.scaling,
99 - cache_config=cache_config)
99 + cache_config=cache_config,
100 + quant_config=quant_config)
101
102 # Attention output projection.
103 self.o_proj = RowParallelLinear(
vllm/model_executor/models/opt.py
+2 -1
@@ -91,7 +91,8 @@ class OPTAttention(nn.Module):
91 self.attn = Attention(self.num_heads,
92 self.head_dim,
93 scale=self.scaling,
94 - cache_config=cache_config)
94 + cache_config=cache_config,
95 + quant_config=quant_config)
96
97 def forward(
98 self,
vllm/model_executor/models/orion.py
+2 -1
@@ -121,7 +121,8 @@ class OrionAttention(nn.Module):
121 self.head_dim,
122 self.scaling,
123 num_kv_heads=self.num_kv_heads,
124 - cache_config=cache_config)
124 + cache_config=cache_config,
125 + quant_config=quant_config)
126
127 def forward(
128 self,
vllm/model_executor/models/phi.py
+2 -1
@@ -110,7 +110,8 @@ class PhiAttention(nn.Module):
110 self.attn = Attention(self.num_heads,
111 self.head_size,
112 scaling,
113 - cache_config=cache_config)
113 + cache_config=cache_config,
114 + quant_config=quant_config)
115
116 def forward(
117 self,
vllm/model_executor/models/qwen.py
+2 -1
@@ -106,7 +106,8 @@ class QWenAttention(nn.Module):
106 self.attn = Attention(self.num_heads,
107 self.head_dim,
108 self.scaling,
109 - cache_config=cache_config)
109 + cache_config=cache_config,
110 + quant_config=quant_config)
111
112 def forward(
113 self,
vllm/model_executor/models/qwen2.py
+2 -1
@@ -141,7 +141,8 @@ class Qwen2Attention(nn.Module):
141 self.scaling,
142 num_kv_heads=self.num_kv_heads,
143 sliding_window=self.sliding_window,
144 - cache_config=cache_config)
144 + cache_config=cache_config,
145 + quant_config=quant_config)
146
147 def forward(
148 self,
vllm/model_executor/models/qwen2_moe.py
+2 -1
@@ -241,7 +241,8 @@ class Qwen2MoeAttention(nn.Module):
241 self.head_dim,
242 self.scaling,
243 num_kv_heads=self.num_kv_heads,
244 - cache_config=cache_config)
244 + cache_config=cache_config,
245 + quant_config=quant_config)
246
247 def forward(
248 self,
vllm/model_executor/models/stablelm.py
+2 -1
@@ -127,7 +127,8 @@ class StablelmAttention(nn.Module):
127 self.head_dim,
128 self.scaling,
129 num_kv_heads=self.num_key_value_heads,
130 - cache_config=cache_config)
130 + cache_config=cache_config,
131 + quant_config=quant_config)
132
133 def forward(
134 self,
vllm/model_executor/models/starcoder2.py
+7 -8
@@ -97,14 +97,13 @@ class Starcoder2Attention(nn.Module):
97 base=int(self.rope_theta),
98 is_neox_style=True,
99 )
100 - self.attn = Attention(
101 - self.num_heads,
102 - self.head_dim,
103 - self.scaling,
104 - num_kv_heads=self.num_kv_heads,
105 - sliding_window=self.sliding_window,
106 - cache_config=cache_config,
107 - )
100 + self.attn = Attention(self.num_heads,
101 + self.head_dim,
102 + self.scaling,
103 + num_kv_heads=self.num_kv_heads,
104 + sliding_window=self.sliding_window,
105 + cache_config=cache_config,
106 + quant_config=quant_config)
107
108 def forward(
109 self,
vllm/model_executor/models/xverse.py
+2 -1
@@ -135,7 +135,8 @@ class XverseAttention(nn.Module):
135 self.scaling,
136 num_kv_heads=self.num_kv_heads,
137 sliding_window=sliding_window,
138 - cache_config=cache_config)
138 + cache_config=cache_config,
139 + quant_config=quant_config)
140
141 def forward(
142 self,
vllm/utils.py
+2
@@ -31,6 +31,8 @@ STR_DTYPE_TO_TORCH_DTYPE = {
31 "bfloat16": torch.bfloat16,
32 "float": torch.float,
33 "fp8": torch.uint8,
34 + "fp8_e4m3": torch.uint8,
35 + "fp8_e5m2": torch.uint8,
36 }
37
38
vllm/worker/model_runner.py
+12 -5
@@ -1,4 +1,5 @@
1 import time
2 +import warnings
3 from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
4
5 import numpy as np
@@ -168,11 +169,21 @@ class ModelRunner:
169 self.model = self.lora_manager.create_lora_manager(self.model)
170
171 if self.kv_cache_dtype == "fp8" and is_hip():
171 - # Currently scaled KV cache is only enabled on ROCm
172 + # Currently only ROCm accepts kv-cache scaling factors
173 + # via quantization_param_path and this will be deprecated
174 + # in the future.
175 if self.model_config.quantization_param_path is not None:
176 if callable(getattr(self.model, "load_kv_cache_scales", None)):
177 + warnings.warn(
178 + "Loading kv cache scaling factor from JSON is "
179 + "deprecated and will be removed. Please include "
180 + "kv cache scaling factors in the model checkpoint.",
181 + FutureWarning,
182 + stacklevel=2)
183 self.model.load_kv_cache_scales(
184 self.model_config.quantization_param_path)
185 + logger.info("Loaded KV cache scaling factors from %s",
186 + self.model_config.quantization_param_path)
187 else:
188 raise RuntimeError(
189 "Using FP8 KV cache and scaling factors provided but "
@@ -183,10 +194,6 @@ class ModelRunner:
194 "Using FP8 KV cache but no scaling factors "
195 "provided. Defaulting to scaling factors of 1.0. "
196 "This may lead to less accurate results!")
186 - elif self.model_config.quantization_param_path is not None:
187 - logger.warning("KV cache scaling factors provided, "
188 - "but the KV cache data type is not FP8. "
189 - "KV cache scaling factors will not be used.")
197
198 def save_sharded_state(
199 self,