@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e0b558a09
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Support FP8 kv cache scales from compressed-tensors (#6528)

Michael Goin committed Jul 23, 2024 at 00:11 UTC 9e0b558a0923004758a9dc91f5d4920ded48b42a
7 files changed +186 -75
tests/quantization/test_compressed_tensors.py
+7
@@ -150,3 +150,10 @@ def test_compressed_tensors_fp8(vllm_runner):
150
151 output = llm.generate_greedy("Hello my name is", max_tokens=20)
152 assert output
153 +
154 +
155 +def test_compressed_tensors_kv_cache(vllm_runner):
156 + model_path = "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"
157 + with vllm_runner(model_path, kv_cache_dtype="fp8") as llm:
158 + output = llm.generate_greedy("Hello world!", max_tokens=20)
159 + assert output
vllm/attention/layer.py
+11 -12
@@ -9,7 +9,7 @@ from vllm.attention.selector import get_attn_backend
9 from vllm.config import CacheConfig
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 -from vllm.model_executor.layers.quantization.fp8 import Fp8KVCacheMethod
12 +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
13
14
15 class Attention(nn.Module):
@@ -59,19 +59,18 @@ class Attention(nn.Module):
59 quant_method = quant_config.get_quant_method(
60 self, prefix=prefix) if quant_config else None
61 if quant_method is not None:
62 - assert isinstance(quant_method, Fp8KVCacheMethod)
62 + assert isinstance(quant_method, BaseKVCacheMethod)
63 # TODO (mgoin): kv cache dtype should be specified in the FP8
64 # checkpoint config and become the "auto" behavior
65 - if "fp8" in self.kv_cache_dtype:
66 - if self.kv_cache_dtype == "fp8_e5m2":
67 - raise ValueError("fp8_e5m2 kv-cache is not supported with "
68 - "fp8 checkpoints.")
69 - # When FP8 quantization is enabled, we make a parameter
70 - # "kv_scale" so that it can be loaded from FP8 checkpoint.
71 - # The k/v_scale will then be converted back to
72 - # self._kv_scale in a native float32 value after weight loading
73 - self.quant_method = quant_method
74 - self.quant_method.create_weights(self)
65 + if self.kv_cache_dtype == "fp8_e5m2":
66 + raise ValueError("fp8_e5m2 kv-cache is not supported with "
67 + "fp8 checkpoints.")
68 + # If quantization is enabled, we make "k_scale" and "v_scale"
69 + # parameters so that it can be loaded from the model checkpoint.
70 + # The k/v_scale will then be converted back to native float32
71 + # values after weight loading.
72 + self.quant_method = quant_method
73 + self.quant_method.create_weights(self)
74
75 # During model initialization, the default dtype is set as the model
76 # weight and activation dtype.
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+58 -5
@@ -5,7 +5,7 @@ from pydantic import BaseModel
5
6 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8 - QuantizationConfig)
8 + QuantizationConfig, QuantizeMethodBase)
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 CompressedTensorsScheme, CompressedTensorsUnquantized,
@@ -15,18 +15,23 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
16 QuantizationType, find_matched_target, is_activation_quantization_format,
17 should_ignore_layer)
18 +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
19 from vllm.platforms import current_platform
20
21
22 class CompressedTensorsConfig(QuantizationConfig):
23
23 - def __init__(self, target_scheme_map: Dict[str, Any], ignore: List[str],
24 - quant_format: str):
24 + def __init__(self,
25 + target_scheme_map: Dict[str, Any],
26 + ignore: List[str],
27 + quant_format: str,
28 + kv_cache_scheme: Optional[Dict[str, Any]] = None):
29
30 self.ignore = ignore
31 self.quant_format = quant_format
32 # Map from [target -> scheme]
33 self.target_scheme_map = target_scheme_map
34 + self.kv_cache_scheme = kv_cache_scheme
35
36 def get_linear_method(self) -> "CompressedTensorsLinearMethod":
37 return CompressedTensorsLinearMethod(self)
@@ -50,9 +55,12 @@ class CompressedTensorsConfig(QuantizationConfig):
55 self,
56 layer: torch.nn.Module,
57 prefix: str,
53 - ) -> Optional["CompressedTensorsLinearMethod"]:
58 + ) -> Optional["QuantizeMethodBase"]:
59 + from vllm.attention.layer import Attention # Avoid circular import
60 if isinstance(layer, LinearBase):
61 return CompressedTensorsLinearMethod(self)
62 + if isinstance(layer, Attention):
63 + return CompressedTensorsKVCacheMethod(self)
64 return None
65
66 @classmethod
@@ -85,7 +93,8 @@ class CompressedTensorsConfig(QuantizationConfig):
93
94 return cls(target_scheme_map=target_scheme_map,
95 ignore=ignore,
88 - quant_format=quant_format)
96 + quant_format=quant_format,
97 + kv_cache_scheme=config.get("kv_cache_scheme"))
98
99 @classmethod
100 def get_config_filenames(cls) -> List[str]:
@@ -309,3 +318,47 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
318 if scheme is None:
319 raise ValueError("A scheme must be defined for each layer")
320 return scheme.apply_weights(layer, x, bias=bias)
321 +
322 +
323 +class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
324 + """
325 + Supports loading kv-cache scaling factors from compressed-tensors
326 + checkpoints.
327 + """
328 +
329 + def __init__(self, quant_config: CompressedTensorsConfig):
330 + self.validate_kv_cache_scheme(quant_config.kv_cache_scheme)
331 + super().__init__(quant_config)
332 +
333 + @staticmethod
334 + def validate_kv_cache_scheme(kv_cache_scheme: Optional[Dict[str, Any]]):
335 + """
336 + Validator for the kv cache scheme. Useful for controlling the
337 + kv cache quantization schemes, that are being supported in vLLM
338 + :param kv_cache_scheme: the compressed-tensors kv cache scheme
339 + """
340 + if kv_cache_scheme is None:
341 + return
342 +
343 + type_ = kv_cache_scheme.get("type")
344 + num_bits = kv_cache_scheme.get("num_bits")
345 +
346 + if type_ != "float" and num_bits != 8:
347 + raise NotImplementedError(
348 + "Currently supported kv cache quantization is "
349 + "num_bits=8, type=float, however "
350 + f"received num_bits={num_bits}, type={type_}")
351 +
352 + strategy = kv_cache_scheme.get("strategy")
353 + if strategy != "tensor":
354 + raise NotImplementedError(
355 + "Only support per-tensor scaling factor "
356 + "for compressed-tensors KV cache. "
357 + f"Expected strategy: tensor, found strategy: {strategy}")
358 +
359 + is_symmetric = kv_cache_scheme.get("symmetric")
360 + if not is_symmetric:
361 + raise NotImplementedError(
362 + "Only support symmetric scaling factor "
363 + "for compressed-tensors KV cache. "
364 + f"However found symmetric: {is_symmetric}")
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+17
@@ -209,6 +209,23 @@ def _find_first_match(value: str,
209 return None
210
211
212 +def get_compressed_tensors_cache_scale(name: str) -> Optional[str]:
213 + """
214 + Check whether the param name matches the format for k/v cache scales
215 + in compressed-tensors. If this is the case, return its equivalent
216 + param name expected by vLLM
217 +
218 + :param name: param name
219 + :return: matching param name for KV cache scale in vLLM
220 + """
221 + if name.endswith(".output_scale") and ".k_proj" in name:
222 + return name.replace(".k_proj.output_scale", ".attn.k_scale")
223 + if name.endswith(".output_scale") and ".v_proj" in name:
224 + return name.replace(".v_proj.output_scale", ".attn.v_scale")
225 + # If no matches, return None
226 + return None
227 +
228 +
229 def _is_equal_or_regex_match(value: str,
230 target: str,
231 check_contains: bool = False) -> bool:
vllm/model_executor/layers/quantization/fp8.py
+5 -58
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
16 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
17 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -400,64 +401,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
401 topk_group=topk_group)
402
403
403 -class Fp8KVCacheMethod(QuantizeMethodBase):
404 - """Supports loading kv-cache scaling factors from FP8 checkpoints.
404 +class Fp8KVCacheMethod(BaseKVCacheMethod):
405 + """
406 + Supports loading kv-cache scaling factors from FP8 checkpoints.
407 """
408
409 def __init__(self, quant_config: Fp8Config):
408 - self.quant_config = quant_config
409 -
410 - def create_weights(self, layer: torch.nn.Module):
411 - """Create "weight" (aka k_scale and v_scale) for an attention layer.
412 -
413 - Args:
414 - layer: The layer that is using the QuantizeMethodBase factory.
415 - """
416 - # Initialize the KV cache scales to -1.0, which is an invalid value.
417 - # If the k/v_scale appears in the checkpoint, it will be
418 - # overwritten when loading weights.
419 - layer.k_scale = Parameter(torch.tensor(-1.0), requires_grad=False)
420 - layer.v_scale = Parameter(torch.tensor(-1.0), requires_grad=False)
421 -
422 - def apply(self, layer: torch.nn.Module) -> torch.Tensor:
423 - raise RuntimeError("Fp8KVCacheMethod.apply should not be called.")
424 -
425 - def process_weights_after_loading(self, layer: Module) -> None:
426 - # If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0
427 - # regardless whether the kv-scale is available in the checkpoint.
428 - if layer.kv_cache_dtype != "auto":
429 - if layer.k_scale > 0.0 and layer.v_scale > 0.0:
430 - # We prefer to use separate k_scale and v_scale if present
431 - k_scale = layer.k_scale.to("cpu").tolist()
432 - v_scale = layer.v_scale.to("cpu").tolist()
433 - elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
434 - # If no scales were loaded (both scales are invalid negative
435 - # values), use the default value of 1.0
436 - k_scale = Parameter(torch.tensor(1.0), requires_grad=False)
437 - v_scale = Parameter(torch.tensor(1.0), requires_grad=False)
438 - else:
439 - # If we find a single kv_scale in the checkpoint, we remap
440 - # kv_scale to k_scale during weight loading, and duplicate
441 - # k_scale to v_scale here
442 - assert layer.k_scale > 0.0
443 - scale_to_duplicate = max(layer.k_scale, layer.v_scale)
444 - k_scale = scale_to_duplicate.to("cpu").tolist()
445 - v_scale = scale_to_duplicate.to("cpu").tolist()
446 -
447 - if not isinstance(k_scale, float) or not isinstance(
448 - v_scale, float):
449 - raise ValueError("Only support per-tensor scaling factor "
450 - "for fp8 KV cache")
451 -
452 - # These are used in the final Attention.forward()
453 - layer._k_scale = k_scale
454 - layer._v_scale = v_scale
455 - if (layer._k_scale == 1.0 and layer._v_scale == 1.0
456 - and "e5m2" not in layer.kv_cache_dtype):
457 - print_warning_once(
458 - "Using KV cache scaling factor 1.0 for fp8_e4m3. This "
459 - "may cause accuracy issues. Please make sure k/v_scale "
460 - "scaling factors are available in the fp8 checkpoint.")
461 -
462 - del layer.k_scale
463 - del layer.v_scale
410 + super().__init__(quant_config)
vllm/model_executor/layers/quantization/kv_cache.py new
+78
@@ -0,0 +1,78 @@
1 +import torch
2 +
3 +from vllm.model_executor.layers.quantization.base_config import (
4 + QuantizationConfig, QuantizeMethodBase)
5 +from vllm.utils import print_warning_once
6 +
7 +
8 +class BaseKVCacheMethod(QuantizeMethodBase):
9 + """
10 + Quant method that adds `_k_scale` and `_v_scale` attributes to the
11 + Attention layer to support loading those scaling factors from checkpoints.
12 + The k/v_scale will be used to:
13 + - quantize k/v_cache entries before saving them to the cache
14 + - dequantize k/v_cache entries before fetching them from the cache
15 +
16 + :param quant_config: the appropriate QuantizationConfig
17 + """
18 +
19 + def __init__(self, quant_config: QuantizationConfig):
20 + self.quant_config = quant_config
21 +
22 + def create_weights(self, layer: torch.nn.Module):
23 + """
24 + Create "weight" (aka k_scale and v_scale) for an attention layer.
25 + """
26 + # Initialize the KV cache scales to -1.0, which is an invalid value.
27 + # If the k/v_scale appears in the checkpoint, it will be
28 + # overwritten when loading weights.
29 + layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0),
30 + requires_grad=False)
31 + layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0),
32 + requires_grad=False)
33 +
34 + def apply(self, layer: torch.nn.Module) -> torch.Tensor:
35 + raise RuntimeError(
36 + f"{self.__class__.__name__}.apply should not be called.")
37 +
38 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
39 + # If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0
40 + # regardless whether the kv-scale is available in the checkpoint.
41 + if layer.kv_cache_dtype != "auto":
42 + if layer.k_scale > 0.0 and layer.v_scale > 0.0:
43 + # We prefer to use separate k_scale and v_scale if present
44 + k_scale = layer.k_scale.to("cpu").tolist()
45 + v_scale = layer.v_scale.to("cpu").tolist()
46 + elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
47 + # If no scales were loaded (both scales are invalid negative
48 + # values), use the default value of 1.0
49 + k_scale = torch.nn.Parameter(torch.tensor(1.0),
50 + requires_grad=False)
51 + v_scale = torch.nn.Parameter(torch.tensor(1.0),
52 + requires_grad=False)
53 + else:
54 + # If we find a single kv_scale in the checkpoint, we remap
55 + # kv_scale to k_scale during weight loading, and duplicate
56 + # k_scale to v_scale here
57 + assert layer.k_scale > 0.0
58 + scale_to_duplicate = max(layer.k_scale, layer.v_scale)
59 + k_scale = scale_to_duplicate.to("cpu").tolist()
60 + v_scale = scale_to_duplicate.to("cpu").tolist()
61 +
62 + if not isinstance(k_scale, float) or not isinstance(
63 + v_scale, float):
64 + raise ValueError("Only support per-tensor scaling factor "
65 + "for fp8 KV cache")
66 +
67 + # These are used in the final Attention.forward()
68 + layer._k_scale = k_scale
69 + layer._v_scale = v_scale
70 + if (layer._k_scale == 1.0 and layer._v_scale == 1.0
71 + and "e5m2" not in layer.kv_cache_dtype):
72 + print_warning_once(
73 + "Using KV cache scaling factor 1.0 for fp8_e4m3. This "
74 + "may cause accuracy issues. Please make sure k/v_scale "
75 + "scaling factors are available in the fp8 checkpoint.")
76 +
77 + del layer.k_scale
78 + del layer.v_scale
vllm/model_executor/models/llama.py
+10
@@ -39,6 +39,8 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 from vllm.model_executor.layers.quantization.base_config import (
41 QuantizationConfig)
42 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 + get_compressed_tensors_cache_scale)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 from vllm.model_executor.layers.sampler import Sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -467,6 +469,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
469 # Models trained using ColossalAI may include these tensors in
470 # the checkpoint. Skip them.
471 continue
472 + if scale_name := get_compressed_tensors_cache_scale(name):
473 + # Loading kv cache scales for compressed-tensors quantization
474 + param = params_dict[scale_name]
475 + weight_loader = getattr(param, "weight_loader",
476 + default_weight_loader)
477 + loaded_weight = loaded_weight[0]
478 + weight_loader(param, loaded_weight)
479 + continue
480 for (param_name, weight_name, shard_id) in stacked_params_mapping:
481 if weight_name not in name:
482 continue