@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0b2da72a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Expand MLA to support most types of quantization (#13181)

Michael Goin committed Feb 14, 2025 at 01:19 UTC f0b2da72a84a7e481ca7ae1e84cedae5bc645611
3 files changed +61 -132
vllm/attention/backends/mla/utils.py
+26 -45
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
26 from vllm.model_executor.layers.quantization.utils.fp8_utils import (
27 apply_fp8_linear_generic, current_platform_fp8_dtype, is_fp8)
28 from vllm.model_executor.layers.quantization.utils.quant_utils import (
29 - scaled_dequantize, scaled_quantize)
29 + scaled_quantize)
30 from vllm.model_executor.layers.rotary_embedding import (
31 DeepseekScalingRotaryEmbedding, RotaryEmbedding)
32
@@ -220,16 +220,6 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
220 .view(-1, self.num_heads, self.kv_lora_rank)
221
222 def process_weights_after_loading(self, act_dtype: torch.dtype):
223 -
224 - def is_layer_fp8(layer: LinearBase) -> bool:
225 - return isinstance(layer.quant_method, Fp8LinearMethod) or\
226 - (isinstance(layer.quant_method, CompressedTensorsLinearMethod)\
227 - and isinstance(layer.scheme, CompressedTensorsW8A8Fp8))
228 -
229 - def quantization_scheme_supported(layer: LinearBase) -> bool:
230 - return isinstance(layer.quant_method, UnquantizedLinearMethod) or \
231 - is_layer_fp8(layer)
232 -
223 # TODO(lucas) This is very gross, we need a more wide scale refactor of
224 # all the FP8 code with a more standard way of
225 # defining schemes/group-shapes, we should also potentially force
@@ -239,7 +229,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
229 def get_scale_group_shapes_for_fp8(layer: LinearBase) -> \
230 Tuple[Tuple[int, int], Tuple[int, int]]:
231 if isinstance(layer.quant_method, Fp8LinearMethod):
242 - if layer.quant_method.block_quant is not None:
232 + if layer.quant_method.block_quant:
233 weight_block_size = \
234 layer.quant_method.quant_config.weight_block_size
235 # per-token-group (1, X), block-quantized (X, Y)
@@ -267,41 +257,32 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
257 f"{layer.quant_method}, please run with VLLM_MLA_DISABLE=1"
258 )
259
270 - def get_scales(layer: LinearBase) -> torch.Tensor:
271 - if hasattr(layer, "weight_scale_inv"):
272 - return layer.weight_scale_inv
273 - return layer.weight_scale
274 -
275 - def get_and_maybe_dequant_weights(layer: LinearBase):
276 - if is_layer_fp8(layer):
277 - if isinstance(layer.quant_method, \
278 - CompressedTensorsLinearMethod) and \
279 - isinstance(layer.scheme, CompressedTensorsW8A8Fp8):
280 - # NOTE(lucas): note sure why but `CompressedTensorsW8A8Fp8`
281 - # seems to store weights as (input, output) instead of
282 - # (output, input) so we need to transpose
283 - weight = layer.weight.T # standardize to (output, input)
284 - else:
285 - weight = layer.weight
286 - _, weight_scale_group_shape = \
287 - get_scale_group_shapes_for_fp8(layer)
288 - scales = get_scales(layer)
289 -
290 - return scaled_dequantize(weight, scales,
291 - weight_scale_group_shape)
292 - else:
260 + def get_layer_weight(layer):
261 + if hasattr(layer, "weight"):
262 return layer.weight
263 + elif hasattr(layer, "qweight"):
264 + return layer.qweight
265 + else:
266 + raise AttributeError(
267 + f"Layer '{layer}' has neither weight nor qweight")
268
295 - if not (quantization_scheme_supported(self.kv_b_proj) and\
296 - quantization_scheme_supported(self.q_proj) and\
297 - quantization_scheme_supported(self.o_proj)):
298 - raise NotImplementedError(
299 - "Only FP8 and UnquantizedLinearMethod are supported for MLA"
300 - ", please run with VLLM_MLA_DISABLE=1")
301 -
302 - weight_dtype = self.kv_b_proj.weight.dtype
303 - assert self.o_proj.weight.dtype == weight_dtype
304 - assert self.q_proj.weight.dtype == weight_dtype
269 + def get_and_maybe_dequant_weights(layer: LinearBase):
270 + if not isinstance(layer.quant_method, UnquantizedLinearMethod):
271 + # NOTE: This should only be used offline, since it's O(N^3)
272 + eye = torch.eye(layer.input_size_per_partition,
273 + dtype=act_dtype,
274 + device=get_layer_weight(layer).device)
275 + dequant_weights = layer.quant_method.apply(layer,
276 + eye,
277 + bias=None)
278 + del eye
279 + # standardize to (output, input)
280 + return dequant_weights.T
281 + return layer.weight
282 +
283 + weight_dtype = get_layer_weight(self.kv_b_proj).dtype
284 + assert get_layer_weight(self.o_proj).dtype == weight_dtype
285 + assert get_layer_weight(self.q_proj).dtype == weight_dtype
286
287 kv_b_proj_weight = get_and_maybe_dequant_weights(self.kv_b_proj).T
288 assert kv_b_proj_weight.shape == (
vllm/config.py
+1 -31
@@ -991,37 +991,7 @@ class ModelConfig:
991
992 @property
993 def use_mla(self) -> bool:
994 - if not self.is_deepseek_mla or envs.VLLM_MLA_DISABLE:
995 - return False
996 -
997 - if self.quantization is not None and self.quantization not in [\
998 - "fp8", "compressed-tensors"]:
999 - logger.warning(
1000 - "MLA is not supported with %s quantization. "
1001 - "Disabling MLA.", self.quantization)
1002 - return False
1003 -
1004 - # If using a "compressed-tensors" checkpoint, check that all groups
1005 - # have fp8 for both weights and activations.
1006 - if self.quantization == "compressed-tensors":
1007 - quant_config = self._parse_quant_hf_config()
1008 - for group_name, cfg in quant_config.get("config_groups", {
1009 - "": {}
1010 - }).items():
1011 - act_cfg = cfg.get("input_activations", {})
1012 - act_type = None if act_cfg is None else act_cfg.get("type", "")
1013 - w_cfg = cfg.get("weights", {})
1014 - w_type = None if w_cfg is None else w_cfg.get("type", "")
1015 - if act_type != "fp8" or w_type != "fp8":
1016 - logger.warning(
1017 - "compressed-tensors MLA support requires fp8 "
1018 - "activations and weights in group '%s', but got "
1019 - "activations type '%s' and weights type '%s'.\n "
1020 - "Full config: %s", group_name, act_type, w_type,
1021 - quant_config)
1022 - return False
1023 -
1024 - return True
994 + return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE
995
996 @property
997 def supported_runner_types(self) -> Set[RunnerType]:
vllm/model_executor/model_loader/loader.py
+34 -56
@@ -153,6 +153,30 @@ def _initialize_model(
153 return model_class(**kwargs)
154
155
156 +def _process_weights_after_loading(model: nn.Module, model_config: ModelConfig,
157 + target_device: torch.device) -> None:
158 + for _, module in model.named_modules():
159 + quant_method = getattr(module, "quant_method", None)
160 + if isinstance(quant_method, QuantizeMethodBase):
161 + # When quant methods need to process weights after loading
162 + # (for repacking, quantizing, etc), they expect parameters
163 + # to be on the global target device. This scope is for the
164 + # case where cpu offloading is used, where we will move the
165 + # parameters onto device for processing and back off after.
166 + with device_loading_context(module, target_device):
167 + quant_method.process_weights_after_loading(module)
168 +
169 + # Currently only used by MLA.
170 + # NOTE: This intentionally happens after other modules so we can easily
171 + # decompress the weights for MLA.
172 + for _, module in model.named_modules():
173 + if isinstance(module, Attention) and \
174 + hasattr(module, "process_weights_after_loading"):
175 + # TODO(lucas): see if there is a way to unify the signatures
176 + # of process_weights_after_loading
177 + module.process_weights_after_loading(model_config.dtype)
178 +
179 +
180 class BaseModelLoader(ABC):
181 """Base class for model loaders."""
182
@@ -376,7 +400,6 @@ class DefaultModelLoader(BaseModelLoader):
400 def load_model(self, vllm_config: VllmConfig) -> nn.Module:
401 device_config = vllm_config.device_config
402 model_config = vllm_config.model_config
379 -
403 target_device = torch.device(device_config.device)
404 with set_default_torch_dtype(model_config.dtype):
405 with target_device:
@@ -394,23 +417,8 @@ class DefaultModelLoader(BaseModelLoader):
417 "Following weights were not initialized from "
418 f"checkpoint: {weights_not_loaded}")
419
397 - for _, module in model.named_modules():
398 - quant_method = getattr(module, "quant_method", None)
399 - if isinstance(quant_method, QuantizeMethodBase):
400 - # When quant methods need to process weights after loading
401 - # (for repacking, quantizing, etc), they expect parameters
402 - # to be on the global target device. This scope is for the
403 - # case where cpu offloading is used, where we will move the
404 - # parameters onto device for processing and back off after.
405 - with device_loading_context(module, target_device):
406 - quant_method.process_weights_after_loading(module)
407 - if isinstance(module, Attention) and \
408 - hasattr(module, "process_weights_after_loading"):
409 - # When attention modules need to process weights after
410 - # currently only used by MLA
411 - # TODO(lucas): see if there is a way to unify the signatures
412 - # of process_weights_after_loading
413 - module.process_weights_after_loading(model_config.dtype)
420 + _process_weights_after_loading(model, model_config, target_device)
421 +
422 return model.eval()
423
424
@@ -429,29 +437,15 @@ class DummyModelLoader(BaseModelLoader):
437 def load_model(self, vllm_config: VllmConfig) -> nn.Module:
438 device_config = vllm_config.device_config
439 model_config = vllm_config.model_config
440 + target_device = torch.device(device_config.device)
441 with set_default_torch_dtype(model_config.dtype):
433 - with torch.device(device_config.device):
442 + with target_device:
443 model = _initialize_model(vllm_config=vllm_config)
444 # NOTE(woosuk): For accurate performance evaluation, we assign
445 # random values to the weights.
446 initialize_dummy_weights(model)
447
439 - for _, module in model.named_modules():
440 - quant_method = getattr(module, "quant_method", None)
441 - if quant_method is not None:
442 - # When quant methods need to process weights after loading
443 - # (for repacking, quantizing, etc), they expect parameters
444 - # to be on the global target device. This scope is for the
445 - # case where cpu offloading is used, where we will move the
446 - # parameters onto device for processing and back off after.
447 - with device_loading_context(
448 - module, torch.device(device_config.device)):
449 - quant_method.process_weights_after_loading(module)
450 - if isinstance(module, Attention) and \
451 - hasattr(module, "process_weights_after_loading"):
452 - # When attention modules need to process weights after
453 - # currently only used by MLA
454 - module.process_weights_after_loading(model_config.dtype)
448 + _process_weights_after_loading(model, model_config, target_device)
449 return model.eval()
450
451
@@ -632,6 +626,7 @@ class ShardedStateLoader(BaseModelLoader):
626 def load_model(self, vllm_config: VllmConfig) -> nn.Module:
627 device_config = vllm_config.device_config
628 model_config = vllm_config.model_config
629 + target_device = torch.device(device_config.device)
630 from safetensors.torch import safe_open
631
632 from vllm.distributed import get_tensor_model_parallel_rank
@@ -640,18 +635,10 @@ class ShardedStateLoader(BaseModelLoader):
635 model_config.revision)
636
637 with set_default_torch_dtype(model_config.dtype):
643 - with torch.device(device_config.device):
638 + with target_device:
639 model = _initialize_model(vllm_config=vllm_config)
645 - for _, module in model.named_modules():
646 - quant_method = getattr(module, "quant_method", None)
647 - if quant_method is not None:
648 - quant_method.process_weights_after_loading(module)
649 - if isinstance(module, Attention) and \
650 - hasattr(module, "process_weights_after_loading"):
651 - # When attention modules need to process weights after
652 - # currently only used by MLA
653 - module.process_weights_after_loading(
654 - model_config.dtype)
640 + _process_weights_after_loading(model, model_config,
641 + target_device)
642 rank = get_tensor_model_parallel_rank()
643 pattern = os.path.join(
644 local_model_path,
@@ -1401,16 +1388,7 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1388 self._get_weights_iterator(model_weights,
1389 model_config.revision))
1390
1404 - for _, module in model.named_modules():
1405 - quant_method = getattr(module, "quant_method", None)
1406 - if quant_method is not None:
1407 - with device_loading_context(module, target_device):
1408 - quant_method.process_weights_after_loading(module)
1409 - if isinstance(module, Attention) and \
1410 - hasattr(module, "process_weights_after_loading"):
1411 - # When attention modules need to process weights after
1412 - # currently only used by MLA
1413 - module.process_weights_after_loading(model_config.dtype)
1391 + _process_weights_after_loading(model, model_config, target_device)
1392 return model.eval()
1393
1394