9025a9a705 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/quantization/test_register_quantization_config.py

+1

@@ -53,6 +53,7 @@ class CustomQuantConfig(QuantizationConfig):

53

54 def __init__(self, num_bits: int = 8) -> None:

55 """Initialize the quantization config."""

56 + super().__init__()

57 self.num_bits = num_bits

58

59 def get_name(self) -> QuantizationMethods:

vllm/lora/models.py

+1 -1

@@ -805,7 +805,7 @@ def create_lora_manager(

805 lora_manager_cls: type[LoRAModelManager] = LoRAModelManager,

806 **kwargs) -> LoRAModelManager:

807 """Create a LoRA adapter for a given model."""

808 - if not hasattr(model, "packed_modules_mapping"):

808 + if not isinstance(model, SupportsLoRA):

809 raise ValueError(f"Model {type(model)} is not supported for LoRA.")

810 lora_manager = lora_manager_cls(

811 model=model,

vllm/lora/worker_manager.py

+1 -4

@@ -111,10 +111,7 @@ class WorkerLoRAManager(AbstractWorkerManager):

111 # For some models like Qwen2VL, we need to use hf_to_vllm_mapper

112 # to ensure correct loading of lora weights.

113 model = self._adapter_manager.model

114 - hf_to_vllm_mapper = None

115 - if (hasattr(model, "hf_to_vllm_mapper")

116 - and model.hf_to_vllm_mapper is not None):

117 - hf_to_vllm_mapper = model.hf_to_vllm_mapper

114 + hf_to_vllm_mapper = getattr(model, "hf_to_vllm_mapper", None)

115

116 lora = self._lora_model_cls.from_local_checkpoint(

117 lora_path,

vllm/model_executor/layers/quantization/base_config.py

+13

@@ -10,6 +10,7 @@ from torch import nn

10

11 if TYPE_CHECKING:

12 from vllm.model_executor.layers.quantization import QuantizationMethods

13 + from vllm.model_executor.models.utils import WeightsMapper

14 else:

15 QuantizationMethods = str

16

@@ -149,3 +150,15 @@ class QuantizationConfig(ABC):

150

151 def get_cache_scale(self, name: str) -> Optional[str]:

152 return None

153 +

154 + def apply_vllm_mapper( # noqa: B027

155 + self, hf_to_vllm_mapper: "WeightsMapper"):

156 + """

157 + Interface for models to update module names referenced in

158 + quantization configs in order to reflect the vllm model structure

159 +

160 + :param hf_to_vllm_mapper: maps from hf model structure (the assumed

161 + structure of the qconfig) to vllm model structure

162 + """

163 + # TODO (@kylesayrs): add implementations for all subclasses

164 + pass

vllm/model_executor/layers/quantization/bitblas.py

+1

@@ -63,6 +63,7 @@ class BitBLASConfig(QuantizationConfig):

63 # (since we have only one group per output channel)

64 desc_act = False

65

66 + super().__init__()

67 self.weight_bits = weight_bits

68 self.group_size = group_size

69 self.desc_act = desc_act

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py

+16 -1

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from contextlib import suppress

5 -from typing import Any, Literal, Optional, cast

5 +from typing import TYPE_CHECKING, Any, Literal, Optional, cast

6

7 import torch

8 from compressed_tensors.config import (CompressionFormat,

@@ -37,6 +37,9 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import

37 cutlass_fp4_supported)

38 from vllm.platforms import current_platform

39

40 +if TYPE_CHECKING:

41 + from vllm.model_executor.models.utils import WeightsMapper

42 +

43 logger = init_logger(__name__)

44

45 __all__ = ["CompressedTensorsLinearMethod"]

@@ -80,6 +83,18 @@ class CompressedTensorsConfig(QuantizationConfig):

83 def get_name(self) -> QuantizationMethods:

84 return "compressed-tensors"

85

86 + def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):

87 + self.target_scheme_map = hf_to_vllm_mapper.apply_dict(

88 + self.target_scheme_map)

89 + self.ignore = hf_to_vllm_mapper.apply_list(self.ignore)

90 + self.sparsity_scheme_map = hf_to_vllm_mapper.apply_dict(

91 + self.sparsity_scheme_map)

92 + self.sparsity_ignore_list = hf_to_vllm_mapper.apply_list(

93 + self.sparsity_ignore_list)

94 + if self.kv_cache_scheme is not None:

95 + self.kv_cache_scheme = hf_to_vllm_mapper.apply_dict(

96 + self.kv_cache_scheme)

97 +

98 def get_quant_method(

99 self,

100 layer: torch.nn.Module,

vllm/model_executor/layers/quantization/fp8.py

+9 -1

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import functools

5 -from typing import Any, Callable, Optional, Union

5 +from typing import TYPE_CHECKING, Any, Callable, Optional, Union

6

7 import torch

8 import torch.nn.functional as F

@@ -39,6 +39,9 @@ from vllm.platforms import current_platform

39 from vllm.scalar_type import scalar_types

40 from vllm.utils import has_deep_gemm

41

42 +if TYPE_CHECKING:

43 + from vllm.model_executor.models.utils import WeightsMapper

44 +

45 ACTIVATION_SCHEMES = ["static", "dynamic"]

46

47 logger = init_logger(__name__)

@@ -100,6 +103,11 @@ class Fp8Config(QuantizationConfig):

103 def get_config_filenames(cls) -> list[str]:

104 return []

105

106 + def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):

107 + if self.ignored_layers is not None:

108 + self.ignored_layers = hf_to_vllm_mapper.apply_list(

109 + self.ignored_layers)

110 +

111 @classmethod

112 def from_config(cls, config: dict[str, Any]) -> "Fp8Config":

113 quant_method = cls.get_from_keys(config, ["quant_method"])

vllm/model_executor/layers/quantization/gptq_bitblas.py

+1

@@ -81,6 +81,7 @@ class GPTQBitBLASConfig(QuantizationConfig):

81 # (since we have only one group per output channel)

82 desc_act = False

83

84 + super().__init__()

85 self.weight_bits = weight_bits

86 self.group_size = group_size

87 self.desc_act = desc_act

vllm/model_executor/layers/quantization/marlin.py

+2

@@ -32,6 +32,8 @@ class MarlinConfig(QuantizationConfig):

32 group_size: int,

33 lm_head_quantized: bool,

34 ) -> None:

35 + super().__init__()

36 +

37 # Group size for the quantization.

38 self.group_size = group_size

39 self.lm_head_quantized = lm_head_quantized

vllm/model_executor/layers/quantization/modelopt.py

+1

@@ -181,6 +181,7 @@ class ModelOptNvFp4Config(QuantizationConfig):

181 exclude_modules: list[str],

182 group_size: int = 16,

183 ) -> None:

184 + super().__init__()

185 self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized

186 if is_checkpoint_nvfp4_serialized:

187 logger.warning(

vllm/model_executor/layers/quantization/torchao.py

+1

@@ -55,6 +55,7 @@ class TorchAOConfig(QuantizationConfig):

55 os.environ["VLLM_DISABLE_COMPILE_CACHE"] = "1"

56 logger.info("Using TorchAO: Setting VLLM_DISABLE_COMPILE_CACHE=1")

57 """

58 + super().__init__()

59 self.torchao_config = torchao_config

60 self.skip_modules = skip_modules or []

61

vllm/model_executor/model_loader/utils.py

+13 -9

@@ -24,6 +24,7 @@ from vllm.model_executor.models import ModelRegistry

24 from vllm.model_executor.models.adapters import (as_classification_model,

25 as_embedding_model,

26 as_reward_model)

27 +from vllm.model_executor.models.interfaces import SupportsQuant

28 from vllm.utils import is_pin_memory_available

29

30 logger = init_logger(__name__)

@@ -294,13 +295,16 @@ def configure_quant_config(quant_config: QuantizationConfig,

295

296 Note that model attributes are passed by reference to quant_config,

297 enabling them to be updated by model_class.__new__ (ex. chatglm, qwen)

298 +

299 + Once the `SupportsQuant` mixin has been added to all models, this

300 + function can be removed

301 """

298 - packed_mapping = getattr(model_class, "packed_modules_mapping", None)

299 - if packed_mapping is not None:

300 - # pass packed_modules_mapping by reference to quant_config

301 - quant_config.packed_modules_mapping = packed_mapping

302 - else:

303 - logger.warning(

304 - "The model class %s has not defined `packed_modules_mapping`, "

305 - "this may lead to incorrect mapping of quantized or ignored "

306 - "modules", model_class.__name__)

302 + if not issubclass(model_class, SupportsQuant):

303 + hf_to_vllm_mapper = getattr(model_class, "hf_to_vllm_mapper", None)

304 + packed_mapping = getattr(model_class, "packed_modules_mapping", None)

305 +

306 + # pass mappings by reference to quant_config

307 + if hf_to_vllm_mapper is not None:

308 + quant_config.apply_vllm_mapper(hf_to_vllm_mapper)

309 + if packed_mapping is not None:

310 + quant_config.packed_modules_mapping = packed_mapping

vllm/model_executor/models/interfaces.py

+20 -3

@@ -18,6 +18,7 @@ from .interfaces_base import is_pooling_model

18

19 if TYPE_CHECKING:

20 from vllm.attention import AttentionMetadata

21 + from vllm.model_executor.models.utils import WeightsMapper

22 from vllm.sequence import IntermediateTensors

23

24 logger = init_logger(__name__)

@@ -566,20 +567,36 @@ def has_step_pooler(model: Union[type[object], object]) -> bool:

567 class SupportsQuant:

568 """The interface required for all models that support quantization."""

569

569 - packed_modules_mapping: ClassVar[dict[str, list[str]]] = {}

570 + hf_to_vllm_mapper: ClassVar[Optional["WeightsMapper"]] = None

571 + packed_modules_mapping: ClassVar[Optional[dict[str, list[str]]]] = None

572 quant_config: Optional[QuantizationConfig] = None

573

574 def __new__(cls, *args, **kwargs) -> Self:

575 instance = super().__new__(cls)

576 +

577 + # find config passed in arguments

578 quant_config = cls._find_quant_config(*args, **kwargs)

579 if quant_config is not None:

580 +

581 + # attach config to model for general use

582 instance.quant_config = quant_config

577 - instance.quant_config.packed_modules_mapping.update(

578 - cls.packed_modules_mapping)

583 +

584 + # apply model mappings to config for proper config-model matching

585 + # NOTE: `TransformersForCausalLM` is not supported due to how this

586 + # class defines `hf_to_vllm_mapper` as a post-init `@property`.

587 + # After this is fixed, get `instance.hf_to_vllm_mapper` directly

588 + if getattr(instance, "hf_to_vllm_mapper", None) is not None:

589 + instance.quant_config.apply_vllm_mapper(

590 + instance.hf_to_vllm_mapper)

591 + if getattr(instance, "packed_modules_mapping", None) is not None:

592 + instance.quant_config.packed_modules_mapping.update(

593 + instance.packed_modules_mapping)

594 +

595 return instance

596

597 @staticmethod

598 def _find_quant_config(*args, **kwargs) -> Optional[QuantizationConfig]:

599 + """Find quant config passed through model constructor args"""

600 from vllm.config import VllmConfig # avoid circular import

601

602 args_values = list(args) + list(kwargs.values())

vllm/model_executor/models/qwen2_5_vl.py

+7 -7

@@ -61,7 +61,7 @@ from vllm.sequence import IntermediateTensors

61 from vllm.transformers_utils.config import uses_mrope

62

63 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

64 - SupportsMultiModal, SupportsPP)

64 + SupportsMultiModal, SupportsPP, SupportsQuant)

65 from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder

66 from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,

67 apply_rotary_pos_emb_vision)

@@ -821,7 +821,8 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):

821 info=Qwen2_5_VLProcessingInfo,

822 dummy_inputs=Qwen2_5_VLDummyInputsBuilder)

823 class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

824 - SupportsLoRA, SupportsPP):

824 + SupportsLoRA, SupportsPP,

825 + SupportsQuant):

826

827 # To ensure correct weight loading and mapping.

828 hf_to_vllm_mapper = WeightsMapper(

@@ -837,7 +838,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

838 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

839 super().__init__()

840 config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config

840 - quant_config = vllm_config.quant_config

841 multimodal_config = vllm_config.model_config.multimodal_config

842

843 self.config = config

@@ -846,7 +846,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

846 self.visual = Qwen2_5_VisionTransformer(

847 config.vision_config,

848 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

849 - quant_config=self._maybe_ignore_quant_config(quant_config),

849 + quant_config=self._maybe_ignore_quant_config(self.quant_config),

850 prefix=maybe_prefix(prefix, "visual"),

851 )

852

@@ -859,12 +859,12 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

859 self.make_empty_intermediate_tensors = (

860 self.language_model.make_empty_intermediate_tensors)

861

862 - def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

862 + def _maybe_ignore_quant_config(self, config: Optional[QuantizationConfig]):

863 # GPTQ configs do not have a list of ignored modules, however AutoGPTQ

864 # seems to avoid vision encoder sections for some models.

865 - if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):

865 + if isinstance(config, (GPTQConfig, GPTQMarlinConfig)):

866 return None

867 - return quant_config

867 + return config

868

869 def _validate_and_reshape_mm_tensor(self, mm_input: object,

870 name: str) -> torch.Tensor:

vllm/model_executor/models/transformers.py

+1

@@ -467,6 +467,7 @@ class TransformersForCausalLM(nn.Module, SupportsQuant, SupportsLoRA,

467 # FIXME(Isotr0py): Don't use any weights mapper for Transformers backend,

468 # this makes thing complicated. We need to remove this mapper after refactor

469 # `TransformersModel` in the future.

470 + # NOTE: `SupportsQuant` can be updated after property decorator is removed

471 @property

472 def hf_to_vllm_mapper(self):

473 prefix_mapper = {

vllm/model_executor/models/utils.py

+14 -1

@@ -4,7 +4,7 @@

4 import itertools

5 from collections.abc import Iterable, Mapping

6 from dataclasses import dataclass, field

7 -from typing import Callable, Literal, Optional, Protocol, Union, overload

7 +from typing import Any, Callable, Literal, Optional, Protocol, Union, overload

8

9 import torch

10 import torch.nn as nn

@@ -64,6 +64,19 @@ class WeightsMapper:

64 return ((out_name, data) for name, data in weights

65 if (out_name := self._map_name(name)) is not None)

66

67 + def apply_list(self, values: list[str]) -> list[str]:

68 + return [

69 + out_name for name in values

70 + if (out_name := self._map_name(name)) is not None

71 + ]

72 +

73 + def apply_dict(self, values: dict[str, Any]) -> dict[str, Any]:

74 + return {

75 + out_name: value

76 + for name, value in values.items()

77 + if (out_name := self._map_name(name)) is not None

78 + }

79 +

80

81 class AutoWeightsLoader:

82 """

vllm/model_executor/utils.py

+5 -2

@@ -58,7 +58,8 @@ def _make_synced_weight_loader(original_weight_loader):

58

59

60 def get_packed_modules_mapping(model: torch.nn.Module) -> dict[str, list[str]]:

61 - parent_map = copy.deepcopy(getattr(model, "packed_modules_mapping", {}))

61 + parent_map = getattr(model, "packed_modules_mapping", None)

62 + parent_map = copy.deepcopy(parent_map) if parent_map is not None else {}

63

64 # don't infer mapping if the model has defined it explicitly.

65 if parent_map:

@@ -66,7 +67,9 @@ def get_packed_modules_mapping(model: torch.nn.Module) -> dict[str, list[str]]:

67

68 # We only check main components instead of whole model submodules

69 for child in model.children():

69 - child_map = getattr(child, "packed_modules_mapping", {})

70 + child_map = getattr(child, "packed_modules_mapping", None)

71 + child_map = copy.deepcopy(child_map) if child_map is not None else {}

72 +

73 if any((k in parent_map and parent_map[k] != v)

74 for k, v in child_map.items()):

75 raise ValueError(

[Quant] [Bugfix] Fix quantization config matching with `hf_to_vllm_mapper` (#20046)