12913d17ba · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/layers/quantization/aqlm.py

+1

@@ -169,6 +169,7 @@ class AQLMConfig(QuantizationConfig):

169 num_codebooks: int,

170 out_group_size: int,

171 ) -> None:

172 + super().__init__()

173 self.in_group_size = in_group_size

174 self.nbits_per_codebook = nbits_per_codebook

175 self.num_codebooks = num_codebooks

vllm/model_executor/layers/quantization/awq.py

+1

@@ -26,6 +26,7 @@ class AWQConfig(QuantizationConfig):

26 zero_point: bool,

27 modules_to_not_convert: Optional[List[str]] = None,

28 ) -> None:

29 + super().__init__()

30 self.weight_bits = weight_bits

31 self.group_size = group_size

32 self.zero_point = zero_point

vllm/model_executor/layers/quantization/awq_marlin.py

+1

@@ -47,6 +47,7 @@ class AWQMarlinConfig(QuantizationConfig):

47 lm_head_quantized: bool,

48 modules_to_not_convert: Optional[List[str]],

49 full_config: Dict[str, Any]) -> None:

50 + super().__init__()

51 self.pack_factor = 32 // weight_bits # packed into int32

52 self.group_size = group_size

53 self.zero_point = zero_point

vllm/model_executor/layers/quantization/base_config.py

+6 -2

@@ -2,7 +2,7 @@

2

3 import inspect

4 from abc import ABC, abstractmethod

5 -from typing import Any, Dict, List, Mapping, Optional, Type

5 +from typing import Any, Dict, List, Optional, Type

6

7 import torch

8 from torch import nn

@@ -59,7 +59,11 @@ def method_has_implemented_embedding(

59

60 class QuantizationConfig(ABC):

61 """Base class for quantization configs."""

62 - packed_modules_mapping: Mapping[str, List[str]] = dict()

62 +

63 + def __init__(self):

64 + super().__init__()

65 + # mapping is updated by models as they initialize

66 + self.packed_modules_mapping: Dict[str, List[str]] = dict()

67

68 @abstractmethod

69 def get_name(self) -> str:

vllm/model_executor/layers/quantization/bitsandbytes.py

+1 -1

@@ -30,7 +30,7 @@ class BitsAndBytesConfig(QuantizationConfig):

30 llm_int8_skip_modules: Optional[List[str]] = None,

31 llm_int8_threshold: float = 6.0,

32 ) -> None:

33 -

33 + super().__init__()

34 self.load_in_8bit = load_in_8bit

35 self.load_in_4bit = load_in_4bit

36 self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py

+1 -1

@@ -51,7 +51,7 @@ class CompressedTensorsConfig(QuantizationConfig):

51 kv_cache_scheme: Optional[Dict[str, Any]] = None,

52 config: Optional[Dict[str, Any]] = None,

53 ):

54 -

54 + super().__init__()

55 self.ignore = ignore

56 self.quant_format = quant_format

57 # Map from [target -> scheme]

vllm/model_executor/layers/quantization/deepspeedfp.py

+1

@@ -25,6 +25,7 @@ class DeepSpeedFPConfig(QuantizationConfig):

25 weight_bits: int = 8,

26 group_size: int = 512,

27 ) -> None:

28 + super().__init__()

29 self.weight_bits = weight_bits

30 self.group_size = group_size

31 self.valid_types = [torch.bfloat16, torch.float16]

vllm/model_executor/layers/quantization/experts_int8.py

+1 -1

@@ -17,7 +17,7 @@ class ExpertsInt8Config(QuantizationConfig):

17 """Config class for Int8 experts quantization."""

18

19 def __init__(self) -> None:

20 - pass

20 + super().__init__()

21

22 @classmethod

23 def get_name(cls) -> str:

vllm/model_executor/layers/quantization/fbgemm_fp8.py

+1

@@ -29,6 +29,7 @@ class FBGEMMFp8Config(QuantizationConfig):

29 """Config class for FBGEMM Fp8."""

30

31 def __init__(self, ignore_list: List[str], input_scale_ub: float):

32 + super().__init__()

33 self.ignore_list = ignore_list if ignore_list else []

34 self.input_scale_ub = input_scale_ub

35

vllm/model_executor/layers/quantization/fp8.py

+1

@@ -47,6 +47,7 @@ class Fp8Config(QuantizationConfig):

47 ignored_layers: Optional[List[str]] = None,

48 weight_block_size: Optional[List[int]] = None,

49 ) -> None:

50 + super().__init__()

51 self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized

52 if is_checkpoint_fp8_serialized:

53 logger.warning("Detected fp8 checkpoint. Please note that the "

vllm/model_executor/layers/quantization/gguf.py

+1 -1

@@ -20,7 +20,7 @@ class GGUFConfig(QuantizationConfig):

20 """Config class for GGUF."""

21

22 def __init__(self, ) -> None:

23 - pass

23 + super().__init__()

24

25 def __repr__(self) -> str:

26 return ("GGUFConfig()")

vllm/model_executor/layers/quantization/gptq.py

+1

@@ -58,6 +58,7 @@ class GPTQConfig(QuantizationConfig):

58 # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},

59 # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers

60 # }

61 + super().__init__()

62 self.dynamic = dynamic

63

64 self.weight_bits = weight_bits

vllm/model_executor/layers/quantization/gptq_marlin.py

+1

@@ -46,6 +46,7 @@ class GPTQMarlinConfig(QuantizationConfig):

46 is_sym: bool, lm_head_quantized: bool,

47 dynamic: Dict[str, Dict[str, Union[int, bool]]],

48 full_config: Dict[str, Any]) -> None:

49 + super().__init__()

50 if desc_act and group_size == -1:

51 # In this case, act_order == True is the same as act_order == False

52 # (since we have only one group per output channel)

vllm/model_executor/layers/quantization/gptq_marlin_24.py

+1

@@ -38,6 +38,7 @@ class GPTQMarlin24Config(QuantizationConfig):

38 weight_bits: int,

39 group_size: int,

40 ) -> None:

41 + super().__init__()

42 quant_type = {

43 4: scalar_types.uint4b8,

44 8: scalar_types.uint8b128,

vllm/model_executor/layers/quantization/hqq_marlin.py

+1

@@ -33,6 +33,7 @@ class HQQMarlinConfig(QuantizationConfig):

33 group_size: int,

34 skip_modules: Optional[List[str]] = None,

35 ) -> None:

36 + super().__init__()

37 assert group_size == 64, ("The only supported HQQ group size is "

38 "currently 64.")

39 assert weight_bits == 4, ("The only supported HQQ quantization "

vllm/model_executor/layers/quantization/ipex_quant.py

+1

@@ -35,6 +35,7 @@ class IPEXConfig(QuantizationConfig):

35 desc_act: Optional[bool] = None,

36 lm_head_quantized: Optional[bool] = None,

37 ) -> None:

38 + super().__init__()

39 self.method = method

40 self.weight_bits = weight_bits

41 self.group_size = group_size

vllm/model_executor/layers/quantization/modelopt.py

+1

@@ -28,6 +28,7 @@ class ModelOptFp8Config(QuantizationConfig):

28 self,

29 is_checkpoint_fp8_serialized: bool = False,

30 ) -> None:

31 + super().__init__()

32 self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized

33 if is_checkpoint_fp8_serialized:

34 logger.warning("Detected ModelOpt fp8 checkpoint. Please note that"

vllm/model_executor/layers/quantization/moe_wna16.py

+1

@@ -24,6 +24,7 @@ class MoeWNA16Config(QuantizationConfig):

24 group_size: int, has_zp: bool, lm_head_quantized: bool,

25 modules_to_not_convert: Optional[List[str]],

26 full_config: Dict[str, Any]) -> None:

27 + super().__init__()

28 self.weight_bits = weight_bits

29 self.group_size = group_size

30 self.has_zp = has_zp

vllm/model_executor/layers/quantization/neuron_quant.py

+1

@@ -20,6 +20,7 @@ class NeuronQuantConfig(QuantizationConfig):

20 dequant_dtype: str = "f16",

21 quantize_method: str = "vector_dynamic",

22 ) -> None:

23 + super().__init__()

24 self.quant_dtype = os.getenv("NEURON_QUANT_DTYPE", "s8")

25 if self.quant_dtype not in SUPPORTED_QUANT_DTYPE_LIST:

26 raise ValueError(

vllm/model_executor/layers/quantization/qqq.py

+1

@@ -39,6 +39,7 @@ class QQQConfig(QuantizationConfig):

39 group_size: int,

40 is_sym: bool = True,

41 ) -> None:

42 + super().__init__()

43 self.weight_bits = weight_bits

44 self.group_size = group_size

45 self.is_sym = is_sym

vllm/model_executor/layers/quantization/quark/quark.py

+1

@@ -30,6 +30,7 @@ class QuarkConfig(QuantizationConfig):

30 kv_cache_group: Optional[List[str]] = None,

31 kv_cache_config: Optional[Dict[str, Any]] = None,

32 pack_method: str = "reorder"):

33 + super().__init__()

34 if kv_cache_group is None:

35 kv_cache_group = []

36 self.quant_config = quant_config

vllm/model_executor/layers/quantization/tpu_int8.py

+1

@@ -21,6 +21,7 @@ class Int8TpuConfig(QuantizationConfig):

21 self,

22 activation_scheme: str = "none",

23 ) -> None:

24 + super().__init__()

25 if activation_scheme not in ACTIVATION_SCHEMES:

26 raise ValueError(

27 f"Unsupported activation scheme {activation_scheme}")

vllm/model_executor/models/clip.py

+3 -2

@@ -15,6 +15,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

15 RowParallelLinear)

16 from vllm.model_executor.layers.quantization import QuantizationConfig

17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

18 +from vllm.model_executor.models.interfaces import SupportsQuant

19

20 from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs

21

@@ -335,10 +336,10 @@ class CLIPVisionTransformer(nn.Module):

336 return encoder_outputs

337

338

338 -class CLIPVisionModel(nn.Module):

339 -

339 +class CLIPVisionModel(nn.Module, SupportsQuant):

340 config_class = CLIPVisionConfig

341 main_input_name = "pixel_values"

342 + packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}

343

344 def __init__(

345 self,

vllm/model_executor/models/interfaces.py

+32

@@ -7,6 +7,8 @@ import torch

7 from typing_extensions import TypeIs, TypeVar

8

9 from vllm.logger import init_logger

10 +from vllm.model_executor.layers.quantization.base_config import (

11 + QuantizationConfig)

12 from vllm.utils import supports_kw

13

14 from .interfaces_base import is_pooling_model

@@ -443,6 +445,36 @@ def supports_cross_encoding(

445 return is_pooling_model(model) and _supports_cross_encoding(model)

446

447

448 +class SupportsQuant:

449 + """The interface required for all models that support quantization."""

450 +

451 + packed_modules_mapping: ClassVar[Dict[str, List[str]]] = {}

452 + quant_config: Optional[QuantizationConfig] = None

453 +

454 + def __new__(cls, *args, **kwargs) -> "SupportsQuant":

455 + instance = super().__new__(cls)

456 + quant_config = cls._find_quant_config(*args, **kwargs)

457 + if quant_config is not None:

458 + instance.quant_config = quant_config

459 + instance.quant_config.packed_modules_mapping.update(

460 + cls.packed_modules_mapping)

461 + return instance

462 +

463 + @staticmethod

464 + def _find_quant_config(*args, **kwargs) -> Optional[QuantizationConfig]:

465 + from vllm.config import VllmConfig # avoid circular import

466 +

467 + args_values = list(args) + list(kwargs.values())

468 + for arg in args_values:

469 + if isinstance(arg, VllmConfig):

470 + return arg.quant_config

471 +

472 + if isinstance(arg, QuantizationConfig):

473 + return arg

474 +

475 + return None

476 +

477 +

478 @runtime_checkable

479 class SupportsTranscription(Protocol):

480 """The interface required for all models that support transcription."""

vllm/model_executor/models/phi3v.py

+5 -5

@@ -50,7 +50,7 @@ from vllm.sequence import IntermediateTensors

50 from vllm.utils import is_list_of

51

52 from .clip import CLIPVisionModel

53 -from .interfaces import SupportsMultiModal, SupportsPP

53 +from .interfaces import SupportsMultiModal, SupportsPP, SupportsQuant

54 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

55 init_vllm_registered_model, maybe_prefix,

56 merge_multimodal_embeddings)

@@ -498,7 +498,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

498 @MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor,

499 info=Phi3VProcessingInfo,

500 dummy_inputs=Phi3VDummyInputsBuilder)

501 -class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

501 +class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

502 + SupportsQuant):

503 hf_to_vllm_mapper = WeightsMapper(

504 orig_to_new_prefix={

505 "model.vision_embed_tokens.wte": "embed_tokens",

@@ -510,7 +511,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

511 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

512 super().__init__()

513 config = vllm_config.model_config.hf_config

513 - quant_config = vllm_config.quant_config

514 multimodal_config = vllm_config.model_config.multimodal_config

515 self.config = config

516 self.multimodal_config = multimodal_config

@@ -520,14 +520,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

520 config.vocab_size,

521 config.hidden_size,

522 org_num_embeddings=config.vocab_size,

523 - quant_config=quant_config,

523 + quant_config=self.quant_config,

524 prefix=maybe_prefix(prefix, "model.embed_tokens"),

525 )

526

527 # TODO: Optionally initializes this for supporting input embeddings.

528 self.vision_embed_tokens = Phi3HDImageEmbedding(

529 config,

530 - quant_config,

530 + self.quant_config,

531 prefix=maybe_prefix(prefix, "model.vision_embed_tokens"))

532

533 self.language_model = init_vllm_registered_model(

[Quant] Add `SupportsQuant` to phi3 and clip (#13104)