@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 22f8a69549
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Directly use compressed-tensors for checkpoint definitions (#8909)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Michael Goin committed Oct 15, 2024 at 18:40 UTC 22f8a69549d30b9b00464141797d274fb6b7e65f
10 files changed +15 -115
requirements-common.txt
+1
@@ -31,3 +31,4 @@ pyyaml
31 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
32 setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
33 einops # Required for Qwen2-VL.
34 +compressed-tensors == 0.6.0 # required for compressed-tensors
requirements-test.txt
-1
@@ -17,7 +17,6 @@ requests
17 ray[adag]==2.35
18 sentence-transformers # required for embedding
19 soundfile # required for audio test
20 -compressed-tensors==0.4.0 # required for compressed-tensors
20 timm # required for internvl test
21 transformers_stream_generator # required for qwen-vl test
22 matplotlib # required for qwen-vl test
tests/quantization/test_compressed_tensors.py
+1 -2
@@ -6,13 +6,12 @@ from typing import Optional
6
7 import pytest
8 import torch
9 +from compressed_tensors.quantization import QuantizationType
10
11 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
12 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
13 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
14 CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
14 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 - QuantizationType)
15
16
17 @pytest.mark.parametrize(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5 -2
@@ -1,6 +1,10 @@
1 from typing import Any, Dict, List, Optional, cast
2
3 import torch
4 +from compressed_tensors.config import CompressionFormat
5 +from compressed_tensors.quantization import (QuantizationArgs,
6 + QuantizationStrategy,
7 + QuantizationType)
8 from pydantic import BaseModel
9
10 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -16,8 +20,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
20 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
21 CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
22 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19 - CompressionFormat, QuantizationArgs, QuantizationStrategy,
20 - QuantizationType, find_matched_target, is_activation_quantization_format,
23 + find_matched_target, is_activation_quantization_format,
24 should_ignore_layer)
25 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
26 from vllm.platforms import current_platform
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+2 -2
@@ -3,14 +3,14 @@ from enum import Enum
3 from typing import Callable, List, Optional
4
5 import torch
6 +from compressed_tensors import CompressionFormat
7 +from compressed_tensors.quantization import QuantizationStrategy
8
9 from vllm import _custom_ops as ops
10 from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11 FusedMoeWeightScaleSupported)
12 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13 WNA16_SUPPORTED_BITS)
12 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
13 - CompressionFormat, QuantizationStrategy)
14 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
15 all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
16 from vllm.model_executor.utils import set_weight_attrs
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+1 -2
@@ -1,11 +1,10 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from compressed_tensors.quantization import QuantizationStrategy
5
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
7 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8 - QuantizationStrategy)
8 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
9 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+1 -2
@@ -1,12 +1,11 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from compressed_tensors.quantization import QuantizationStrategy
5 from torch.nn import Parameter
6
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
8 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 - QuantizationStrategy)
9 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
10 apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
11 requantize_with_max_scale)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+1 -2
@@ -1,13 +1,12 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from compressed_tensors.quantization import QuantizationStrategy
5 from torch.nn import Parameter
6
7 from vllm.logger import init_logger
8 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9 CompressedTensorsScheme)
9 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10 - QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 apply_int8_linear, convert_to_channelwise)
12 from vllm.model_executor.parameter import (BasevLLMParameter,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1 -2
@@ -1,12 +1,11 @@
1 from typing import Callable, List, Optional, Set
2
3 import torch
4 +from compressed_tensors.quantization import ActivationOrdering
5
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
8 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 - ActivationOrdering)
9 from vllm.model_executor.layers.quantization.kernels import (
10 MPLinearLayerConfig, choose_mp_linear_kernel)
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+2 -100
@@ -1,111 +1,13 @@
1 import re
2 -from enum import Enum
3 -from typing import Any, Dict, Iterable, Optional, Union
2 +from typing import Iterable, Optional
3
5 -from pydantic import BaseModel, Field, field_validator
4 +from compressed_tensors import CompressionFormat
5 from torch.nn import Module
6
7 from vllm.model_executor.layers.quantization.utils.quant_utils import (
8 FUSED_LAYER_NAME_MAPPING)
9
10
12 -class CompressionFormat(Enum):
13 - dense = "dense"
14 - sparse_bitmask = "sparse-bitmask"
15 - naive_quantized = "naive-quantized"
16 - float_quantized = "float-quantized"
17 - int_quantized = "int-quantized"
18 - pack_quantized = "pack-quantized"
19 - marlin_24 = "marlin-24"
20 -
21 -
22 -class QuantizationType(str, Enum):
23 - """
24 - Enum storing quantization type options
25 - """
26 -
27 - INT = "int"
28 - FLOAT = "float"
29 -
30 -
31 -class QuantizationStrategy(str, Enum):
32 - """
33 - Enum storing quantization strategy options
34 - """
35 -
36 - TENSOR = "tensor"
37 - CHANNEL = "channel"
38 - GROUP = "group"
39 - BLOCK = "block"
40 - TOKEN = "token"
41 -
42 -
43 -class ActivationOrdering(str, Enum):
44 - """
45 - Enum storing strategies for activation ordering
46 -
47 - Group: reorder groups and weight\n
48 - Weight: only reorder weight, not groups. Slightly lower latency and
49 - accuracy compared to group actorder\n
50 - """
51 -
52 - GROUP = "group"
53 - WEIGHT = "weight"
54 -
55 -
56 -class QuantizationArgs(BaseModel):
57 - """
58 - User facing arguments used to define a quantization config
59 - for weights or activations
60 -
61 - :param num_bits: quantization bit depth
62 - :param type: dtype to quantized to, either int or float
63 - :param symmetric: whether or not quantization scale is symmetric
64 - :param strategy: string determining the scope of scale/zero-point to apply
65 - :param group_size: group length to use for the group strategy
66 - :param block_structure: 2d block structure to use for the block
67 - strategy, must be of the format "2x4", "8x16", etc.
68 - :param dynamic: set True to perform dynamic quantization -
69 - values will not be calibrated during calibration phase,
70 - instead during inference new quantization ranges will be
71 - observed with every sample. Defaults to False for static
72 - quantization. Note that enabling dynamic quantization
73 - will change the default observer to a memoryless one
74 - :param actorder: whether to apply group quantization in decreasing order of
75 - activation. Defaults to None for arbitrary ordering
76 - """
77 -
78 - num_bits: int = 8
79 - type: QuantizationType = QuantizationType.INT
80 - symmetric: bool = True
81 - group_size: Optional[int] = None
82 - strategy: Optional[QuantizationStrategy] = None
83 - block_structure: Optional[str] = None
84 - dynamic: bool = False
85 - actorder: Union[ActivationOrdering, bool, None] = None
86 - observer: str = Field(
87 - default="minmax",
88 - description=("The class to use to compute the quantization param - "
89 - "scale and zero-point'"),
90 - )
91 - observer_kwargs: Dict[str, Any] = Field(
92 - default_factory=dict,
93 - description=
94 - ("optional dict of kwargs to be passed directly to torch quantization "
95 - "Observers constructor excluding quantization range or symmetry"),
96 - )
97 -
98 - @field_validator("actorder", mode="before")
99 - def validate_actorder(cls, value) -> Optional[ActivationOrdering]:
100 - if isinstance(value, bool):
101 - return ActivationOrdering.GROUP if value else None
102 -
103 - if isinstance(value, str):
104 - return ActivationOrdering(value.lower())
105 -
106 - return value
107 -
108 -
11 def is_activation_quantization_format(format: str) -> bool:
12 _ACTIVATION_QUANTIZATION_FORMATS = [
13 CompressionFormat.naive_quantized.value,