A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Directly use compressed-tensors for checkpoint definitions (#8909)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Michael Goin committed
Oct 15, 2024 at 18:40 UTC
22f8a69549d30b9b00464141797d274fb6b7e65f
10 files changed
+15
-115
requirements-common.txt
+1
@@ -31,3 +31,4 @@ pyyaml
31
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
32
setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
33
einops # Required for Qwen2-VL.
34
+compressed-tensors == 0.6.0 # required for compressed-tensors
requirements-test.txt
-1
@@ -17,7 +17,6 @@ requests
17
ray[adag]==2.35
18
sentence-transformers # required for embedding
19
soundfile # required for audio test
20
-compressed-tensors==0.4.0 # required for compressed-tensors
20
timm # required for internvl test
21
transformers_stream_generator # required for qwen-vl test
22
matplotlib # required for qwen-vl test
tests/quantization/test_compressed_tensors.py
+1
-2
@@ -6,13 +6,12 @@ from typing import Optional
6
7
import pytest
8
import torch
9
+from compressed_tensors.quantization import QuantizationType
10
11
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
12
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
13
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
14
CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
14
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
- QuantizationType)
15
16
17
@pytest.mark.parametrize(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
-2
@@ -1,6 +1,10 @@
1
from typing import Any, Dict, List, Optional, cast
2
3
import torch
4
+from compressed_tensors.config import CompressionFormat
5
+from compressed_tensors.quantization import (QuantizationArgs,
6
+ QuantizationStrategy,
7
+ QuantizationType)
8
from pydantic import BaseModel
9
10
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -16,8 +20,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
20
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
21
CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
22
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19
- CompressionFormat, QuantizationArgs, QuantizationStrategy,
20
- QuantizationType, find_matched_target, is_activation_quantization_format,
23
+ find_matched_target, is_activation_quantization_format,
24
should_ignore_layer)
25
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
26
from vllm.platforms import current_platform
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+2
-2
@@ -3,14 +3,14 @@ from enum import Enum
3
from typing import Callable, List, Optional
4
5
import torch
6
+from compressed_tensors import CompressionFormat
7
+from compressed_tensors.quantization import QuantizationStrategy
8
9
from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11
FusedMoeWeightScaleSupported)
12
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13
WNA16_SUPPORTED_BITS)
12
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
13
- CompressionFormat, QuantizationStrategy)
14
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
15
all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
16
from vllm.model_executor.utils import set_weight_attrs
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+1
-2
@@ -1,11 +1,10 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from compressed_tensors.quantization import QuantizationStrategy
5
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
7
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8
- QuantizationStrategy)
8
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
9
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+1
-2
@@ -1,12 +1,11 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from compressed_tensors.quantization import QuantizationStrategy
5
from torch.nn import Parameter
6
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
8
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
- QuantizationStrategy)
9
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
10
apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
11
requantize_with_max_scale)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+1
-2
@@ -1,13 +1,12 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from compressed_tensors.quantization import QuantizationStrategy
5
from torch.nn import Parameter
6
7
from vllm.logger import init_logger
8
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9
CompressedTensorsScheme)
9
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10
- QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
apply_int8_linear, convert_to_channelwise)
12
from vllm.model_executor.parameter import (BasevLLMParameter,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1
-2
@@ -1,12 +1,11 @@
1
from typing import Callable, List, Optional, Set
2
3
import torch
4
+from compressed_tensors.quantization import ActivationOrdering
5
6
from vllm.logger import init_logger
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
8
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
- ActivationOrdering)
9
from vllm.model_executor.layers.quantization.kernels import (
10
MPLinearLayerConfig, choose_mp_linear_kernel)
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+2
-100
@@ -1,111 +1,13 @@
1
import re
2
-from enum import Enum
3
-from typing import Any, Dict, Iterable, Optional, Union
2
+from typing import Iterable, Optional
3
5
-from pydantic import BaseModel, Field, field_validator
4
+from compressed_tensors import CompressionFormat
5
from torch.nn import Module
6
7
from vllm.model_executor.layers.quantization.utils.quant_utils import (
8
FUSED_LAYER_NAME_MAPPING)
9
10
12
-class CompressionFormat(Enum):
13
- dense = "dense"
14
- sparse_bitmask = "sparse-bitmask"
15
- naive_quantized = "naive-quantized"
16
- float_quantized = "float-quantized"
17
- int_quantized = "int-quantized"
18
- pack_quantized = "pack-quantized"
19
- marlin_24 = "marlin-24"
20
-
21
-
22
-class QuantizationType(str, Enum):
23
- """
24
- Enum storing quantization type options
25
- """
26
-
27
- INT = "int"
28
- FLOAT = "float"
29
-
30
-
31
-class QuantizationStrategy(str, Enum):
32
- """
33
- Enum storing quantization strategy options
34
- """
35
-
36
- TENSOR = "tensor"
37
- CHANNEL = "channel"
38
- GROUP = "group"
39
- BLOCK = "block"
40
- TOKEN = "token"
41
-
42
-
43
-class ActivationOrdering(str, Enum):
44
- """
45
- Enum storing strategies for activation ordering
46
-
47
- Group: reorder groups and weight\n
48
- Weight: only reorder weight, not groups. Slightly lower latency and
49
- accuracy compared to group actorder\n
50
- """
51
-
52
- GROUP = "group"
53
- WEIGHT = "weight"
54
-
55
-
56
-class QuantizationArgs(BaseModel):
57
- """
58
- User facing arguments used to define a quantization config
59
- for weights or activations
60
-
61
- :param num_bits: quantization bit depth
62
- :param type: dtype to quantized to, either int or float
63
- :param symmetric: whether or not quantization scale is symmetric
64
- :param strategy: string determining the scope of scale/zero-point to apply
65
- :param group_size: group length to use for the group strategy
66
- :param block_structure: 2d block structure to use for the block
67
- strategy, must be of the format "2x4", "8x16", etc.
68
- :param dynamic: set True to perform dynamic quantization -
69
- values will not be calibrated during calibration phase,
70
- instead during inference new quantization ranges will be
71
- observed with every sample. Defaults to False for static
72
- quantization. Note that enabling dynamic quantization
73
- will change the default observer to a memoryless one
74
- :param actorder: whether to apply group quantization in decreasing order of
75
- activation. Defaults to None for arbitrary ordering
76
- """
77
-
78
- num_bits: int = 8
79
- type: QuantizationType = QuantizationType.INT
80
- symmetric: bool = True
81
- group_size: Optional[int] = None
82
- strategy: Optional[QuantizationStrategy] = None
83
- block_structure: Optional[str] = None
84
- dynamic: bool = False
85
- actorder: Union[ActivationOrdering, bool, None] = None
86
- observer: str = Field(
87
- default="minmax",
88
- description=("The class to use to compute the quantization param - "
89
- "scale and zero-point'"),
90
- )
91
- observer_kwargs: Dict[str, Any] = Field(
92
- default_factory=dict,
93
- description=
94
- ("optional dict of kwargs to be passed directly to torch quantization "
95
- "Observers constructor excluding quantization range or symmetry"),
96
- )
97
-
98
- @field_validator("actorder", mode="before")
99
- def validate_actorder(cls, value) -> Optional[ActivationOrdering]:
100
- if isinstance(value, bool):
101
- return ActivationOrdering.GROUP if value else None
102
-
103
- if isinstance(value, str):
104
- return ActivationOrdering(value.lower())
105
-
106
- return value
107
-
108
-
11
def is_activation_quantization_format(format: str) -> bool:
12
_ACTIVATION_QUANTIZATION_FORMATS = [
13
CompressionFormat.naive_quantized.value,