A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Revert `compressed-tensors` code reuse (#7521)
Kyle Sayers committed
Aug 14, 2024 at 18:07 UTC
f55a9aea4573ed5282ee3221182993495ddc3709
8 files changed
+85
-13
requirements-common.txt
-1
@@ -24,4 +24,3 @@ librosa # Required for audio processing
24
soundfile # Required for audio processing
25
gguf == 0.9.1
26
importlib_metadata
27
-compressed-tensors == 0.5.0
requirements-test.txt
+1
-1
@@ -17,7 +17,7 @@ peft
17
requests
18
ray
19
sentence-transformers # required for embedding
20
-compressed-tensors==0.5.0 # required for compressed-tensors
20
+compressed-tensors==0.4.0 # required for compressed-tensors
21
timm # required for internvl test
22
23
# TODO: Add this after fully implementing llava(mantis)
tests/quantization/test_compressed_tensors.py
+2
-1
@@ -5,12 +5,13 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
5
6
import pytest
7
import torch
8
-from compressed_tensors.quantization import QuantizationType
8
9
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
11
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
12
CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14
+ QuantizationType)
15
16
17
@pytest.mark.parametrize("model_args", [
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2
-5
@@ -1,10 +1,6 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
-from compressed_tensors.config import CompressionFormat
5
-from compressed_tensors.quantization import (QuantizationArgs,
6
- QuantizationStrategy,
7
- QuantizationType)
4
from pydantic import BaseModel
5
6
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
@@ -17,7 +13,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13
CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
14
CompressedTensorsWNA16)
15
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
20
- find_matched_target, is_activation_quantization_format,
16
+ CompressionFormat, QuantizationArgs, QuantizationStrategy,
17
+ QuantizationType, find_matched_target, is_activation_quantization_format,
18
should_ignore_layer)
19
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
20
from vllm.platforms import current_platform
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+2
-1
@@ -1,10 +1,11 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
-from compressed_tensors.quantization import QuantizationStrategy
4
5
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
6
CompressedTensorsScheme)
7
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8
+ QuantizationStrategy)
9
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
10
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
11
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+2
-1
@@ -1,11 +1,12 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
-from compressed_tensors.quantization import QuantizationStrategy
4
from torch.nn import Parameter
5
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
8
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
+ QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
12
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+2
-1
@@ -1,11 +1,12 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
-from compressed_tensors.quantization import QuantizationStrategy
4
from torch.nn import Parameter
5
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
8
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
+ QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
apply_int8_linear, convert_to_channelwise)
12
from vllm.model_executor.parameter import (BasevLLMParameter,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+74
-2
@@ -1,13 +1,85 @@
1
import re
2
-from typing import Iterable, Optional
2
+from enum import Enum
3
+from typing import Any, Dict, Iterable, Optional
4
4
-from compressed_tensors import CompressionFormat
5
+from pydantic import BaseModel, Field
6
from torch.nn import Module
7
8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
9
FUSED_LAYER_NAME_MAPPING)
10
11
12
+class CompressionFormat(Enum):
13
+ dense = "dense"
14
+ sparse_bitmask = "sparse-bitmask"
15
+ naive_quantized = "naive-quantized"
16
+ float_quantized = "float-quantized"
17
+ int_quantized = "int-quantized"
18
+ pack_quantized = "pack-quantized"
19
+ marlin_24 = "marlin-24"
20
+
21
+
22
+class QuantizationType(str, Enum):
23
+ """
24
+ Enum storing quantization type options
25
+ """
26
+
27
+ INT = "int"
28
+ FLOAT = "float"
29
+
30
+
31
+class QuantizationStrategy(str, Enum):
32
+ """
33
+ Enum storing quantization strategy options
34
+ """
35
+
36
+ TENSOR = "tensor"
37
+ CHANNEL = "channel"
38
+ GROUP = "group"
39
+ BLOCK = "block"
40
+ TOKEN = "token"
41
+
42
+
43
+class QuantizationArgs(BaseModel):
44
+ """
45
+ User facing arguments used to define a quantization config
46
+ for weights or activations
47
+
48
+ :param num_bits: quantization bit depth
49
+ :param type: dtype to quantized to, either int or float
50
+ :param symmetric: whether or not quantization scale is symmetric
51
+ :param strategy: string determining the scope of scale/zero-point to apply
52
+ :param group_size: group length to use for the group strategy
53
+ :param block_structure: 2d block structure to use for the block
54
+ strategy, must be of the format "2x4", "8x16", etc.
55
+ :param dynamic: set True to perform dynamic quantization -
56
+ values will not be calibrated during calibration phase,
57
+ instead during inference new quantization ranges will be
58
+ observed with every sample. Defaults to False for static
59
+ quantization. Note that enabling dynamic quantization
60
+ will change the default observer to a memoryless one
61
+ """
62
+
63
+ num_bits: int = 8
64
+ type: QuantizationType = QuantizationType.INT
65
+ symmetric: bool = True
66
+ group_size: Optional[int] = None
67
+ strategy: Optional[QuantizationStrategy] = None
68
+ block_structure: Optional[str] = None
69
+ dynamic: bool = False
70
+ observer: str = Field(
71
+ default="minmax",
72
+ description=("The class to use to compute the quantization param - "
73
+ "scale and zero-point'"),
74
+ )
75
+ observer_kwargs: Dict[str, Any] = Field(
76
+ default_factory=dict,
77
+ description=
78
+ ("optional dict of kwargs to be passed directly to torch quantization "
79
+ "Observers constructor excluding quantization range or symmetry"),
80
+ )
81
+
82
+
83
def is_activation_quantization_format(format: str) -> bool:
84
_ACTIVATION_QUANTIZATION_FORMATS = [
85
CompressionFormat.naive_quantized.value,