A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] `compressed-tensors` code reuse (#7277)
Kyle Sayers committed
Aug 13, 2024 at 19:05 UTC
373538f973ac4ea93a4b675655b893bf495ac050
8 files changed
+13
-85
requirements-common.txt
+1
@@ -23,3 +23,4 @@ pyzmq
23
librosa # Required for audio processing
24
soundfile # Required for audio processing
25
gguf == 0.9.1
26
+compressed-tensors == 0.5.0
requirements-test.txt
+1
-1
@@ -17,7 +17,7 @@ peft
17
requests
18
ray
19
sentence-transformers # required for embedding
20
-compressed-tensors==0.4.0 # required for compressed-tensors
20
+compressed-tensors==0.5.0 # required for compressed-tensors
21
timm # required for internvl test
22
23
# TODO: Add this after fully implementing llava(mantis)
tests/quantization/test_compressed_tensors.py
+1
-2
@@ -5,13 +5,12 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
5
6
import pytest
7
import torch
8
+from compressed_tensors.quantization import QuantizationType
9
10
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13
CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14
- QuantizationType)
14
15
16
@pytest.mark.parametrize("model_args", [
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
-2
@@ -1,6 +1,10 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
+from compressed_tensors.config import CompressionFormat
5
+from compressed_tensors.quantization import (QuantizationArgs,
6
+ QuantizationStrategy,
7
+ QuantizationType)
8
from pydantic import BaseModel
9
10
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
@@ -13,8 +17,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
17
CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
18
CompressedTensorsWNA16)
19
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
16
- CompressionFormat, QuantizationArgs, QuantizationStrategy,
17
- QuantizationType, find_matched_target, is_activation_quantization_format,
20
+ find_matched_target, is_activation_quantization_format,
21
should_ignore_layer)
22
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
23
from vllm.platforms import current_platform
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+1
-2
@@ -1,11 +1,10 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from compressed_tensors.quantization import QuantizationStrategy
5
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
7
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8
- QuantizationStrategy)
8
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
9
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+1
-2
@@ -1,12 +1,11 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from compressed_tensors.quantization import QuantizationStrategy
5
from torch.nn import Parameter
6
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
8
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
- QuantizationStrategy)
9
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
10
apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
11
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+1
-2
@@ -1,12 +1,11 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
+from compressed_tensors.quantization import QuantizationStrategy
5
from torch.nn import Parameter
6
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
8
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
- QuantizationStrategy)
9
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
10
apply_int8_linear, convert_to_channelwise)
11
from vllm.model_executor.parameter import (BasevLLMParameter,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+2
-74
@@ -1,85 +1,13 @@
1
import re
2
-from enum import Enum
3
-from typing import Any, Dict, Iterable, Optional
2
+from typing import Iterable, Optional
3
5
-from pydantic import BaseModel, Field
4
+from compressed_tensors import CompressionFormat
5
from torch.nn import Module
6
7
from vllm.model_executor.layers.quantization.utils.quant_utils import (
8
FUSED_LAYER_NAME_MAPPING)
9
10
12
-class CompressionFormat(Enum):
13
- dense = "dense"
14
- sparse_bitmask = "sparse-bitmask"
15
- naive_quantized = "naive-quantized"
16
- float_quantized = "float-quantized"
17
- int_quantized = "int-quantized"
18
- pack_quantized = "pack-quantized"
19
- marlin_24 = "marlin-24"
20
-
21
-
22
-class QuantizationType(str, Enum):
23
- """
24
- Enum storing quantization type options
25
- """
26
-
27
- INT = "int"
28
- FLOAT = "float"
29
-
30
-
31
-class QuantizationStrategy(str, Enum):
32
- """
33
- Enum storing quantization strategy options
34
- """
35
-
36
- TENSOR = "tensor"
37
- CHANNEL = "channel"
38
- GROUP = "group"
39
- BLOCK = "block"
40
- TOKEN = "token"
41
-
42
-
43
-class QuantizationArgs(BaseModel):
44
- """
45
- User facing arguments used to define a quantization config
46
- for weights or activations
47
-
48
- :param num_bits: quantization bit depth
49
- :param type: dtype to quantized to, either int or float
50
- :param symmetric: whether or not quantization scale is symmetric
51
- :param strategy: string determining the scope of scale/zero-point to apply
52
- :param group_size: group length to use for the group strategy
53
- :param block_structure: 2d block structure to use for the block
54
- strategy, must be of the format "2x4", "8x16", etc.
55
- :param dynamic: set True to perform dynamic quantization -
56
- values will not be calibrated during calibration phase,
57
- instead during inference new quantization ranges will be
58
- observed with every sample. Defaults to False for static
59
- quantization. Note that enabling dynamic quantization
60
- will change the default observer to a memoryless one
61
- """
62
-
63
- num_bits: int = 8
64
- type: QuantizationType = QuantizationType.INT
65
- symmetric: bool = True
66
- group_size: Optional[int] = None
67
- strategy: Optional[QuantizationStrategy] = None
68
- block_structure: Optional[str] = None
69
- dynamic: bool = False
70
- observer: str = Field(
71
- default="minmax",
72
- description=("The class to use to compute the quantization param - "
73
- "scale and zero-point'"),
74
- )
75
- observer_kwargs: Dict[str, Any] = Field(
76
- default_factory=dict,
77
- description=
78
- ("optional dict of kwargs to be passed directly to torch quantization "
79
- "Observers constructor excluding quantization range or symmetry"),
80
- )
81
-
82
-
11
def is_activation_quantization_format(format: str) -> bool:
12
_ACTIVATION_QUANTIZATION_FORMATS = [
13
CompressionFormat.naive_quantized.value,