@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f55a9aea45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Revert `compressed-tensors` code reuse (#7521)

Kyle Sayers committed Aug 14, 2024 at 18:07 UTC f55a9aea4573ed5282ee3221182993495ddc3709
8 files changed +85 -13
requirements-common.txt
-1
@@ -24,4 +24,3 @@ librosa # Required for audio processing
24 soundfile # Required for audio processing
25 gguf == 0.9.1
26 importlib_metadata
27 -compressed-tensors == 0.5.0
requirements-test.txt
+1 -1
@@ -17,7 +17,7 @@ peft
17 requests
18 ray
19 sentence-transformers # required for embedding
20 -compressed-tensors==0.5.0 # required for compressed-tensors
20 +compressed-tensors==0.4.0 # required for compressed-tensors
21 timm # required for internvl test
22
23 # TODO: Add this after fully implementing llava(mantis)
tests/quantization/test_compressed_tensors.py
+2 -1
@@ -5,12 +5,13 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
5
6 import pytest
7 import torch
8 -from compressed_tensors.quantization import QuantizationType
8
9 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
11 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
12 CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14 + QuantizationType)
15
16
17 @pytest.mark.parametrize("model_args", [
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2 -5
@@ -1,10 +1,6 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 -from compressed_tensors.config import CompressionFormat
5 -from compressed_tensors.quantization import (QuantizationArgs,
6 - QuantizationStrategy,
7 - QuantizationType)
4 from pydantic import BaseModel
5
6 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
@@ -17,7 +13,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13 CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
14 CompressedTensorsWNA16)
15 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
20 - find_matched_target, is_activation_quantization_format,
16 + CompressionFormat, QuantizationArgs, QuantizationStrategy,
17 + QuantizationType, find_matched_target, is_activation_quantization_format,
18 should_ignore_layer)
19 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
20 from vllm.platforms import current_platform
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+2 -1
@@ -1,10 +1,11 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 -from compressed_tensors.quantization import QuantizationStrategy
4
5 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
6 CompressedTensorsScheme)
7 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8 + QuantizationStrategy)
9 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
10 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
11 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+2 -1
@@ -1,11 +1,12 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 -from compressed_tensors.quantization import QuantizationStrategy
4 from torch.nn import Parameter
5
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 + QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
12 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+2 -1
@@ -1,11 +1,12 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 -from compressed_tensors.quantization import QuantizationStrategy
4 from torch.nn import Parameter
5
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 + QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 apply_int8_linear, convert_to_channelwise)
12 from vllm.model_executor.parameter import (BasevLLMParameter,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+74 -2
@@ -1,13 +1,85 @@
1 import re
2 -from typing import Iterable, Optional
2 +from enum import Enum
3 +from typing import Any, Dict, Iterable, Optional
4
4 -from compressed_tensors import CompressionFormat
5 +from pydantic import BaseModel, Field
6 from torch.nn import Module
7
8 from vllm.model_executor.layers.quantization.utils.quant_utils import (
9 FUSED_LAYER_NAME_MAPPING)
10
11
12 +class CompressionFormat(Enum):
13 + dense = "dense"
14 + sparse_bitmask = "sparse-bitmask"
15 + naive_quantized = "naive-quantized"
16 + float_quantized = "float-quantized"
17 + int_quantized = "int-quantized"
18 + pack_quantized = "pack-quantized"
19 + marlin_24 = "marlin-24"
20 +
21 +
22 +class QuantizationType(str, Enum):
23 + """
24 + Enum storing quantization type options
25 + """
26 +
27 + INT = "int"
28 + FLOAT = "float"
29 +
30 +
31 +class QuantizationStrategy(str, Enum):
32 + """
33 + Enum storing quantization strategy options
34 + """
35 +
36 + TENSOR = "tensor"
37 + CHANNEL = "channel"
38 + GROUP = "group"
39 + BLOCK = "block"
40 + TOKEN = "token"
41 +
42 +
43 +class QuantizationArgs(BaseModel):
44 + """
45 + User facing arguments used to define a quantization config
46 + for weights or activations
47 +
48 + :param num_bits: quantization bit depth
49 + :param type: dtype to quantized to, either int or float
50 + :param symmetric: whether or not quantization scale is symmetric
51 + :param strategy: string determining the scope of scale/zero-point to apply
52 + :param group_size: group length to use for the group strategy
53 + :param block_structure: 2d block structure to use for the block
54 + strategy, must be of the format "2x4", "8x16", etc.
55 + :param dynamic: set True to perform dynamic quantization -
56 + values will not be calibrated during calibration phase,
57 + instead during inference new quantization ranges will be
58 + observed with every sample. Defaults to False for static
59 + quantization. Note that enabling dynamic quantization
60 + will change the default observer to a memoryless one
61 + """
62 +
63 + num_bits: int = 8
64 + type: QuantizationType = QuantizationType.INT
65 + symmetric: bool = True
66 + group_size: Optional[int] = None
67 + strategy: Optional[QuantizationStrategy] = None
68 + block_structure: Optional[str] = None
69 + dynamic: bool = False
70 + observer: str = Field(
71 + default="minmax",
72 + description=("The class to use to compute the quantization param - "
73 + "scale and zero-point'"),
74 + )
75 + observer_kwargs: Dict[str, Any] = Field(
76 + default_factory=dict,
77 + description=
78 + ("optional dict of kwargs to be passed directly to torch quantization "
79 + "Observers constructor excluding quantization range or symmetry"),
80 + )
81 +
82 +
83 def is_activation_quantization_format(format: str) -> bool:
84 _ACTIVATION_QUANTIZATION_FORMATS = [
85 CompressionFormat.naive_quantized.value,