@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 373538f973
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] `compressed-tensors` code reuse (#7277)

Kyle Sayers committed Aug 13, 2024 at 19:05 UTC 373538f973ac4ea93a4b675655b893bf495ac050
8 files changed +13 -85
requirements-common.txt
+1
@@ -23,3 +23,4 @@ pyzmq
23 librosa # Required for audio processing
24 soundfile # Required for audio processing
25 gguf == 0.9.1
26 +compressed-tensors == 0.5.0
requirements-test.txt
+1 -1
@@ -17,7 +17,7 @@ peft
17 requests
18 ray
19 sentence-transformers # required for embedding
20 -compressed-tensors==0.4.0 # required for compressed-tensors
20 +compressed-tensors==0.5.0 # required for compressed-tensors
21 timm # required for internvl test
22
23 # TODO: Add this after fully implementing llava(mantis)
tests/quantization/test_compressed_tensors.py
+1 -2
@@ -5,13 +5,12 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
5
6 import pytest
7 import torch
8 +from compressed_tensors.quantization import QuantizationType
9
10 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
13 CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14 - QuantizationType)
14
15
16 @pytest.mark.parametrize("model_args", [
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5 -2
@@ -1,6 +1,10 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 +from compressed_tensors.config import CompressionFormat
5 +from compressed_tensors.quantization import (QuantizationArgs,
6 + QuantizationStrategy,
7 + QuantizationType)
8 from pydantic import BaseModel
9
10 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
@@ -13,8 +17,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
17 CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
18 CompressedTensorsWNA16)
19 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
16 - CompressionFormat, QuantizationArgs, QuantizationStrategy,
17 - QuantizationType, find_matched_target, is_activation_quantization_format,
20 + find_matched_target, is_activation_quantization_format,
21 should_ignore_layer)
22 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
23 from vllm.platforms import current_platform
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+1 -2
@@ -1,11 +1,10 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from compressed_tensors.quantization import QuantizationStrategy
5
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
7 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
8 - QuantizationStrategy)
8 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
9 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+1 -2
@@ -1,12 +1,11 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from compressed_tensors.quantization import QuantizationStrategy
5 from torch.nn import Parameter
6
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
8 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 - QuantizationStrategy)
9 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
10 apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
11 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+1 -2
@@ -1,12 +1,11 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 +from compressed_tensors.quantization import QuantizationStrategy
5 from torch.nn import Parameter
6
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
8 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 - QuantizationStrategy)
9 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
10 apply_int8_linear, convert_to_channelwise)
11 from vllm.model_executor.parameter import (BasevLLMParameter,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+2 -74
@@ -1,85 +1,13 @@
1 import re
2 -from enum import Enum
3 -from typing import Any, Dict, Iterable, Optional
2 +from typing import Iterable, Optional
3
5 -from pydantic import BaseModel, Field
4 +from compressed_tensors import CompressionFormat
5 from torch.nn import Module
6
7 from vllm.model_executor.layers.quantization.utils.quant_utils import (
8 FUSED_LAYER_NAME_MAPPING)
9
10
12 -class CompressionFormat(Enum):
13 - dense = "dense"
14 - sparse_bitmask = "sparse-bitmask"
15 - naive_quantized = "naive-quantized"
16 - float_quantized = "float-quantized"
17 - int_quantized = "int-quantized"
18 - pack_quantized = "pack-quantized"
19 - marlin_24 = "marlin-24"
20 -
21 -
22 -class QuantizationType(str, Enum):
23 - """
24 - Enum storing quantization type options
25 - """
26 -
27 - INT = "int"
28 - FLOAT = "float"
29 -
30 -
31 -class QuantizationStrategy(str, Enum):
32 - """
33 - Enum storing quantization strategy options
34 - """
35 -
36 - TENSOR = "tensor"
37 - CHANNEL = "channel"
38 - GROUP = "group"
39 - BLOCK = "block"
40 - TOKEN = "token"
41 -
42 -
43 -class QuantizationArgs(BaseModel):
44 - """
45 - User facing arguments used to define a quantization config
46 - for weights or activations
47 -
48 - :param num_bits: quantization bit depth
49 - :param type: dtype to quantized to, either int or float
50 - :param symmetric: whether or not quantization scale is symmetric
51 - :param strategy: string determining the scope of scale/zero-point to apply
52 - :param group_size: group length to use for the group strategy
53 - :param block_structure: 2d block structure to use for the block
54 - strategy, must be of the format "2x4", "8x16", etc.
55 - :param dynamic: set True to perform dynamic quantization -
56 - values will not be calibrated during calibration phase,
57 - instead during inference new quantization ranges will be
58 - observed with every sample. Defaults to False for static
59 - quantization. Note that enabling dynamic quantization
60 - will change the default observer to a memoryless one
61 - """
62 -
63 - num_bits: int = 8
64 - type: QuantizationType = QuantizationType.INT
65 - symmetric: bool = True
66 - group_size: Optional[int] = None
67 - strategy: Optional[QuantizationStrategy] = None
68 - block_structure: Optional[str] = None
69 - dynamic: bool = False
70 - observer: str = Field(
71 - default="minmax",
72 - description=("The class to use to compute the quantization param - "
73 - "scale and zero-point'"),
74 - )
75 - observer_kwargs: Dict[str, Any] = Field(
76 - default_factory=dict,
77 - description=
78 - ("optional dict of kwargs to be passed directly to torch quantization "
79 - "Observers constructor excluding quantization range or symmetry"),
80 - )
81 -
82 -
11 def is_activation_quantization_format(format: str) -> bool:
12 _ACTIVATION_QUANTIZATION_FORMATS = [
13 CompressionFormat.naive_quantized.value,