@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 890d8d960b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] `compressed-tensors` marlin 24 support (#5435)

Dipika Sikka committed Jun 17, 2024 at 12:32 UTC 890d8d960bb441b4ac46588492db7f16b6da78d7
5 files changed +196 -19
tests/quantization/test_compressed_tensors.py
+20 -3
@@ -9,7 +9,8 @@ import torch
9 from vllm import SamplingParams
10 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11 CompressedTensorsLinearMethod, CompressedTensorsW4A16,
12 - CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor)
12 + CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
13 + CompressedTensorsW8A8StaticTensor)
14
15
16 def test_compressed_tensors_w8a8_static_setup(vllm_runner):
@@ -51,8 +52,7 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):
52
53 def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
54 model_path = "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2"
54 - with vllm_runner(model_path, enforce_eager=True,
55 - dtype=torch.float16) as llm:
55 + with vllm_runner(model_path, dtype=torch.float16) as llm:
56 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
57 layer = model.model.layers[0]
58
@@ -83,3 +83,20 @@ def test_compressed_tensors_w4a16(vllm_runner, w4a16_args):
83 assert qkv_proj.weight_packed.dtype is torch.int32
84 assert qkv_proj.weight_scale.dtype is torch.float16
85 assert qkv_proj.weight_packed.pack_factor == 8
86 +
87 +
88 +def test_compressed_tensors_w4a16_marlin24(vllm_runner):
89 + model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t"
90 + with vllm_runner(model_path) as llm:
91 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
92 + layer = model.model.layers[0]
93 +
94 + qkv_proj = layer.self_attn.qkv_proj
95 +
96 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
97 + assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
98 + assert qkv_proj.weight_packed.dtype is torch.int32
99 +
100 + sampling_params = SamplingParams()
101 + output = llm.generate("Hello world!", sampling_params=sampling_params)
102 + assert output
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+32 -16
@@ -8,16 +8,20 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8 QuantizationConfig)
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 CompressedTensorsScheme, CompressedTensorsW4A16,
11 - CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor)
11 + CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
12 + CompressedTensorsW8A8StaticTensor)
13 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
13 - QuantizationArgs, QuantizationStrategy, find_first_name_or_class_match)
14 + CompressionFormat, QuantizationArgs, QuantizationStrategy,
15 + find_first_name_or_class_match)
16
17
18 class CompressedTensorsConfig(QuantizationConfig):
19
18 - def __init__(self, layer_quant_details: Dict[str, Any], ignore: List[str]):
20 + def __init__(self, layer_quant_details: Dict[str, Any], ignore: List[str],
21 + quant_format: str):
22 self.ignore = ignore
23 self.layer_quant_details = layer_quant_details
24 + self.quant_format = quant_format
25
26 def get_linear_method(self) -> "CompressedTensorsLinearMethod":
27 return CompressedTensorsLinearMethod(self)
@@ -46,6 +50,7 @@ class CompressedTensorsConfig(QuantizationConfig):
50 def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
51 layer_quant_details: Dict[str, Any] = dict()
52 ignore: List[str] = config.get("ignore", None)
53 + quant_format: str = config.get("format", None)
54
55 # The quant_config has multiple config_groups, each containing
56 # an input_activations key with details about how the activations are
@@ -69,7 +74,9 @@ class CompressedTensorsConfig(QuantizationConfig):
74 except Exception:
75 layer_quant_details[target]["input_activations"] = None
76
72 - return cls(layer_quant_details=layer_quant_details, ignore=ignore)
77 + return cls(layer_quant_details=layer_quant_details,
78 + ignore=ignore,
79 + quant_format=quant_format)
80
81 @classmethod
82 def get_config_filenames(cls) -> List[str]:
@@ -110,17 +117,26 @@ class CompressedTensorsConfig(QuantizationConfig):
117 input_quant: BaseModel) -> "CompressedTensorsScheme":
118
119 if self._is_w4a16(weight_quant, input_quant):
113 - return CompressedTensorsW4A16(num_bits=weight_quant.num_bits,
114 - strategy=weight_quant.strategy,
115 - group_size=weight_quant.group_size)
116 -
117 - if self._is_static_tensor_w8a8(weight_quant, input_quant):
118 - return CompressedTensorsW8A8StaticTensor()
119 -
120 - if self._is_dynamic_token_w8a8(weight_quant, input_quant):
121 - return CompressedTensorsW8A8DynamicToken()
122 -
123 - raise NotImplementedError("Scheme not supported.")
120 + if self.quant_format == CompressionFormat.marlin_24.value:
121 + return CompressedTensorsW4A16Sparse24(
122 + strategy=weight_quant.strategy,
123 + num_bits=weight_quant.num_bits,
124 + group_size=weight_quant.group_size)
125 + if self.quant_format == CompressionFormat.pack_quantized.value:
126 + return CompressedTensorsW4A16(
127 + num_bits=weight_quant.num_bits,
128 + strategy=weight_quant.strategy,
129 + group_size=weight_quant.group_size)
130 +
131 + if self.quant_format == CompressionFormat.int_quantized.value:
132 + if self._is_static_tensor_w8a8(weight_quant, input_quant):
133 + return CompressedTensorsW8A8StaticTensor()
134 +
135 + if self._is_dynamic_token_w8a8(weight_quant, input_quant):
136 + return CompressedTensorsW8A8DynamicToken()
137 +
138 + raise NotImplementedError(
139 + "No compressed-tensors compatible scheme was found.")
140
141 def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
142
@@ -165,9 +181,9 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
181 scheme = self.quantization_config.get_scheme(layer=layer)
182 scheme.create_weights(
183 layer=layer,
184 + input_size=input_size,
185 input_size_per_partition=input_size_per_partition,
186 output_partition_sizes=output_partition_sizes,
170 - input_size=input_size,
187 output_size=output_size,
188 params_dtype=params_dtype,
189 weight_loader=weight_loader)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+2
@@ -2,6 +2,8 @@ from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2 from .compressed_tensors_unquantized import ( # noqa: F401
3 CompressedTensorsUnquantized)
4 from .compressed_tensors_w4a16 import CompressedTensorsW4A16 # noqa: F401
5 +from .compressed_tensors_w4a16_24 import ( # noqa: F401
6 + CompressedTensorsW4A16Sparse24)
7 from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
8 CompressedTensorsW8A8DynamicToken)
9 from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py new
+134
@@ -0,0 +1,134 @@
1 +from typing import Callable, List, Optional
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm import _custom_ops as ops
7 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 + CompressedTensorsScheme)
9 +from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10 + GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N)
11 +from vllm.model_executor.utils import set_weight_attrs
12 +
13 +__all__ = ["CompressedTensorsW4A16Sparse24"]
14 +
15 +
16 +class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
17 +
18 + def __init__(self,
19 + strategy: str,
20 + num_bits: int,
21 + group_size: Optional[int] = None):
22 + self.strategy = strategy
23 + self.group_size = group_size
24 + self.num_bits = num_bits
25 + self.tile_size = 16
26 +
27 + if self.strategy == "group" and self.group_size is None:
28 + raise ValueError(
29 + "group_size must be given when using strategy group")
30 +
31 + def create_weights(self, layer: torch.nn.Module, input_size: int,
32 + output_partition_sizes: List[int],
33 + input_size_per_partition: int,
34 + params_dtype: torch.dtype, weight_loader: Callable,
35 + **kwargs):
36 +
37 + pack_factor = 32 // self.num_bits
38 + output_size_per_partition = sum(output_partition_sizes)
39 +
40 + qweight = Parameter(
41 + torch.empty(
42 + input_size_per_partition // self.tile_size // 2,
43 + output_size_per_partition * self.tile_size // pack_factor,
44 + dtype=torch.int32,
45 + ),
46 + requires_grad=False,
47 + )
48 + set_weight_attrs(
49 + qweight,
50 + {
51 + "input_dim": 0,
52 + "output_dim": 1,
53 + "packed_dim": 1,
54 + "pack_factor": pack_factor,
55 + "marlin_tile_size": self.tile_size,
56 + "weight_loader": weight_loader
57 + },
58 + )
59 +
60 + layer.register_parameter("weight_packed", qweight)
61 +
62 + input_groups = (1 if self.group_size is None else
63 + input_size_per_partition // self.group_size)
64 +
65 + scales = Parameter(
66 + torch.empty(
67 + input_groups,
68 + output_size_per_partition,
69 + dtype=params_dtype,
70 + ),
71 + requires_grad=False,
72 + )
73 + set_weight_attrs(
74 + scales,
75 + {
76 + "output_dim": 1,
77 + "input_dim": None if input_groups == 1 else 0,
78 + "weight_loader": weight_loader
79 + },
80 + )
81 + layer.register_parameter("scale_packed", scales)
82 +
83 + weight_shape = Parameter(torch.empty(2, dtype=torch.int64),
84 + requires_grad=False)
85 +
86 + layer.register_parameter("weight_shape", weight_shape)
87 + set_weight_attrs(weight_shape, {"weight_loader": weight_loader})
88 +
89 + meta = Parameter(
90 + torch.empty(
91 + input_size_per_partition // 8 // 2 // 2,
92 + output_size_per_partition * 2,
93 + dtype=torch.int16,
94 + ),
95 + requires_grad=False,
96 + )
97 + set_weight_attrs(
98 + meta,
99 + {
100 + "input_dim": 0,
101 + "packed_dim": 1,
102 + "pack_factor": 1,
103 + "output_dim": 1,
104 + "marlin_tile_size": 2,
105 + "weight_loader": weight_loader
106 + },
107 + )
108 + layer.register_parameter("meta", meta)
109 +
110 + max_workspace_size = (
111 + output_size_per_partition //
112 + GPTQ_MARLIN_24_MIN_THREAD_N) * GPTQ_MARLIN_24_MAX_PARALLEL
113 + workspace = Parameter(torch.zeros(max_workspace_size, dtype=torch.int),
114 + requires_grad=False)
115 + layer.workspace = workspace
116 +
117 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
118 + qweight = layer.weight_packed
119 + meta = layer.meta
120 + scales = layer.scale_packed
121 + workspace = layer.workspace
122 +
123 + x_2d = x.view(-1, x.shape[-1])
124 +
125 + size_m = x_2d.shape[0]
126 + size_k = x_2d.shape[1]
127 + size_n = scales.shape[1]
128 +
129 + output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
130 + workspace, self.num_bits, size_m,
131 + size_n, size_k)
132 +
133 + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
134 + return output
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+8
@@ -6,6 +6,14 @@ from pydantic import BaseModel, Field
6 from torch.nn import Module
7
8
9 +class CompressionFormat(Enum):
10 + dense = "dense"
11 + sparse_bitmask = "sparse-bitmask"
12 + int_quantized = "int-quantized"
13 + pack_quantized = "pack-quantized"
14 + marlin_24 = "marlin-24"
15 +
16 +
17 class QuantizationType(str, Enum):
18 """
19 Enum storing quantization type options