@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95db455e7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Add channel-wise quantization support for w8a8 dynamic per token activation quantization (#5542)

Dipika Sikka committed Jun 18, 2024 at 12:45 UTC 95db455e7f337e99ffafd0b14367a7cbc11dca43
4 files changed +45 -32
tests/quantization/test_compressed_tensors.py
+9 -4
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
14
15
16 def test_compressed_tensors_w8a8_static_setup(vllm_runner):
17 - model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
17 + model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
18 with vllm_runner(model_path, enforce_eager=True) as llm:
19 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
20 layer = model.model.layers[0]
@@ -43,15 +43,19 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
43
44
45 def test_compressed_tensors_no_enforce_eager(vllm_runner):
46 - model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
46 + model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
47 with vllm_runner(model_path) as llm:
48 sampling_params = SamplingParams()
49 output = llm.generate("Hello world!", sampling_params=sampling_params)
50 assert output
51
52
53 -def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
54 - model_path = "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2"
53 +@pytest.mark.parametrize("model_args", [
54 + ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
55 + ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", "channel"),
56 +])
57 +def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
58 + model_path, strategy = model_args
59 with vllm_runner(model_path, dtype=torch.float16) as llm:
60 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
61 layer = model.model.layers[0]
@@ -60,6 +64,7 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
64
65 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
66 assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
67 + assert qkv_proj.scheme.strategy == strategy
68 assert qkv_proj.weight.dtype is torch.int8
69
70
vllm/model_executor/layers/linear.py
-13
@@ -468,13 +468,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
468 "MergedColumnParallelLinear, assume the weight is "
469 "the same for all partitions.")
470
471 - if fp8_scales_shard_indexer is None:
472 - if len(param_data.shape) == 0:
473 - param_data = param_data.reshape(1)
474 -
475 - if len(loaded_weight.shape) == 0:
476 - loaded_weight = loaded_weight.reshape(1)
477 -
471 assert param_data.shape == loaded_weight.shape
472 param_data.copy_(loaded_weight)
473
@@ -686,12 +679,6 @@ class QKVParallelLinear(ColumnParallelLinear):
679 "QKVParallelLinear, assume the weight is the same "
680 "for all partitions.")
681
689 - if len(param_data.shape) == 0:
690 - param_data = param_data.reshape(1)
691 -
692 - if len(loaded_weight.shape) == 0:
693 - loaded_weight = loaded_weight.reshape(1)
694 -
682 assert param_data.shape == loaded_weight.shape
683 param_data.copy_(loaded_weight)
684
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+8 -6
@@ -95,14 +95,15 @@ class CompressedTensorsConfig(QuantizationConfig):
95 def _is_dynamic_token_w8a8(self, weight_quant: BaseModel,
96 input_quant: BaseModel) -> bool:
97 is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
98 - is_token_tensor = (weight_quant.strategy
99 - == QuantizationStrategy.TENSOR.value) and (
100 - input_quant.strategy
101 - == QuantizationStrategy.TOKEN.value)
98 + weight_strategy = (
99 + weight_quant.strategy == QuantizationStrategy.TENSOR.value
100 + or weight_quant.strategy == QuantizationStrategy.CHANNEL.value)
101 + is_token = (weight_strategy and input_quant.strategy
102 + == QuantizationStrategy.TOKEN.value)
103 is_symmetric = weight_quant.symmetric and input_quant.symmetric
104 is_dynamic = not weight_quant.dynamic and input_quant.dynamic
105
105 - return is_8_bits and is_token_tensor and is_symmetric and is_dynamic
106 + return is_8_bits and is_token and is_symmetric and is_dynamic
107
108 def _is_w4a16(self, weight_quant: BaseModel,
109 input_quant: BaseModel) -> bool:
@@ -133,7 +134,8 @@ class CompressedTensorsConfig(QuantizationConfig):
134 return CompressedTensorsW8A8StaticTensor()
135
136 if self._is_dynamic_token_w8a8(weight_quant, input_quant):
136 - return CompressedTensorsW8A8DynamicToken()
137 + return CompressedTensorsW8A8DynamicToken(
138 + strategy=weight_quant.strategy)
139
140 raise NotImplementedError(
141 "No compressed-tensors compatible scheme was found.")
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
+28 -9
@@ -6,6 +6,8 @@ from torch.nn import Parameter
6 from vllm import _custom_ops as custom_ops
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10 + QuantizationStrategy)
11 from vllm.model_executor.utils import set_weight_attrs
12
13 __all__ = ["CompressedTensorsW8A8DynamicToken"]
@@ -13,6 +15,9 @@ __all__ = ["CompressedTensorsW8A8DynamicToken"]
15
16 class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
17
18 + def __init__(self, strategy: str):
19 + self.strategy = strategy
20 +
21 def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
22 if isinstance(shard_id, int):
23 return shard_id
@@ -45,11 +50,17 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
50 # CompressedTensorsW8A8StaticTensor::create_weights for further
51 # information.
52 is_tensor_partitioned = len(output_partition_sizes) != 1
48 - weight_scale_dim = sum(
49 - output_partition_sizes) if is_tensor_partitioned else 1
53 + # when doing channel-wise quantization, number of scales
54 + # is equal to output_dim
55 + weight_scale_dim = sum(output_partition_sizes) if (
56 + is_tensor_partitioned
57 + or self.strategy == QuantizationStrategy.CHANNEL) else 1
58 +
59 + shape: Union[Tuple[int], Tuple[int, int]] = (weight_scale_dim, )
60 + if self.strategy == QuantizationStrategy.CHANNEL:
61 + shape = (weight_scale_dim, 1)
62
51 - weight_scale = Parameter(torch.empty(weight_scale_dim,
52 - dtype=torch.float32),
63 + weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
64 requires_grad=False)
65
66 weight = Parameter(torch.empty(sum(output_partition_sizes),
@@ -67,12 +78,20 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
78 })
79
80 layer.register_parameter("weight_scale", weight_scale)
70 - set_weight_attrs(
71 - weight_scale, {
72 - "weight_loader": weight_loader,
73 - "shard_splitter": self.scales_shard_splitter,
74 - "logical_widths": output_partition_sizes
81 + set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
82 +
83 + # Don't need a shard_splitter for channel-wise quantization
84 + # Use the default loading method
85 + if self.strategy == QuantizationStrategy.CHANNEL:
86 + set_weight_attrs(weight_scale, {
87 + "output_dim": 0,
88 })
89 + else:
90 + set_weight_attrs(
91 + weight_scale, {
92 + "logical_widths": output_partition_sizes,
93 + "shard_splitter": self.scales_shard_splitter,
94 + })
95
96 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
97 weight = layer.weight