A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] Support Models With Bias in `compressed-tensors` integration (#6356)
Robert Shaw committed
Jul 12, 2024 at 11:11 UTC
aea19f0989667968465087338b568694f61c6391
12 files changed
+58
-21
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
2
+model_name: "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.593
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.588
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise -b "auto" -l 1000 -f 5 -t 1
2
+model_name: "nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.595
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.582
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -2,3 +2,4 @@ Meta-Llama-3-8B-Instruct.yaml
2
Meta-Llama-3-8B-Instruct-FP8.yaml
3
Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5
+Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46
done
47
48
lm_eval --model vllm \
49
- --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true \
49
+ --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray" \
50
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51
--batch_size $BATCH_SIZE
tests/models/test_compressed_tensors.py
+3
@@ -12,7 +12,10 @@ from tests.quantization.utils import is_quant_method_supported
12
from .utils import check_logprobs_close
13
14
MODELS = [
15
+ # No bias
16
"nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test",
17
+ # Bias
18
+ "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
19
]
20
21
MAX_TOKENS = 32
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+1
-4
@@ -267,10 +267,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
267
268
"""
269
270
- if bias is not None:
271
- raise ValueError("bias is not supported for this linear method")
272
-
270
scheme = layer.scheme
271
if scheme is None:
272
raise ValueError("A scheme must be defined for each layer")
276
- return scheme.apply_weights(layer, x)
273
+ return scheme.apply_weights(layer, x, bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+5
-2
@@ -1,4 +1,5 @@
1
from abc import ABC, abstractmethod
2
+from typing import Optional
3
4
import torch
5
@@ -20,14 +21,16 @@ class CompressedTensorsScheme(ABC):
21
raise NotImplementedError
22
23
@abstractmethod
23
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
24
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
25
+ bias: Optional[torch.Tensor]):
26
"""
27
Run the forward pass for the particular scheme. This is where
28
scheme-specific dequant/quant steps/kernels should be applied.
29
28
- :param layer: toch.nn.Module with the registered weights and
30
+ :param layer: torch.nn.Module with the registered weights and
31
other parameters relevant to the particular scheme.
32
:param x: input to the layer
33
+ :param bias: bias parameter
34
35
"""
36
raise NotImplementedError
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+5
-4
@@ -1,4 +1,4 @@
1
-from typing import Callable, List
1
+from typing import Callable, List, Optional
2
3
import torch
4
import torch.nn.functional as F
@@ -37,6 +37,7 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
37
layer.register_parameter("weight", weight)
38
set_weight_attrs(weight, {"weight_loader": weight_loader})
39
40
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
41
- weight = layer.weight
42
- return F.linear(x, weight)
40
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
41
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
42
+
43
+ return F.linear(x, layer.weight, bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+7
-1
@@ -118,7 +118,9 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
118
requires_grad=False)
119
layer.workspace = workspace
120
121
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
121
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
122
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
123
+
124
qweight = layer.weight_packed
125
meta = layer.meta
126
scales = layer.scale_packed
@@ -135,4 +137,8 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
137
size_n, size_k)
138
139
output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
140
+
141
+ if bias is not None:
142
+ output.add_(bias) # In-place add
143
+
144
return output
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+6
-3
@@ -1,4 +1,4 @@
1
-from typing import Callable, List
1
+from typing import Callable, List, Optional
2
3
import torch
4
from torch.nn import Parameter
@@ -78,8 +78,11 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
78
**layer_kwargs)
79
layer.register_parameter("input_scale", scale)
80
81
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
81
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
82
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
83
+
84
return apply_int8_linear(input=x,
85
weight=layer.weight,
86
weight_scale=layer.weight_scale,
85
- input_scale=layer.input_scale)
87
+ input_scale=layer.input_scale,
88
+ bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+5
-2
@@ -148,7 +148,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
148
group_size=layer.group_size)
149
replace_tensor(layer, "weight_scale", marlin_scales)
150
151
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
151
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
152
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
153
+
154
return apply_marlin_linear(
155
input=x,
156
weight=layer.weight_packed,
@@ -159,4 +161,5 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
161
num_bits=self.num_bits,
162
output_size_per_partition=layer.output_size_per_partition,
163
input_size_per_partition=layer.input_size_per_partition,
162
- is_k_full=True)
164
+ is_k_full=True,
165
+ bias=bias)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
-4
@@ -148,9 +148,6 @@ def apply_int8_linear(
148
input_scale: torch.Tensor,
149
bias: Optional[torch.Tensor] = None,
150
):
151
- if bias is not None:
152
- raise NotImplementedError("W8A8 with int8 does not yet support bias.")
153
-
151
# ops.scaled_int8_quant supports both dynamic and static quant.
152
# * dynamic, layer.input_scale is None and x_scale computed from x.
153
# * static, layer.input_scale is scalar and x_scale is input_scale.
@@ -160,4 +157,5 @@ def apply_int8_linear(
157
weight,
158
scale_a=x_scale,
159
scale_b=weight_scale,
163
- out_dtype=input.dtype)
160
+ out_dtype=input.dtype,
161
+ bias=bias)