@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aea19f0989
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Support Models With Bias in `compressed-tensors` integration (#6356)

Robert Shaw committed Jul 12, 2024 at 11:11 UTC aea19f0989667968465087338b568694f61c6391
12 files changed +58 -21
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
2 +model_name: "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.593
8 + - name: "exact_match,flexible-extract"
9 + value: 0.588
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise -b "auto" -l 1000 -f 5 -t 1
2 +model_name: "nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.595
8 + - name: "exact_match,flexible-extract"
9 + value: 0.582
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -2,3 +2,4 @@ Meta-Llama-3-8B-Instruct.yaml
2 Meta-Llama-3-8B-Instruct-FP8.yaml
3 Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5 +Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1 -1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46 done
47
48 lm_eval --model vllm \
49 - --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true \
49 + --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray" \
50 --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 --batch_size $BATCH_SIZE
tests/models/test_compressed_tensors.py
+3
@@ -12,7 +12,10 @@ from tests.quantization.utils import is_quant_method_supported
12 from .utils import check_logprobs_close
13
14 MODELS = [
15 + # No bias
16 "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test",
17 + # Bias
18 + "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
19 ]
20
21 MAX_TOKENS = 32
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+1 -4
@@ -267,10 +267,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
267
268 """
269
270 - if bias is not None:
271 - raise ValueError("bias is not supported for this linear method")
272 -
270 scheme = layer.scheme
271 if scheme is None:
272 raise ValueError("A scheme must be defined for each layer")
276 - return scheme.apply_weights(layer, x)
273 + return scheme.apply_weights(layer, x, bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+5 -2
@@ -1,4 +1,5 @@
1 from abc import ABC, abstractmethod
2 +from typing import Optional
3
4 import torch
5
@@ -20,14 +21,16 @@ class CompressedTensorsScheme(ABC):
21 raise NotImplementedError
22
23 @abstractmethod
23 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
24 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
25 + bias: Optional[torch.Tensor]):
26 """
27 Run the forward pass for the particular scheme. This is where
28 scheme-specific dequant/quant steps/kernels should be applied.
29
28 - :param layer: toch.nn.Module with the registered weights and
30 + :param layer: torch.nn.Module with the registered weights and
31 other parameters relevant to the particular scheme.
32 :param x: input to the layer
33 + :param bias: bias parameter
34
35 """
36 raise NotImplementedError
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+5 -4
@@ -1,4 +1,4 @@
1 -from typing import Callable, List
1 +from typing import Callable, List, Optional
2
3 import torch
4 import torch.nn.functional as F
@@ -37,6 +37,7 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
37 layer.register_parameter("weight", weight)
38 set_weight_attrs(weight, {"weight_loader": weight_loader})
39
40 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
41 - weight = layer.weight
42 - return F.linear(x, weight)
40 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
41 + bias: Optional[torch.Tensor]) -> torch.Tensor:
42 +
43 + return F.linear(x, layer.weight, bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+7 -1
@@ -118,7 +118,9 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
118 requires_grad=False)
119 layer.workspace = workspace
120
121 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
121 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
122 + bias: Optional[torch.Tensor]) -> torch.Tensor:
123 +
124 qweight = layer.weight_packed
125 meta = layer.meta
126 scales = layer.scale_packed
@@ -135,4 +137,8 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
137 size_n, size_k)
138
139 output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
140 +
141 + if bias is not None:
142 + output.add_(bias) # In-place add
143 +
144 return output
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+6 -3
@@ -1,4 +1,4 @@
1 -from typing import Callable, List
1 +from typing import Callable, List, Optional
2
3 import torch
4 from torch.nn import Parameter
@@ -78,8 +78,11 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
78 **layer_kwargs)
79 layer.register_parameter("input_scale", scale)
80
81 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
81 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
82 + bias: Optional[torch.Tensor]) -> torch.Tensor:
83 +
84 return apply_int8_linear(input=x,
85 weight=layer.weight,
86 weight_scale=layer.weight_scale,
85 - input_scale=layer.input_scale)
87 + input_scale=layer.input_scale,
88 + bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+5 -2
@@ -148,7 +148,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
148 group_size=layer.group_size)
149 replace_tensor(layer, "weight_scale", marlin_scales)
150
151 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
151 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
152 + bias: Optional[torch.Tensor]) -> torch.Tensor:
153 +
154 return apply_marlin_linear(
155 input=x,
156 weight=layer.weight_packed,
@@ -159,4 +161,5 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
161 num_bits=self.num_bits,
162 output_size_per_partition=layer.output_size_per_partition,
163 input_size_per_partition=layer.input_size_per_partition,
162 - is_k_full=True)
164 + is_k_full=True,
165 + bias=bias)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2 -4
@@ -148,9 +148,6 @@ def apply_int8_linear(
148 input_scale: torch.Tensor,
149 bias: Optional[torch.Tensor] = None,
150 ):
151 - if bias is not None:
152 - raise NotImplementedError("W8A8 with int8 does not yet support bias.")
153 -
151 # ops.scaled_int8_quant supports both dynamic and static quant.
152 # * dynamic, layer.input_scale is None and x_scale computed from x.
153 # * static, layer.input_scale is scalar and x_scale is input_scale.
@@ -160,4 +157,5 @@ def apply_int8_linear(
157 weight,
158 scale_a=x_scale,
159 scale_b=weight_scale,
163 - out_dtype=input.dtype)
160 + out_dtype=input.dtype,
161 + bias=bias)