A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Implement AWQ quantization support for LLaMA (#1032)
Co-authored-by: Robert Irvine <robert@seamlessml.com> Co-authored-by: root <rirv938@gmail.com> Co-authored-by: Casper <casperbh.96@gmail.com> Co-authored-by: julian-q <julianhquevedo@gmail.com>
Woosuk Kwon committed
Sep 16, 2023 at 00:03 UTC
e3e79e9e8a2224e03a711c3d1ef7a35daa447083
19 files changed
+1178
-208
.gitignore
+4
@@ -173,3 +173,7 @@ cython_debug/
173
174
# Sphinx documentation
175
_build/
176
+
177
+# vim swap files
178
+*.swo
179
+*.swp
benchmarks/benchmark_latency.py
+14
-4
@@ -18,6 +18,7 @@ def main(args: argparse.Namespace):
18
llm = LLM(
19
model=args.model,
20
tokenizer=args.tokenizer,
21
+ quantization=args.quantization,
22
tensor_parallel_size=args.tensor_parallel_size,
23
max_num_seqs=args.batch_size,
24
max_num_batched_tokens=args.batch_size * args.input_len,
@@ -63,19 +64,28 @@ def main(args: argparse.Namespace):
64
if __name__ == '__main__':
65
parser = argparse.ArgumentParser(
66
description='Benchmark the latency of processing a single batch of '
66
- 'requests till completion.')
67
+ 'requests till completion.')
68
parser.add_argument('--model', type=str, default='facebook/opt-125m')
69
parser.add_argument('--tokenizer', type=str, default=None)
70
+ parser.add_argument('--quantization',
71
+ '-q',
72
+ choices=['awq', None],
73
+ default=None)
74
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
75
parser.add_argument('--input-len', type=int, default=32)
76
parser.add_argument('--output-len', type=int, default=128)
77
parser.add_argument('--batch-size', type=int, default=8)
73
- parser.add_argument('--n', type=int, default=1,
78
+ parser.add_argument('--n',
79
+ type=int,
80
+ default=1,
81
help='Number of generated sequences per prompt.')
82
parser.add_argument('--use-beam-search', action='store_true')
76
- parser.add_argument('--num-iters', type=int, default=3,
83
+ parser.add_argument('--num-iters',
84
+ type=int,
85
+ default=3,
86
help='Number of iterations to run.')
78
- parser.add_argument('--trust-remote-code', action='store_true',
87
+ parser.add_argument('--trust-remote-code',
88
+ action='store_true',
89
help='trust remote code from huggingface')
90
args = parser.parse_args()
91
main(args)
benchmarks/benchmark_throughput.py
+44
-30
@@ -3,7 +3,7 @@ import argparse
3
import json
4
import random
5
import time
6
-from typing import List, Tuple
6
+from typing import List, Optional, Tuple
7
8
import torch
9
from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
@@ -22,15 +22,10 @@ def sample_requests(
22
with open(dataset_path) as f:
23
dataset = json.load(f)
24
# Filter out the conversations with less than 2 turns.
25
- dataset = [
26
- data for data in dataset
27
- if len(data["conversations"]) >= 2
28
- ]
25
+ dataset = [data for data in dataset if len(data["conversations"]) >= 2]
26
# Only keep the first two turns of each conversation.
30
- dataset = [
31
- (data["conversations"][0]["value"], data["conversations"][1]["value"])
32
- for data in dataset
33
- ]
27
+ dataset = [(data["conversations"][0]["value"],
28
+ data["conversations"][1]["value"]) for data in dataset]
29
30
# Tokenize the prompts and completions.
31
prompts = [prompt for prompt, _ in dataset]
@@ -63,6 +58,7 @@ def run_vllm(
58
requests: List[Tuple[str, int, int]],
59
model: str,
60
tokenizer: str,
61
+ quantization: Optional[str],
62
tensor_parallel_size: int,
63
seed: int,
64
n: int,
@@ -72,6 +68,7 @@ def run_vllm(
68
llm = LLM(
69
model=model,
70
tokenizer=tokenizer,
71
+ quantization=quantization,
72
tensor_parallel_size=tensor_parallel_size,
73
seed=seed,
74
trust_remote_code=trust_remote_code,
@@ -111,8 +108,8 @@ def run_hf(
108
trust_remote_code: bool,
109
) -> float:
110
assert not use_beam_search
114
- llm = AutoModelForCausalLM.from_pretrained(model,
115
- torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
111
+ llm = AutoModelForCausalLM.from_pretrained(
112
+ model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
113
if llm.config.model_type == "llama":
114
# To enable padding in the HF backend.
115
tokenizer.pad_token = tokenizer.eos_token
@@ -132,13 +129,14 @@ def run_hf(
129
if len(batch) < max_batch_size and i != len(requests) - 1:
130
# Check if we can add more requests to the batch.
131
_, next_prompt_len, next_output_len = requests[i + 1]
135
- if (max(max_prompt_len, next_prompt_len) + max(
136
- max_output_len, next_output_len)) <= 2048:
132
+ if (max(max_prompt_len, next_prompt_len) +
133
+ max(max_output_len, next_output_len)) <= 2048:
134
# We can add more requests to the batch.
135
continue
136
137
# Generate the sequences.
141
- input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids
138
+ input_ids = tokenizer(batch, return_tensors="pt",
139
+ padding=True).input_ids
140
llm_outputs = llm.generate(
141
input_ids=input_ids.cuda(),
142
do_sample=not use_beam_search,
@@ -165,44 +163,58 @@ def main(args: argparse.Namespace):
163
random.seed(args.seed)
164
165
# Sample the requests.
168
- tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
166
+ tokenizer = get_tokenizer(args.tokenizer,
167
+ trust_remote_code=args.trust_remote_code)
168
requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
169
170
if args.backend == "vllm":
172
- elapsed_time = run_vllm(
173
- requests, args.model, args.tokenizer, args.tensor_parallel_size,
174
- args.seed, args.n, args.use_beam_search, args.trust_remote_code)
171
+ elapsed_time = run_vllm(requests, args.model, args.tokenizer,
172
+ args.quantization, args.tensor_parallel_size,
173
+ args.seed, args.n, args.use_beam_search,
174
+ args.trust_remote_code)
175
elif args.backend == "hf":
176
assert args.tensor_parallel_size == 1
177
- elapsed_time = run_hf(
178
- requests, args.model, tokenizer, args.n, args.use_beam_search,
179
- args.hf_max_batch_size, args.trust_remote_code)
177
+ elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
178
+ args.use_beam_search, args.hf_max_batch_size,
179
+ args.trust_remote_code)
180
else:
181
raise ValueError(f"Unknown backend: {args.backend}")
182
- total_num_tokens = sum(
183
- prompt_len + output_len
184
- for _, prompt_len, output_len in requests
185
- )
182
+ total_num_tokens = sum(prompt_len + output_len
183
+ for _, prompt_len, output_len in requests)
184
print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
185
f"{total_num_tokens / elapsed_time:.2f} tokens/s")
186
187
188
if __name__ == "__main__":
189
parser = argparse.ArgumentParser(description="Benchmark the throughput.")
192
- parser.add_argument("--backend", type=str, choices=["vllm", "hf"],
190
+ parser.add_argument("--backend",
191
+ type=str,
192
+ choices=["vllm", "hf"],
193
default="vllm")
194
- parser.add_argument("--dataset", type=str, required=True,
194
+ parser.add_argument("--dataset",
195
+ type=str,
196
+ required=True,
197
help="Path to the dataset.")
198
parser.add_argument("--model", type=str, default="facebook/opt-125m")
199
parser.add_argument("--tokenizer", type=str, default=None)
200
+ parser.add_argument('--quantization',
201
+ '-q',
202
+ choices=['awq', None],
203
+ default=None)
204
parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
199
- parser.add_argument("--n", type=int, default=1,
205
+ parser.add_argument("--n",
206
+ type=int,
207
+ default=1,
208
help="Number of generated sequences per prompt.")
209
parser.add_argument("--use-beam-search", action="store_true")
202
- parser.add_argument("--num-prompts", type=int, default=1000,
210
+ parser.add_argument("--num-prompts",
211
+ type=int,
212
+ default=1000,
213
help="Number of prompts to process.")
214
parser.add_argument("--seed", type=int, default=0)
205
- parser.add_argument("--hf-max-batch-size", type=int, default=None,
215
+ parser.add_argument("--hf-max-batch-size",
216
+ type=int,
217
+ default=None,
218
help="Maximum batch size for HF backend.")
219
parser.add_argument('--trust-remote-code',
220
action='store_true',
@@ -215,6 +227,8 @@ if __name__ == "__main__":
227
elif args.backend == "hf":
228
if args.hf_max_batch_size is None:
229
raise ValueError("HF max batch size is required for HF backend.")
230
+ if args.quantization is not None:
231
+ raise ValueError("Quantization is only for vLLM backend.")
232
if args.tokenizer is None:
233
args.tokenizer = args.model
234
csrc/quantization.cpp
new
+15
@@ -0,0 +1,15 @@
1
+#include <torch/extension.h>
2
+
3
+torch::Tensor awq_gemm(
4
+ torch::Tensor _in_feats,
5
+ torch::Tensor _kernel,
6
+ torch::Tensor _scaling_factors,
7
+ torch::Tensor _zeros,
8
+ int split_k_iters);
9
+
10
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
11
+ m.def(
12
+ "awq_gemm",
13
+ &awq_gemm,
14
+ "Quantized GEMM for AWQ");
15
+}
csrc/quantization/awq/dequantize.cuh
new
+79
@@ -0,0 +1,79 @@
1
+/*
2
+Adapted from https://github.com/mit-han-lab/llm-awq
3
+Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
4
+@article{lin2023awq,
5
+ title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
6
+ author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
7
+ journal={arXiv},
8
+ year={2023}
9
+}
10
+*/
11
+
12
+#pragma once
13
+
14
+
15
+__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
16
+{
17
+ uint4 result;
18
+
19
+ uint32_t* h = reinterpret_cast<uint32_t*>(&result);
20
+ uint32_t const i4s = reinterpret_cast<uint32_t const&>(source);
21
+
22
+ // First, we extract the i4s and construct an intermediate fp16 number.
23
+ static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa;
24
+ static constexpr uint32_t BOTTOM_MASK = 0x000f000f;
25
+ static constexpr uint32_t TOP_MASK = 0x00f000f0;
26
+ static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400;
27
+
28
+ // Note that the entire sequence only requires 1 shift instruction. This is thanks to the register packing
29
+ // format and the fact that we force our integers to be unsigned, and account for this in the fp16 subtractions.
30
+ // In addition, I exploit the fact that sub and fma have the same throughput in order to convert elt_23 and
31
+ // elt_67 to fp16 without having to shift them to the bottom bits before hand.
32
+
33
+ // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW dependency if we issue
34
+ // immediately before required.
35
+ const uint32_t top_i4s = i4s >> 8;
36
+ // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400
37
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
38
+ : "=r"(h[0])
39
+ : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
40
+ // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400
41
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
42
+ : "=r"(h[1])
43
+ : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
44
+ // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400
45
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
46
+ : "=r"(h[2])
47
+ : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
48
+ // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400
49
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
50
+ : "=r"(h[3])
51
+ : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
52
+
53
+ // I use inline PTX below because I am not sure if the compiler will emit float2half instructions if I use the
54
+ // half2 ctor. In this case, I chose performance reliability over code readability.
55
+
56
+ // This is the half2 {1032, 1032} represented as an integer.
57
+ // static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408;
58
+ // Haotian: subtract {1024, 1024} instead, we do not need to map to [-8, 7]
59
+ static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400;
60
+ // This is the half2 {1 / 16, 1 / 16} represented as an integer.
61
+ static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00;
62
+ // This is the half2 {-72, -72} represented as an integer.
63
+ // static constexpr uint32_t NEG_72 = 0xd480d480;
64
+ // Haotian: Let's use {-64, -64}.
65
+ static constexpr uint32_t NEG_64 = 0xd400d400;
66
+
67
+ // Finally, we construct the output numbers.
68
+ // Convert elt_01
69
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM));
70
+ // Convert elt_23
71
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
72
+ // Convert elt_45
73
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM));
74
+ // Convert elt_67
75
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
76
+
77
+ return result;
78
+}
79
+
csrc/quantization/awq/gemm_kernels.cu
new
+477
@@ -0,0 +1,477 @@
1
+/*
2
+Adapted from https://github.com/mit-han-lab/llm-awq
3
+@article{lin2023awq,
4
+ title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
5
+ author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
6
+ journal={arXiv},
7
+ year={2023}
8
+}
9
+ */
10
+
11
+
12
+#include <torch/extension.h>
13
+#include <c10/cuda/CUDAGuard.h>
14
+
15
+#include "dequantize.cuh"
16
+
17
+#include <cuda_fp16.h>
18
+
19
+// Pack two half values.
20
+static inline __device__ __host__ unsigned
21
+__pack_half2(const half x, const half y) {
22
+ unsigned v0 = *((unsigned short *)&x);
23
+ unsigned v1 = *((unsigned short *)&y);
24
+ return (v1 << 16) | v0;
25
+}
26
+
27
+__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
28
+{
29
+ static constexpr uint32_t ZERO = 0x0;
30
+ float C_warp[32];
31
+ __shared__ half A_shared[16 * (32 + 8)];
32
+ __shared__ half B_shared[32 * (128 + 8)];
33
+
34
+ __shared__ half scaling_factors_shared[128];
35
+ __shared__ half zeros_shared[128];
36
+
37
+ int j_factors1 = ((OC + 128 - 1) / 128);
38
+ int blockIdx_x = 0;
39
+ int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
40
+ int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
41
+
42
+ half A_shared_warp[8];
43
+ half B_shared_warp[32];
44
+ for (int j_0_4_init = 0; j_0_4_init < 4; ++j_0_4_init) {
45
+ for (int i = 0; i < 8; ++i) {
46
+ C_warp[(j_0_4_init * 8) + i] = 0.0;
47
+ }
48
+ }
49
+
50
+ static constexpr int row_stride_warp = 32 * 8 / 32;
51
+ static constexpr int row_stride = 2 * 32 * 8 / 128;
52
+ bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 128;
53
+ // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
54
+ bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
55
+ // bool wb_C_flag = (threadIdx.x / 4) < M;
56
+
57
+ half* A_ptr = A
58
+ + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
59
+ + (((int)threadIdx.x) % (32 / 8)) * 8;
60
+
61
+ int* B_ptr = B
62
+ + ((int)threadIdx.y) * (OC / 8) * 2
63
+ + (((int)threadIdx.x) / (128 / 8)) * (OC / 8)
64
+ + (((int)blockIdx_y) % j_factors1) * (128 / 8)
65
+ + (((int)threadIdx.x) % (128 / 8)) * 1;
66
+// Why * 1 in the above line?
67
+
68
+ half* A_shared_ptr = A_shared
69
+ + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
70
+ + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
71
+ + (((int)threadIdx.x) % (32 / 8) ) * 8;
72
+
73
+ half* B_shared_ptr = B_shared
74
+ + ((int)threadIdx.y) * (row_stride / 2) * (128 + 8)
75
+ + (((int)threadIdx.x) / (128 / 8)) * (128 + 8)
76
+ + (((int)threadIdx.x) % (128 / 8)) * 8;
77
+
78
+ int* zeros_ptr = zeros
79
+ + (((int)blockIdx_y) % j_factors1) * (128 / 8)
80
+ + ((int)threadIdx.x) % (128 / 8);
81
+
82
+ half* scaling_factors_ptr = scaling_factors
83
+ + (((int)blockIdx_y) % j_factors1) * (128)
84
+ + (((int)threadIdx.x) % (128 / 8)) * 8;
85
+
86
+ half* C_ptr = C
87
+ + blockIdx_z * M * OC // blockIdz.x -> split_k dim
88
+ + (((int)blockIdx_y) % j_factors1) * 128
89
+ + ((int)threadIdx.y) * 64
90
+ + (((int)threadIdx.x) % 4) * 2;
91
+
92
+ // preload s.f. and zeros
93
+ int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
94
+ if ((k_bound - 1) * split_k_iters * 32 + blockIdx_z * 32 >= IC) k_bound -= 1;
95
+ for (int _k_0_0 = 0; _k_0_0 < k_bound; ++_k_0_0) {
96
+ int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
97
+ __syncthreads();
98
+ // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
99
+ if (ld_A_flag)
100
+ {
101
+ *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
102
+ }
103
+ else
104
+ {
105
+ *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
106
+ }
107
+
108
+ // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
109
+ uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
110
+ uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
111
+ uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
112
+ /*
113
+ if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
114
+ printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
115
+ }
116
+ */
117
+ // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
118
+ int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
119
+
120
+ for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 8; ++ax0_ax1_fused_0) {
121
+
122
+ // B: 32 x 136 (128+8) float16
123
+ // each warp: 32 x 4
124
+ // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
125
+ // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
126
+ // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
127
+ uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
128
+ uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
129
+ //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
130
+
131
+ // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
132
+ // - zero and * scale
133
+ // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
134
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
135
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
136
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
137
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
138
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
139
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
140
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
141
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
142
+ /*
143
+ if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
144
+ printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
145
+ }
146
+ */
147
+
148
+ // write back
149
+ *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (128 + 8)) = B_loaded_fp16;
150
+ }
151
+ __syncthreads();
152
+
153
+ for (int k_0_1 = 0; k_0_1 < 2; ++k_0_1) {
154
+ {
155
+ unsigned int addr;
156
+ __asm__ __volatile__(
157
+ "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
158
+ : "=r"(addr)
159
+ : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
160
+ );
161
+
162
+
163
+ __asm__ __volatile__(
164
+ "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
165
+ "{%0, %1, %2, %3}, [%4];\n"
166
+ : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
167
+ : "r"(addr)
168
+ );
169
+ }
170
+
171
+ for (int ax1_0 = 0; ax1_0 < 4; ++ax1_0) {
172
+ {
173
+ unsigned int addr;
174
+ __asm__ __volatile__(
175
+ "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
176
+ : "=r"(addr)
177
+ : "l"((void *)((&(B_shared[(((k_0_1 * 2176) + (((int)threadIdx.y) * 64)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 136) + ((((int)threadIdx.x) >> 4) * 8))))
178
+ );
179
+ __asm__ __volatile__(
180
+ "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
181
+ "{%0, %1, %2, %3}, [%4];\n"
182
+ : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
183
+ : "r"(addr)
184
+ );
185
+ }
186
+ }
187
+ for (int j_0_4 = 0; j_0_4 < 4; ++j_0_4) {
188
+ {
189
+ __asm__ __volatile__(
190
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
191
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
192
+ : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
193
+ : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
194
+ }
195
+
196
+ {
197
+ __asm__ __volatile__(
198
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
199
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
200
+ : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
201
+ : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
202
+ }
203
+ }
204
+ }
205
+ }
206
+
207
+// TODO: Shang: Hoist loop invariance.
208
+ for (int ax1_0_1 = 0; ax1_0_1 < 4; ++ax1_0_1) {
209
+ for (int local_id = 0; local_id < 8; ++local_id) {
210
+ int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
211
+ if (row_offset < M)
212
+ {
213
+ *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
214
+ }
215
+ }
216
+ }
217
+}
218
+
219
+
220
+__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
221
+{
222
+ static constexpr uint32_t ZERO = 0x0;
223
+ float C_warp[32];
224
+ __shared__ half A_shared[16 * (32 + 8)];
225
+ __shared__ half B_shared[32 * (64 + 8)];
226
+
227
+ __shared__ half scaling_factors_shared[64];
228
+ __shared__ half zeros_shared[64];
229
+
230
+ int j_factors1 = ((OC + 64 - 1) / 64);
231
+
232
+ int blockIdx_x = 0;
233
+ int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
234
+ int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
235
+
236
+ half A_shared_warp[8];
237
+ half B_shared_warp[16];
238
+ for (int j_0_4_init = 0; j_0_4_init < 2; ++j_0_4_init) {
239
+ for (int i = 0; i < 8; ++i) {
240
+ C_warp[(j_0_4_init * 8) + i] = 0.0;
241
+ }
242
+ }
243
+
244
+ static constexpr int row_stride_warp = 32 * 8 / 32;
245
+ static constexpr int row_stride = 2 * 32 * 8 / 64;
246
+ bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 64;
247
+ // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
248
+ bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
249
+ // bool wb_C_flag = (threadIdx.x / 4) < M;
250
+
251
+ half* A_ptr = A
252
+ + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
253
+ + (((int)threadIdx.x) % (32 / 8)) * 8;
254
+
255
+ int* B_ptr = B
256
+ + ((int)threadIdx.y) * (OC / 8) * 4
257
+ + (((int)threadIdx.x) / (64 / 8)) * (OC / 8)
258
+ + (((int)blockIdx_y) % j_factors1) * (64 / 8)
259
+ + (((int)threadIdx.x) % (64 / 8)) * 1;
260
+// Why * 1 in the above line?
261
+
262
+ half* A_shared_ptr = A_shared
263
+ + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
264
+ + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
265
+ + (((int)threadIdx.x) % (32 / 8) ) * 8;
266
+
267
+ half* B_shared_ptr = B_shared
268
+ + ((int)threadIdx.y) * (row_stride / 2) * (64 + 8)
269
+ + (((int)threadIdx.x) / (64 / 8)) * (64 + 8)
270
+ + (((int)threadIdx.x) % (64 / 8)) * 8;
271
+
272
+ int* zeros_ptr = zeros
273
+ + (((int)blockIdx_y) % j_factors1) * (64 / 8)
274
+ + ((int)threadIdx.x) % (64 / 8);
275
+
276
+ half* scaling_factors_ptr = scaling_factors
277
+ + (((int)blockIdx_y) % j_factors1) * (64)
278
+ + (((int)threadIdx.x) % (64 / 8)) * 8;
279
+
280
+ half* C_ptr = C
281
+ + blockIdx_z * M * OC // blockIdz.x -> split_k dim
282
+ + (((int)blockIdx_y) % j_factors1) * 64
283
+ + ((int)threadIdx.y) * 32
284
+ + (((int)threadIdx.x) % 4) * 2;
285
+
286
+ // preload s.f. and zeros
287
+ int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
288
+ if ((k_bound - 1) * split_k_iters * 32 + blockIdx_z * 32 >= IC) k_bound -= 1;
289
+ for (int _k_0_0 = 0; _k_0_0 < k_bound; ++_k_0_0) {
290
+ int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
291
+ __syncthreads();
292
+ // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
293
+ if (ld_A_flag)
294
+ {
295
+ *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
296
+ }
297
+ else
298
+ {
299
+ *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
300
+ }
301
+
302
+ // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
303
+ uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
304
+ uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
305
+ uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
306
+ /*
307
+ if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
308
+ printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
309
+ }
310
+ */
311
+ // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
312
+ int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
313
+
314
+ for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 4; ++ax0_ax1_fused_0) {
315
+
316
+ // B: 32 x 136 (128+8) float16
317
+ // each warp: 32 x 4
318
+ // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
319
+ // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
320
+ // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
321
+ uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
322
+ uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
323
+ //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
324
+
325
+ // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
326
+ // - zero and * scale
327
+ // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
328
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
329
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
330
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
331
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
332
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
333
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
334
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
335
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
336
+ /*
337
+ if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
338
+ printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
339
+ }
340
+ */
341
+
342
+ // write back
343
+ *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (64 + 8)) = B_loaded_fp16;
344
+ }
345
+ __syncthreads();
346
+
347
+ for (int k_0_1 = 0; k_0_1 < 2; ++k_0_1)
348
+ {
349
+ {
350
+ unsigned int addr;
351
+ __asm__ __volatile__(
352
+ "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
353
+ : "=r"(addr)
354
+ : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
355
+ );
356
+ __asm__ __volatile__(
357
+ "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
358
+ "{%0, %1, %2, %3}, [%4];\n"
359
+ : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
360
+ : "r"(addr)
361
+ );
362
+ }
363
+
364
+
365
+ for (int ax1_0 = 0; ax1_0 < 2; ++ax1_0)
366
+ {
367
+ {
368
+ unsigned int addr;
369
+ __asm__ __volatile__(
370
+ "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
371
+ : "=r"(addr)
372
+ : "l"((void *)((&(B_shared[(((k_0_1 * 1152) + (((int)threadIdx.y) * 32)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 72) + ((((int)threadIdx.x) >> 4) * 8))))
373
+ );
374
+ __asm__ __volatile__(
375
+ "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
376
+ "{%0, %1, %2, %3}, [%4];\n"
377
+ : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
378
+ : "r"(addr)
379
+ );
380
+ }
381
+ }
382
+
383
+ for (int j_0_4 = 0; j_0_4 < 2; ++j_0_4)
384
+ {
385
+
386
+ {
387
+ __asm__ __volatile__(
388
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
389
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
390
+ : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
391
+ : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
392
+ }
393
+
394
+ {
395
+ __asm__ __volatile__(
396
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
397
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
398
+ : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
399
+ : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
400
+ }
401
+ }
402
+ }
403
+ }
404
+
405
+// TODO: Shang: Hoist loop invariance.
406
+ for (int ax1_0_1 = 0; ax1_0_1 < 2; ++ax1_0_1) {
407
+ for (int local_id = 0; local_id < 8; ++local_id) {
408
+ int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
409
+ if (row_offset < M)
410
+ {
411
+ *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
412
+ }
413
+ }
414
+ }
415
+}
416
+
417
+// in_feats: M, IC [float16]
418
+// kernel: IC, OC // 8 [int32] -> cast to IC, OC [uint4b]
419
+// scaling_factors: IC // G, OC [float16]
420
+// zeros: IC // G, OC // 8 [int32] -> cast to IC // G, OC [uint4b]
421
+// assume that batch_size < 16 for now
422
+
423
+torch::Tensor awq_gemm(
424
+ torch::Tensor _in_feats,
425
+ torch::Tensor _kernel,
426
+ torch::Tensor _scaling_factors,
427
+ torch::Tensor _zeros,
428
+ int split_k_iters)
429
+{
430
+ int num_in_feats = _in_feats.size(0);
431
+ int num_in_channels = _in_feats.size(1);
432
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
433
+
434
+ auto options = torch::TensorOptions().dtype(_in_feats.dtype()).device(_in_feats.device());
435
+ at::Tensor _out_feats = torch::empty({split_k_iters, num_in_feats, _kernel.size(1) * 8}, options);
436
+ int num_out_feats = _out_feats.size(-2);
437
+ int num_out_channels = _out_feats.size(-1);
438
+
439
+ auto in_feats = reinterpret_cast<half*>(_in_feats.data_ptr<at::Half>());
440
+ auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
441
+ auto out_feats = reinterpret_cast<half*>(_out_feats.data_ptr<at::Half>());
442
+ auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
443
+ auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
444
+ int group_size = num_in_channels / _scaling_factors.size(0);
445
+
446
+ if (num_out_channels % 64 != 0)
447
+ throw std::invalid_argument("OC is not multiple of cta_N = 64");
448
+ if (num_out_channels % 8 != 0)
449
+ throw std::invalid_argument("OC is not multiple of pack_num = 8");
450
+ if (group_size % 32 != 0)
451
+ throw std::invalid_argument("Group size should be a multiple of 32");
452
+ if (num_out_channels % group_size != 0)
453
+ throw std::invalid_argument("OC is not multiple of Group size");
454
+
455
+ if (num_out_channels % 128 == 0)
456
+ {
457
+ int j_factors1 = num_out_channels / 128 / 1;
458
+ dim3 num_blocks((num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
459
+ // threadIdx.x: 32
460
+ // threadIdx.y: i_factors[2] * j_factors[2]
461
+ dim3 threads_per_block(32, 2);
462
+ gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
463
+ group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
464
+ }
465
+ else if (num_out_channels % 64 == 0)
466
+ {
467
+ int j_factors1 = num_out_channels / 64 / 1;
468
+ dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
469
+
470
+ // threadIdx.x: 32
471
+ // threadIdx.y: i_factors[2] * j_factors[2]
472
+ dim3 threads_per_block(32, 2);
473
+ gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
474
+ group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
475
+ }
476
+ return _out_feats.sum(0);
477
+}
setup.py
+14
@@ -146,6 +146,20 @@ activation_extension = CUDAExtension(
146
)
147
ext_modules.append(activation_extension)
148
149
+# Quantization kernels.
150
+quantization_extension = CUDAExtension(
151
+ name="vllm.quantization_ops",
152
+ sources=[
153
+ "csrc/quantization.cpp",
154
+ "csrc/quantization/awq/gemm_kernels.cu",
155
+ ],
156
+ extra_compile_args={
157
+ "cxx": CXX_FLAGS,
158
+ "nvcc": NVCC_FLAGS,
159
+ },
160
+)
161
+ext_modules.append(quantization_extension)
162
+
163
164
def get_path(*filepath) -> str:
165
return os.path.join(ROOT_DIR, *filepath)
vllm/config.py
+16
@@ -43,6 +43,8 @@ class ModelConfig:
43
version.
44
max_model_len: Maximum length of a sequence (including prompt and
45
output). If None, will be derived from the model.
46
+ quantization: Quantization method that was used to quantize the model
47
+ weights. If None, we assume the model weights are not quantized.
48
"""
49
50
def __init__(
@@ -57,6 +59,7 @@ class ModelConfig:
59
seed: int,
60
revision: Optional[str],
61
max_model_len: Optional[int] = None,
62
+ quantization: Optional[str] = None,
63
) -> None:
64
self.model = model
65
self.tokenizer = tokenizer
@@ -66,11 +69,13 @@ class ModelConfig:
69
self.load_format = load_format
70
self.seed = seed
71
self.revision = revision
72
+ self.quantization = quantization
73
74
self.hf_config = get_config(model, trust_remote_code, revision)
75
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
76
self._verify_load_format()
77
self._verify_tokenizer_mode()
78
+ self._verify_quantization()
79
self.max_model_len = None
80
if max_model_len is not None:
81
derived_max_model_len = self.get_max_model_len()
@@ -100,6 +105,17 @@ class ModelConfig:
105
"either 'auto' or 'slow'.")
106
self.tokenizer_mode = tokenizer_mode
107
108
+ def _verify_quantization(self) -> None:
109
+ supported_quantization = ["awq"]
110
+ if self.quantization is None:
111
+ return
112
+ quantization = self.quantization.lower()
113
+ if quantization not in supported_quantization:
114
+ raise ValueError(
115
+ f"Unknown quantization: {self.quantization}. Must be one of "
116
+ f"{supported_quantization}.")
117
+ self.quantization = quantization
118
+
119
def verify_with_parallel_config(
120
self,
121
parallel_config: "ParallelConfig",
vllm/engine/arg_utils.py
+9
-3
@@ -29,6 +29,7 @@ class EngineArgs:
29
max_num_seqs: int = 256
30
disable_log_stats: bool = False
31
revision: Optional[str] = None
32
+ quantization: Optional[str] = None
33
34
def __post_init__(self):
35
if self.tokenizer is None:
@@ -88,7 +89,6 @@ class EngineArgs:
89
'a numpy cache to speed up the loading. '
90
'"dummy" will initialize the weights with random values, '
91
'which is mainly for profiling.')
91
- # TODO(woosuk): Support FP32.
92
parser.add_argument(
93
'--dtype',
94
type=str,
@@ -150,6 +150,13 @@ class EngineArgs:
150
parser.add_argument('--disable-log-stats',
151
action='store_true',
152
help='disable logging statistics')
153
+ # Quantization settings.
154
+ parser.add_argument('--quantization',
155
+ '-q',
156
+ type=str,
157
+ choices=['awq', None],
158
+ default=None,
159
+ help='Method used to quantize the weights')
160
return parser
161
162
@classmethod
@@ -163,12 +170,11 @@ class EngineArgs:
170
def create_engine_configs(
171
self,
172
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
166
- # Initialize the configs.
173
model_config = ModelConfig(self.model, self.tokenizer,
174
self.tokenizer_mode, self.trust_remote_code,
175
self.download_dir, self.load_format,
176
self.dtype, self.seed, self.revision,
171
- self.max_model_len)
177
+ self.max_model_len, self.quantization)
178
cache_config = CacheConfig(self.block_size,
179
self.gpu_memory_utilization,
180
self.swap_space)
vllm/engine/llm_engine.py
+1
@@ -80,6 +80,7 @@ class LLMEngine:
80
f"download_dir={model_config.download_dir!r}, "
81
f"load_format={model_config.load_format}, "
82
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
83
+ f"quantization={model_config.quantization}, "
84
f"seed={model_config.seed})")
85
# TODO(woosuk): Print more configs in debug mode.
86
vllm/model_executor/layers/quantized_linear/__init__.py
new
+37
@@ -0,0 +1,37 @@
1
+from vllm.model_executor.layers.quantized_linear.awq import (
2
+ AWQColumnParallelLinear, AWQRowParallelLinear)
3
+from vllm.model_executor.parallel_utils.tensor_parallel import (
4
+ ColumnParallelLinear, RowParallelLinear)
5
+
6
+_QUANTIZED_LINEAR_REGISTRY = {
7
+ "awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
8
+}
9
+
10
+
11
+class ParallelLinear:
12
+
13
+ @classmethod
14
+ def column(cls, *args, **kwargs) -> ColumnParallelLinear:
15
+ quant_config = kwargs.get("quant_config", None)
16
+ if quant_config is None:
17
+ return ColumnParallelLinear(*args, **kwargs)
18
+
19
+ name = quant_config.get_name()
20
+ if name not in _QUANTIZED_LINEAR_REGISTRY:
21
+ raise ValueError(f"No quantized linear is found for {name}")
22
+
23
+ quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][0]
24
+ return quant_linear_cls(*args, **kwargs)
25
+
26
+ @classmethod
27
+ def row(cls, *args, **kwargs) -> RowParallelLinear:
28
+ quant_config = kwargs.get("quant_config", None)
29
+ if quant_config is None:
30
+ return RowParallelLinear(*args, **kwargs)
31
+
32
+ name = quant_config.get_name()
33
+ if name not in _QUANTIZED_LINEAR_REGISTRY:
34
+ raise ValueError(f"No quantized linear is found for {name}")
35
+
36
+ quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][1]
37
+ return quant_linear_cls(*args, **kwargs)
vllm/model_executor/layers/quantized_linear/awq.py
new
+102
@@ -0,0 +1,102 @@
1
+from typing import Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import quantization_ops
7
+from vllm.model_executor.parallel_utils.tensor_parallel.layers import (
8
+ ColumnParallelLinear, RowParallelLinear)
9
+
10
+
11
+class AWQColumnParallelLinear(ColumnParallelLinear):
12
+
13
+ def create_weights(self, dtype: torch.dtype) -> None:
14
+ assert self.input_size % self.quant_config.weight_bits == 0
15
+ assert (self.output_size_per_partition %
16
+ self.quant_config.pack_factor == 0)
17
+ self.qweight = Parameter(
18
+ torch.empty(
19
+ self.input_size,
20
+ self.output_size_per_partition //
21
+ self.quant_config.pack_factor,
22
+ device="cuda",
23
+ dtype=torch.int32,
24
+ ),
25
+ requires_grad=False,
26
+ )
27
+ self.qzeros = Parameter(
28
+ torch.empty(
29
+ self.input_size // self.quant_config.group_size,
30
+ self.output_size_per_partition //
31
+ self.quant_config.pack_factor,
32
+ device="cuda",
33
+ dtype=torch.int32,
34
+ ),
35
+ requires_grad=False,
36
+ )
37
+ self.scales = Parameter(
38
+ torch.empty(
39
+ self.input_size // self.quant_config.group_size,
40
+ self.output_size_per_partition,
41
+ device="cuda",
42
+ dtype=dtype,
43
+ ),
44
+ requires_grad=False,
45
+ )
46
+
47
+ def apply_weights(
48
+ self,
49
+ x: torch.Tensor,
50
+ bias: Optional[torch.Tensor],
51
+ ) -> torch.Tensor:
52
+ pack_factor = self.quant_config.pack_factor
53
+ out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
54
+ reshaped_x = x.reshape(-1, x.shape[-1])
55
+ out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
56
+ self.qzeros, pack_factor)
57
+ if bias is not None:
58
+ out = out + bias
59
+ return out.reshape(out_shape)
60
+
61
+
62
+class AWQRowParallelLinear(RowParallelLinear):
63
+
64
+ def create_weights(self, dtype: torch.dtype) -> None:
65
+ assert (self.input_size_per_partition %
66
+ self.quant_config.weight_bits == 0)
67
+ assert self.output_size % self.quant_config.pack_factor == 0
68
+ self.qweight = Parameter(
69
+ torch.empty(
70
+ self.input_size_per_partition,
71
+ self.output_size // self.quant_config.pack_factor,
72
+ device="cuda",
73
+ dtype=torch.int32,
74
+ ),
75
+ requires_grad=False,
76
+ )
77
+ self.qzeros = Parameter(
78
+ torch.empty(
79
+ self.input_size_per_partition // self.quant_config.group_size,
80
+ self.output_size // self.quant_config.pack_factor,
81
+ device="cuda",
82
+ dtype=torch.int32,
83
+ ),
84
+ requires_grad=False,
85
+ )
86
+ self.scales = Parameter(
87
+ torch.empty(
88
+ self.input_size_per_partition // self.quant_config.group_size,
89
+ self.output_size,
90
+ device="cuda",
91
+ dtype=dtype,
92
+ ),
93
+ requires_grad=False,
94
+ )
95
+
96
+ def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
97
+ pack_factor = self.quant_config.pack_factor
98
+ out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
99
+ reshaped_x = x.reshape(-1, x.shape[-1])
100
+ out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
101
+ self.qzeros, pack_factor)
102
+ return out.reshape(out_shape)
vllm/model_executor/model_loader.py
+28
-2
@@ -8,7 +8,8 @@ from transformers import PretrainedConfig
8
9
from vllm.config import ModelConfig
10
from vllm.model_executor.models import * # pylint: disable=wildcard-import
11
-from vllm.model_executor.weight_utils import initialize_dummy_weights
11
+from vllm.model_executor.weight_utils import (get_quant_config,
12
+ initialize_dummy_weights)
13
14
# TODO(woosuk): Lazy-load the model classes.
15
_MODEL_REGISTRY = {
@@ -30,6 +31,11 @@ _MODEL_REGISTRY = {
31
"RWForCausalLM": FalconForCausalLM,
32
}
33
34
+# FIXME(woosuk): Remove this once all models support quantization.
35
+_MODEL_CLASSES_SUPPORT_QUANTIZATION = [
36
+ LlamaForCausalLM,
37
+]
38
+
39
40
@contextlib.contextmanager
41
def _set_default_torch_dtype(dtype: torch.dtype):
@@ -52,10 +58,30 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
58
59
def get_model(model_config: ModelConfig) -> nn.Module:
60
model_class = _get_model_architecture(model_config.hf_config)
61
+
62
+ # Get the quantization config.
63
+ quant_config = None
64
+ if model_config.quantization is not None:
65
+ if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
66
+ raise ValueError(
67
+ f"Quantization is not supported for {model_class}.")
68
+ quant_config = get_quant_config(model_config.quantization,
69
+ model_config.model,
70
+ model_config.download_dir)
71
+ supported_dtypes = quant_config.get_supported_act_dtypes()
72
+ if model_config.dtype not in supported_dtypes:
73
+ raise ValueError(
74
+ f"{model_config.dtype} is not supported for quantization "
75
+ f"method {model_config.quantization}. Supported dtypes: "
76
+ f"{supported_dtypes}")
77
+
78
with _set_default_torch_dtype(model_config.dtype):
79
# Create a model instance.
80
# The weights will be initialized as empty tensors.
58
- model = model_class(model_config.hf_config)
81
+ if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
82
+ model = model_class(model_config.hf_config, quant_config)
83
+ else:
84
+ model = model_class(model_config.hf_config)
85
if model_config.load_format == "dummy":
86
model = model.cuda()
87
# NOTE(woosuk): For accurate performance evaluation, we assign
vllm/model_executor/models/llama.py
+91
-34
@@ -36,13 +36,15 @@ from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38
from vllm.model_executor.layers.sampler import Sampler
39
-from vllm.model_executor.weight_utils import (
40
- load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
41
- hf_model_weights_iterator)
39
+from vllm.model_executor.layers.quantized_linear import ParallelLinear
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
from vllm.model_executor.parallel_utils.tensor_parallel import (
45
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
43
+ VocabParallelEmbedding)
44
+from vllm.model_executor.quantization_utils import QuantizationConfig
45
+from vllm.model_executor.weight_utils import (
46
+ load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
47
+ hf_model_weights_iterator)
48
from vllm.sequence import SamplerOutput
49
50
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -55,18 +57,21 @@ class LlamaMLP(nn.Module):
57
hidden_size: int,
58
intermediate_size: int,
59
hidden_act: str,
58
- ):
60
+ quant_config: Optional[QuantizationConfig] = None,
61
+ ) -> None:
62
super().__init__()
60
- self.gate_up_proj = ColumnParallelLinear(hidden_size,
61
- 2 * intermediate_size,
62
- bias=False,
63
- gather_output=False,
64
- perform_initialization=False)
65
- self.down_proj = RowParallelLinear(intermediate_size,
66
- hidden_size,
67
- bias=False,
68
- input_is_parallel=True,
69
- perform_initialization=False)
63
+ self.gate_up_proj = ParallelLinear.column(hidden_size,
64
+ 2 * intermediate_size,
65
+ bias=False,
66
+ gather_output=False,
67
+ perform_initialization=False,
68
+ quant_config=quant_config)
69
+ self.down_proj = ParallelLinear.row(intermediate_size,
70
+ hidden_size,
71
+ bias=False,
72
+ input_is_parallel=True,
73
+ perform_initialization=False,
74
+ quant_config=quant_config)
75
if hidden_act != "silu":
76
raise ValueError(f"Unsupported activation: {hidden_act}. "
77
"Only silu is supported for now.")
@@ -87,7 +92,8 @@ class LlamaAttention(nn.Module):
92
num_heads: int,
93
num_kv_heads: int,
94
rope_theta: float = 10000,
90
- ):
95
+ quant_config: Optional[QuantizationConfig] = None,
96
+ ) -> None:
97
super().__init__()
98
self.hidden_size = hidden_size
99
tp_size = get_tensor_model_parallel_world_size()
@@ -103,20 +109,22 @@ class LlamaAttention(nn.Module):
109
self.scaling = self.head_dim**-0.5
110
self.rope_theta = rope_theta
111
106
- self.qkv_proj = ColumnParallelLinear(
112
+ self.qkv_proj = ParallelLinear.column(
113
hidden_size,
114
(self.total_num_heads + 2 * self.total_num_kv_heads) *
115
self.head_dim,
116
bias=False,
117
gather_output=False,
118
perform_initialization=False,
119
+ quant_config=quant_config,
120
)
114
- self.o_proj = RowParallelLinear(
121
+ self.o_proj = ParallelLinear.row(
122
self.total_num_heads * self.head_dim,
123
hidden_size,
124
bias=False,
125
input_is_parallel=True,
126
perform_initialization=False,
127
+ quant_config=quant_config,
128
)
129
self.attn = PagedAttentionWithRoPE(self.num_heads,
130
self.head_dim,
@@ -144,7 +152,11 @@ class LlamaAttention(nn.Module):
152
153
class LlamaDecoderLayer(nn.Module):
154
147
- def __init__(self, config: LlamaConfig):
155
+ def __init__(
156
+ self,
157
+ config: LlamaConfig,
158
+ quant_config: Optional[QuantizationConfig] = None,
159
+ ) -> None:
160
super().__init__()
161
self.hidden_size = config.hidden_size
162
# Requires transformers > 4.32.0
@@ -154,11 +166,13 @@ class LlamaDecoderLayer(nn.Module):
166
num_heads=config.num_attention_heads,
167
num_kv_heads=config.num_key_value_heads,
168
rope_theta=rope_theta,
169
+ quant_config=quant_config,
170
)
171
self.mlp = LlamaMLP(
172
hidden_size=self.hidden_size,
173
intermediate_size=config.intermediate_size,
174
hidden_act=config.hidden_act,
175
+ quant_config=quant_config,
176
)
177
self.input_layernorm = RMSNorm(config.hidden_size,
178
eps=config.rms_norm_eps)
@@ -195,7 +209,11 @@ class LlamaDecoderLayer(nn.Module):
209
210
class LlamaModel(nn.Module):
211
198
- def __init__(self, config: LlamaConfig):
212
+ def __init__(
213
+ self,
214
+ config: LlamaConfig,
215
+ quant_config: Optional[QuantizationConfig] = None,
216
+ ) -> None:
217
super().__init__()
218
self.config = config
219
self.padding_idx = config.pad_token_id
@@ -205,7 +223,8 @@ class LlamaModel(nn.Module):
223
self.embed_tokens = VocabParallelEmbedding(
224
vocab_size, config.hidden_size, perform_initialization=False)
225
self.layers = nn.ModuleList([
208
- LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
226
+ LlamaDecoderLayer(config, quant_config)
227
+ for _ in range(config.num_hidden_layers)
228
])
229
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
230
@@ -237,16 +256,23 @@ class LlamaModel(nn.Module):
256
257
class LlamaForCausalLM(nn.Module):
258
240
- def __init__(self, config):
259
+ def __init__(
260
+ self,
261
+ config: LlamaConfig,
262
+ quant_config: Optional[QuantizationConfig] = None,
263
+ ) -> None:
264
super().__init__()
265
self.config = config
243
- self.model = LlamaModel(config)
266
+ self.quant_config = quant_config
267
+ self.model = LlamaModel(config, quant_config)
268
vocab_size = ((config.vocab_size + 63) // 64) * 64
245
- self.lm_head = ColumnParallelLinear(config.hidden_size,
246
- vocab_size,
247
- bias=False,
248
- gather_output=False,
249
- perform_initialization=False)
269
+ # NOTE: The LM head is not quantized.
270
+ self.lm_head = ParallelLinear.column(config.hidden_size,
271
+ vocab_size,
272
+ bias=False,
273
+ gather_output=False,
274
+ perform_initialization=False,
275
+ quant_config=None)
276
self.sampler = Sampler(config.vocab_size)
277
278
def forward(
@@ -263,16 +289,28 @@ class LlamaForCausalLM(nn.Module):
289
input_metadata)
290
return next_tokens
291
266
- _column_parallel_weights = [
267
- "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
268
- ]
269
- _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
292
+ _column_parallel_layers = []
293
+ _row_parallel_layers = ["o_proj", "down_proj"]
294
295
def load_weights(self,
296
model_name_or_path: str,
297
cache_dir: Optional[str] = None,
298
load_format: str = "auto",
299
revision: Optional[str] = None):
300
+ if self.quant_config is None:
301
+ weight_suffixes = ["weight"]
302
+ else:
303
+ weight_suffixes = self.quant_config.get_tp_tensor_names()
304
+
305
+ column_parallel_weights: List[str] = []
306
+ for layer in self._column_parallel_layers:
307
+ for suffix in weight_suffixes:
308
+ column_parallel_weights.append(f"{layer}.{suffix}")
309
+ row_parallel_weights: List[str] = []
310
+ for layer in self._row_parallel_layers:
311
+ for suffix in weight_suffixes:
312
+ row_parallel_weights.append(f"{layer}.{suffix}")
313
+
314
tp_size = get_tensor_model_parallel_world_size()
315
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
316
q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -293,11 +331,25 @@ class LlamaForCausalLM(nn.Module):
331
if "rotary_emb.inv_freq" in name:
332
continue
333
334
+ is_packed = False
335
+ is_transposed = False
336
+ if self.quant_config is not None:
337
+ is_packed = self.quant_config.is_packed(name)
338
+ is_transposed = self.quant_config.is_transposed(name)
339
+ if is_transposed:
340
+ loaded_weight = loaded_weight.T
341
+
342
is_attention_weight = False
343
for weight_name, shard_size, offset in attention_weight_specs:
344
if weight_name not in name:
345
continue
346
param = state_dict[name.replace(weight_name, "qkv_proj")]
347
+ if is_transposed:
348
+ param = param.T
349
+
350
+ if is_packed:
351
+ shard_size //= self.quant_config.pack_factor
352
+ offset //= self.quant_config.pack_factor
353
354
loaded_weight = loaded_weight[
355
shard_size * tensor_model_parallel_rank:shard_size *
@@ -316,6 +368,9 @@ class LlamaForCausalLM(nn.Module):
368
if weight_name not in name:
369
continue
370
param = state_dict[name.replace(weight_name, "gate_up_proj")]
371
+ if is_transposed:
372
+ param = param.T
373
+
374
shard_size = param.shape[0] // 2
375
loaded_weight = loaded_weight[
376
shard_size * tensor_model_parallel_rank:shard_size *
@@ -330,6 +385,8 @@ class LlamaForCausalLM(nn.Module):
385
continue
386
387
param = state_dict[name]
388
+ if is_transposed:
389
+ param = param.T
390
391
if "embed_tokens" in name or "lm_head" in name:
392
load_padded_tensor_parallel_vocab(param, loaded_weight,
@@ -337,6 +394,6 @@ class LlamaForCausalLM(nn.Module):
394
continue
395
396
load_tensor_parallel_weights(param, loaded_weight, name,
340
- self._column_parallel_weights,
341
- self._row_parallel_weights,
397
+ column_parallel_weights,
398
+ row_parallel_weights,
399
tensor_model_parallel_rank)
vllm/model_executor/parallel_utils/tensor_parallel/layers.py
+47
-130
@@ -4,7 +4,7 @@
4
5
# Parts of the code here are adapted from PyTorch
6
# repo: https://github.com/pytorch/pytorch
7
-
7
+from typing import Optional
8
9
import torch
10
import torch.nn.functional as F
@@ -16,13 +16,11 @@ from vllm.model_executor.parallel_utils.parallel_state import (
16
get_tensor_model_parallel_world_size,
17
)
18
from .mappings import (
19
- copy_to_tensor_model_parallel_region,
19
gather_from_tensor_model_parallel_region,
20
reduce_from_tensor_model_parallel_region,
21
scatter_to_tensor_model_parallel_region,
22
)
23
25
-from .random import get_cuda_rng_tracker
24
from .utils import (
25
divide,
26
VocabUtility,
@@ -65,59 +63,6 @@ def copy_tensor_model_parallel_attributes(destination_tensor, source_tensor):
63
maybe_copy(attribute)
64
65
68
-def _initialize_affine_weight_gpu(weight, init_method,
69
- partition_dim, stride=1):
70
- """Initialize affine weight for model parallel on GPU."""
71
-
72
- set_tensor_model_parallel_attributes(tensor=weight,
73
- is_parallel=True,
74
- dim=partition_dim,
75
- stride=stride)
76
-
77
- with get_cuda_rng_tracker().fork():
78
- init_method(weight)
79
-
80
-
81
-def _initialize_affine_weight_cpu(weight, output_size, input_size,
82
- per_partition_size, partition_dim,
83
- init_method, stride=1,
84
- return_master_weight=False,
85
- *, params_dtype=None):
86
- """Initialize affine weight for model parallel.
87
-
88
- Build the master weight on all processes and scatter
89
- the relevant chunk."""
90
-
91
- set_tensor_model_parallel_attributes(tensor=weight,
92
- is_parallel=True,
93
- dim=partition_dim,
94
- stride=stride)
95
-
96
- if params_dtype is None:
97
- params_dtype = torch.get_default_dtype()
98
-
99
- # Initialize master weight
100
- master_weight = torch.empty(output_size, input_size,
101
- dtype=torch.float,
102
- requires_grad=False)
103
- init_method(master_weight)
104
- master_weight = master_weight.to(dtype=params_dtype)
105
-
106
- # Split and copy
107
- per_partition_per_stride_size = divide(per_partition_size, stride)
108
- weight_list = torch.split(master_weight, per_partition_per_stride_size,
109
- dim=partition_dim)
110
- rank = get_tensor_model_parallel_rank()
111
- world_size = get_tensor_model_parallel_world_size()
112
- my_weight_list = weight_list[rank::world_size]
113
-
114
- with torch.no_grad():
115
- torch.cat(my_weight_list, dim=partition_dim, out=weight)
116
- if return_master_weight:
117
- return master_weight
118
- return None
119
-
120
-
66
class VocabParallelEmbedding(torch.nn.Module):
67
"""Embedding parallelized in the vocabulary dimension.
68
@@ -140,6 +85,9 @@ class VocabParallelEmbedding(torch.nn.Module):
85
use_cpu_initialization: bool=False,
86
perform_initialization: bool=True):
87
super(VocabParallelEmbedding, self).__init__()
88
+ assert not perform_initialization
89
+ assert not use_cpu_initialization
90
+
91
# Keep the input dimensions.
92
self.num_embeddings = num_embeddings
93
self.embedding_dim = embedding_dim
@@ -162,24 +110,10 @@ class VocabParallelEmbedding(torch.nn.Module):
110
self.num_embeddings_per_partition = self.vocab_end_index - \
111
self.vocab_start_index
112
165
- # Allocate weights and initialize.
166
- if use_cpu_initialization:
167
- self.weight = Parameter(torch.empty(
168
- self.num_embeddings_per_partition, self.embedding_dim,
169
- dtype=params_dtype))
170
- if perform_initialization:
171
- _initialize_affine_weight_cpu(
172
- self.weight, self.num_embeddings, self.embedding_dim,
173
- self.num_embeddings_per_partition, 0, init_method,
174
- params_dtype=params_dtype)
175
- else:
176
- self.weight = Parameter(torch.empty(
177
- self.num_embeddings_per_partition, self.embedding_dim,
178
- device=torch.cuda.current_device(), dtype=params_dtype))
179
- if perform_initialization:
180
- _initialize_affine_weight_gpu(self.weight, init_method,
181
- partition_dim=0, stride=1)
182
-
113
+ self.weight = Parameter(torch.empty(
114
+ self.num_embeddings_per_partition, self.embedding_dim,
115
+ device=torch.cuda.current_device(), dtype=params_dtype))
116
+
117
def forward(self, input_):
118
if self.tensor_model_parallel_size > 1:
119
# Build the mask.
@@ -239,8 +173,11 @@ class ColumnParallelLinear(torch.nn.Module):
173
params_dtype=None,
174
use_cpu_initialization=False,
175
perform_initialization=True,
176
+ quant_config=None,
177
):
178
super(ColumnParallelLinear, self).__init__()
179
+ assert not perform_initialization
180
+ assert not use_cpu_initialization
181
182
# Keep input parameters
183
self.input_size = input_size
@@ -250,6 +187,7 @@ class ColumnParallelLinear(torch.nn.Module):
187
self.world_size = get_tensor_model_parallel_world_size()
188
self.output_size_per_partition = divide(output_size, self.world_size)
189
self.skip_bias_add = skip_bias_add
190
+ self.quant_config = quant_config
191
192
if params_dtype is None:
193
params_dtype = torch.get_default_dtype()
@@ -257,33 +195,13 @@ class ColumnParallelLinear(torch.nn.Module):
195
# Parameters.
196
# Note: torch.nn.functional.linear performs XA^T + b and as a result
197
# we allocate the transpose.
260
- # Initialize weight.
261
- if use_cpu_initialization:
262
- self.weight = Parameter(torch.empty(self.output_size_per_partition,
263
- self.input_size,
264
- dtype=params_dtype))
265
- if perform_initialization:
266
- self.master_weight = _initialize_affine_weight_cpu(
267
- self.weight, self.output_size, self.input_size,
268
- self.output_size_per_partition, 0, init_method,
269
- stride=stride, return_master_weight=keep_master_weight_for_test)
270
- else:
271
- self.weight = Parameter(torch.empty(
272
- self.output_size_per_partition, self.input_size,
273
- device=torch.cuda.current_device(), dtype=params_dtype))
274
- if perform_initialization:
275
- _initialize_affine_weight_gpu(self.weight, init_method,
276
- partition_dim=0, stride=stride)
198
+ self.create_weights(params_dtype)
199
200
if bias:
279
- if use_cpu_initialization:
280
- self.bias = Parameter(torch.empty(
281
- self.output_size_per_partition, dtype=params_dtype))
282
- else:
283
- self.bias = Parameter(torch.empty(
284
- self.output_size_per_partition,
285
- device=torch.cuda.current_device(),
286
- dtype=params_dtype))
201
+ self.bias = Parameter(torch.empty(
202
+ self.output_size_per_partition,
203
+ device=torch.cuda.current_device(),
204
+ dtype=params_dtype))
205
set_tensor_model_parallel_attributes(self.bias, True, 0, stride)
206
# Always initialize bias to zero.
207
with torch.no_grad():
@@ -291,6 +209,17 @@ class ColumnParallelLinear(torch.nn.Module):
209
else:
210
self.register_parameter('bias', None)
211
212
+ def create_weights(self, dtype: torch.dtype) -> None:
213
+ self.weight = Parameter(torch.empty(
214
+ self.output_size_per_partition, self.input_size,
215
+ device=torch.cuda.current_device(), dtype=dtype))
216
+
217
+ def apply_weights(
218
+ self,
219
+ x: torch.Tensor,
220
+ bias: Optional[torch.Tensor],
221
+ ) -> torch.Tensor:
222
+ return F.linear(x, self.weight, bias)
223
224
def forward(self, input_):
225
"""Forward of ColumnParallelLinear
@@ -306,7 +235,7 @@ class ColumnParallelLinear(torch.nn.Module):
235
236
input_parallel = input_
237
# Matrix multiply.
309
- output_parallel = F.linear(input_parallel, self.weight, bias)
238
+ output_parallel = self.apply_weights(input_parallel, bias)
239
if self.gather_output:
240
# All-gather across the partitions.
241
output = gather_from_tensor_model_parallel_region(output_parallel)
@@ -361,8 +290,11 @@ class RowParallelLinear(torch.nn.Module):
290
use_cpu_initialization=False,
291
perform_initialization=True,
292
reduce_results=True,
293
+ quant_config=None,
294
):
295
super(RowParallelLinear, self).__init__()
296
+ assert not perform_initialization
297
+ assert not use_cpu_initialization
298
299
# Keep input parameters
300
self.input_size = input_size
@@ -376,47 +308,32 @@ class RowParallelLinear(torch.nn.Module):
308
self.world_size = get_tensor_model_parallel_world_size()
309
self.input_size_per_partition = divide(input_size, self.world_size)
310
self.skip_bias_add = skip_bias_add
311
+ self.quant_config = quant_config
312
+
313
+ self.create_weights(params_dtype)
314
315
if not reduce_results and (bias and not skip_bias_add):
316
raise ValueError("When not reduce the results, adding bias to the "
317
"results can lead to incorrect results")
318
384
- # Parameters.
385
- # Note: torch.nn.functional.linear performs XA^T + b and as a result
386
- # we allocate the transpose.
387
- # Initialize weight.
388
- if use_cpu_initialization:
389
- self.weight = Parameter(torch.empty(self.output_size,
390
- self.input_size_per_partition,
391
- dtype=params_dtype))
392
- if perform_initialization:
393
- self.master_weight = _initialize_affine_weight_cpu(
394
- self.weight, self.output_size, self.input_size,
395
- self.input_size_per_partition, 1, init_method,
396
- stride=stride, return_master_weight=keep_master_weight_for_test,
397
- params_dtype=params_dtype)
398
- else:
399
- self.weight = Parameter(torch.empty(
400
- self.output_size, self.input_size_per_partition,
401
- device=torch.cuda.current_device(), dtype=params_dtype))
402
- if perform_initialization:
403
- _initialize_affine_weight_gpu(self.weight, init_method,
404
- partition_dim=1, stride=stride)
319
if bias:
406
- if use_cpu_initialization:
407
- self.bias = Parameter(torch.empty(self.output_size,
408
- dtype=params_dtype))
409
- else:
410
- self.bias = Parameter(torch.empty(
411
- self.output_size, device=torch.cuda.current_device(),
412
- dtype=params_dtype))
320
+ self.bias = Parameter(torch.empty(
321
+ self.output_size, device=torch.cuda.current_device(),
322
+ dtype=params_dtype))
323
324
# Always initialize bias to zero.
325
with torch.no_grad():
326
self.bias.zero_()
327
else:
328
self.register_parameter('bias', None)
419
- self.weight_t = self.weight.t()
329
+
330
+ def create_weights(self, dtype: torch.dtype) -> None:
331
+ self.weight = Parameter(torch.empty(
332
+ self.output_size, self.input_size_per_partition,
333
+ device=torch.cuda.current_device(), dtype=dtype))
334
+
335
+ def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
336
+ return F.linear(x, self.weight)
337
338
def forward(self, input_):
339
"""Forward of RowParallelLinear
@@ -434,7 +351,7 @@ class RowParallelLinear(torch.nn.Module):
351
else:
352
input_parallel = scatter_to_tensor_model_parallel_region(input_)
353
# Matrix multiply.
437
- output_parallel = F.linear(input_parallel, self.weight)
354
+ output_parallel = self.apply_weights(input_parallel)
355
if self.reduce_results and self.world_size > 1:
356
output_ = reduce_from_tensor_model_parallel_region(output_parallel)
357
else:
vllm/model_executor/quantization_utils/__init__.py
new
+20
@@ -0,0 +1,20 @@
1
+from typing import Type
2
+
3
+from vllm.model_executor.quantization_utils.awq import AWQConfig
4
+from vllm.model_executor.quantization_utils.base import QuantizationConfig
5
+
6
+_QUANTIZATION_REGISTRY = {
7
+ "awq": AWQConfig,
8
+}
9
+
10
+
11
+def get_quant_class(quantization: str) -> Type[QuantizationConfig]:
12
+ if quantization not in _QUANTIZATION_REGISTRY:
13
+ raise ValueError(f"Invalid quantization method: {quantization}")
14
+ return _QUANTIZATION_REGISTRY[quantization]
15
+
16
+
17
+__all__ = [
18
+ "QuantizationConfig",
19
+ "get_quant_class",
20
+]
vllm/model_executor/quantization_utils/awq.py
new
+67
@@ -0,0 +1,67 @@
1
+from typing import Any, Dict, List
2
+
3
+import torch
4
+
5
+from vllm.model_executor.quantization_utils.base import QuantizationConfig
6
+
7
+
8
+class AWQConfig(QuantizationConfig):
9
+ """Config class for AWQ.
10
+
11
+ Reference: https://arxiv.org/abs/2306.00978
12
+ """
13
+
14
+ def __init__(
15
+ self,
16
+ weight_bits: int,
17
+ group_size: int,
18
+ zero_point: bool,
19
+ ) -> None:
20
+ self.weight_bits = weight_bits
21
+ self.group_size = group_size
22
+ self.zero_point = zero_point
23
+
24
+ if self.weight_bits != 4:
25
+ raise ValueError(
26
+ "Currently, only 4-bit weight quantization is supported for "
27
+ f"AWQ, but got {self.weight_bits} bits.")
28
+ self.pack_factor = 32 // self.weight_bits
29
+
30
+ def __repr__(self) -> str:
31
+ return (f"AWQConfig(weight_bits={self.weight_bits}, "
32
+ f"group_size={self.group_size}, "
33
+ f"zero_point={self.zero_point})")
34
+
35
+ @classmethod
36
+ def get_name(cls) -> str:
37
+ return "awq"
38
+
39
+ @classmethod
40
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41
+ return [torch.half]
42
+
43
+ @classmethod
44
+ def get_config_filenames(cls) -> List[str]:
45
+ return [
46
+ "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
47
+ "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq # pylint: disable=line-too-long
48
+ ]
49
+
50
+ @classmethod
51
+ def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
52
+ weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
53
+ group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
54
+ zero_point = cls.get_from_keys(config, ["zero_point"])
55
+ return cls(weight_bits, group_size, zero_point)
56
+
57
+ @classmethod
58
+ def get_packed_tensor_names(cls) -> List[str]:
59
+ return ["qweight", "qzeros"]
60
+
61
+ @classmethod
62
+ def get_transposed_tensor_names(cls) -> List[str]:
63
+ return ["qweight", "qzeros", "scales"]
64
+
65
+ @classmethod
66
+ def get_tp_tensor_names(cls) -> List[str]:
67
+ return ["qweight", "qzeros", "scales"]
vllm/model_executor/quantization_utils/base.py
new
+65
@@ -0,0 +1,65 @@
1
+from typing import Any, Dict, List
2
+
3
+import torch
4
+
5
+
6
+class QuantizationConfig:
7
+
8
+ @classmethod
9
+ def get_name(cls) -> str:
10
+ """Name of the quantization method."""
11
+ raise NotImplementedError
12
+
13
+ @classmethod
14
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
15
+ """List of supported activation dtypes."""
16
+ raise NotImplementedError
17
+
18
+ @classmethod
19
+ def get_config_filenames(cls) -> List[str]:
20
+ """List of filenames to search for in the model directory."""
21
+ raise NotImplementedError
22
+
23
+ @classmethod
24
+ def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
25
+ """Create a config class from the model's quantization config."""
26
+ raise NotImplementedError
27
+
28
+ @staticmethod
29
+ def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
30
+ """Get a value from the model's quantization config."""
31
+ for key in keys:
32
+ if key in config:
33
+ return config[key]
34
+ raise ValueError(f"Cannot find any of {keys} in the model's "
35
+ "quantization config.")
36
+
37
+ @classmethod
38
+ def get_packed_tensor_names(cls) -> List[str]:
39
+ raise NotImplementedError
40
+
41
+ @classmethod
42
+ def is_packed(cls, tensor_name: str) -> bool:
43
+ """Returns True if a tensor is packed.
44
+
45
+ A tensor is considered packed if each element in the tensor is a
46
+ packed representation of multiple elements in the original tensor.
47
+ For example, an INT32 element in the tensor may represent 8 INT4
48
+ elements in the original tensor.
49
+ """
50
+ return any(tag in tensor_name for tag in cls.get_packed_tensor_names())
51
+
52
+ @classmethod
53
+ def get_transposed_tensor_names(cls) -> List[str]:
54
+ raise NotImplementedError
55
+
56
+ @classmethod
57
+ def is_transposed(cls, tensor_name: str) -> bool:
58
+ """Returns True if a tensor is transposed relative to nn.Linear.weight.
59
+ """
60
+ return any(tag in tensor_name
61
+ for tag in cls.get_transposed_tensor_names())
62
+
63
+ @classmethod
64
+ def get_tp_tensor_names(cls) -> List[str]:
65
+ raise NotImplementedError
vllm/model_executor/weight_utils.py
+48
-5
@@ -4,7 +4,7 @@ import glob
4
import json
5
import os
6
from collections import defaultdict
7
-from typing import Iterator, List, Optional, Tuple, Any
7
+from typing import Any, Iterator, List, Optional, Tuple
8
9
from huggingface_hub import snapshot_download
10
from safetensors.torch import load_file, save_file, safe_open
@@ -13,6 +13,8 @@ import torch
13
from tqdm.auto import tqdm
14
15
from vllm.logger import init_logger
16
+from vllm.model_executor.quantization_utils import get_quant_class
17
+from vllm.model_executor.quantization_utils.base import QuantizationConfig
18
19
logger = init_logger(__name__)
20
@@ -44,7 +46,7 @@ def _shared_pointers(tensors):
46
def convert_bin_to_safetensor_file(
47
pt_filename: str,
48
sf_filename: str,
47
-):
49
+) -> None:
50
loaded = torch.load(pt_filename, map_location="cpu")
51
if "state_dict" in loaded:
52
loaded = loaded["state_dict"]
@@ -78,16 +80,55 @@ def convert_bin_to_safetensor_file(
80
raise RuntimeError(f"The output tensors do not match for key {k}")
81
82
83
+# TODO(woosuk): Move this to other place.
84
+def get_quant_config(
85
+ quantization: str,
86
+ model_name_or_path: str,
87
+ cache_dir: Optional[str] = None,
88
+) -> QuantizationConfig:
89
+ is_local = os.path.isdir(model_name_or_path)
90
+ if not is_local:
91
+ # Download the config files.
92
+ with get_lock(model_name_or_path, cache_dir):
93
+ hf_folder = snapshot_download(model_name_or_path,
94
+ allow_patterns="*.json",
95
+ cache_dir=cache_dir,
96
+ tqdm_class=Disabledtqdm)
97
+ else:
98
+ hf_folder = model_name_or_path
99
+ config_files = glob.glob(os.path.join(hf_folder, "*.json"))
100
+
101
+ quant_cls = get_quant_class(quantization)
102
+ quant_config_files = [
103
+ f for f in config_files if any(
104
+ f.endswith(x) for x in quant_cls.get_config_filenames())
105
+ ]
106
+ if len(quant_config_files) == 0:
107
+ raise ValueError(f"Cannot find the config file for {quantization}")
108
+ if len(quant_config_files) > 1:
109
+ raise ValueError(f"Found multiple config files for {quantization}: "
110
+ f"{quant_config_files}")
111
+
112
+ quant_config_file = quant_config_files[0]
113
+ with open(quant_config_file, "r") as f:
114
+ config = json.load(f)
115
+ return quant_cls.from_config(config)
116
+
117
+
118
def prepare_hf_model_weights(
119
model_name_or_path: str,
120
cache_dir: Optional[str] = None,
121
use_safetensors: bool = False,
122
fall_back_to_pt: bool = True,
123
revision: Optional[str] = None,
87
-):
124
+) -> Tuple[str, List[str], bool]:
125
# Download model weights from huggingface.
126
is_local = os.path.isdir(model_name_or_path)
90
- allow_patterns = "*.safetensors" if use_safetensors else "*.bin"
127
+ if use_safetensors:
128
+ allow_patterns = ["*.safetensors"]
129
+ else:
130
+ # Some quantized models use .pt files for storing the weights.
131
+ allow_patterns = ["*.bin", "*.pt"]
132
if not is_local:
133
# Use file lock to prevent multiple processes from
134
# downloading the same model weights at the same time.
@@ -99,7 +140,9 @@ def prepare_hf_model_weights(
140
revision=revision)
141
else:
142
hf_folder = model_name_or_path
102
- hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
143
+ hf_weights_files: List[str] = []
144
+ for pattern in allow_patterns:
145
+ hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
146
if not use_safetensors:
147
hf_weights_files = [
148
x for x in hf_weights_files if not x.endswith("training_args.bin")