@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3e79e9e8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Implement AWQ quantization support for LLaMA (#1032)

Co-authored-by: Robert Irvine <robert@seamlessml.com> Co-authored-by: root <rirv938@gmail.com> Co-authored-by: Casper <casperbh.96@gmail.com> Co-authored-by: julian-q <julianhquevedo@gmail.com>

Woosuk Kwon committed Sep 16, 2023 at 00:03 UTC e3e79e9e8a2224e03a711c3d1ef7a35daa447083
19 files changed +1178 -208
.gitignore
+4
@@ -173,3 +173,7 @@ cython_debug/
173
174 # Sphinx documentation
175 _build/
176 +
177 +# vim swap files
178 +*.swo
179 +*.swp
benchmarks/benchmark_latency.py
+14 -4
@@ -18,6 +18,7 @@ def main(args: argparse.Namespace):
18 llm = LLM(
19 model=args.model,
20 tokenizer=args.tokenizer,
21 + quantization=args.quantization,
22 tensor_parallel_size=args.tensor_parallel_size,
23 max_num_seqs=args.batch_size,
24 max_num_batched_tokens=args.batch_size * args.input_len,
@@ -63,19 +64,28 @@ def main(args: argparse.Namespace):
64 if __name__ == '__main__':
65 parser = argparse.ArgumentParser(
66 description='Benchmark the latency of processing a single batch of '
66 - 'requests till completion.')
67 + 'requests till completion.')
68 parser.add_argument('--model', type=str, default='facebook/opt-125m')
69 parser.add_argument('--tokenizer', type=str, default=None)
70 + parser.add_argument('--quantization',
71 + '-q',
72 + choices=['awq', None],
73 + default=None)
74 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
75 parser.add_argument('--input-len', type=int, default=32)
76 parser.add_argument('--output-len', type=int, default=128)
77 parser.add_argument('--batch-size', type=int, default=8)
73 - parser.add_argument('--n', type=int, default=1,
78 + parser.add_argument('--n',
79 + type=int,
80 + default=1,
81 help='Number of generated sequences per prompt.')
82 parser.add_argument('--use-beam-search', action='store_true')
76 - parser.add_argument('--num-iters', type=int, default=3,
83 + parser.add_argument('--num-iters',
84 + type=int,
85 + default=3,
86 help='Number of iterations to run.')
78 - parser.add_argument('--trust-remote-code', action='store_true',
87 + parser.add_argument('--trust-remote-code',
88 + action='store_true',
89 help='trust remote code from huggingface')
90 args = parser.parse_args()
91 main(args)
benchmarks/benchmark_throughput.py
+44 -30
@@ -3,7 +3,7 @@ import argparse
3 import json
4 import random
5 import time
6 -from typing import List, Tuple
6 +from typing import List, Optional, Tuple
7
8 import torch
9 from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
@@ -22,15 +22,10 @@ def sample_requests(
22 with open(dataset_path) as f:
23 dataset = json.load(f)
24 # Filter out the conversations with less than 2 turns.
25 - dataset = [
26 - data for data in dataset
27 - if len(data["conversations"]) >= 2
28 - ]
25 + dataset = [data for data in dataset if len(data["conversations"]) >= 2]
26 # Only keep the first two turns of each conversation.
30 - dataset = [
31 - (data["conversations"][0]["value"], data["conversations"][1]["value"])
32 - for data in dataset
33 - ]
27 + dataset = [(data["conversations"][0]["value"],
28 + data["conversations"][1]["value"]) for data in dataset]
29
30 # Tokenize the prompts and completions.
31 prompts = [prompt for prompt, _ in dataset]
@@ -63,6 +58,7 @@ def run_vllm(
58 requests: List[Tuple[str, int, int]],
59 model: str,
60 tokenizer: str,
61 + quantization: Optional[str],
62 tensor_parallel_size: int,
63 seed: int,
64 n: int,
@@ -72,6 +68,7 @@ def run_vllm(
68 llm = LLM(
69 model=model,
70 tokenizer=tokenizer,
71 + quantization=quantization,
72 tensor_parallel_size=tensor_parallel_size,
73 seed=seed,
74 trust_remote_code=trust_remote_code,
@@ -111,8 +108,8 @@ def run_hf(
108 trust_remote_code: bool,
109 ) -> float:
110 assert not use_beam_search
114 - llm = AutoModelForCausalLM.from_pretrained(model,
115 - torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
111 + llm = AutoModelForCausalLM.from_pretrained(
112 + model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
113 if llm.config.model_type == "llama":
114 # To enable padding in the HF backend.
115 tokenizer.pad_token = tokenizer.eos_token
@@ -132,13 +129,14 @@ def run_hf(
129 if len(batch) < max_batch_size and i != len(requests) - 1:
130 # Check if we can add more requests to the batch.
131 _, next_prompt_len, next_output_len = requests[i + 1]
135 - if (max(max_prompt_len, next_prompt_len) + max(
136 - max_output_len, next_output_len)) <= 2048:
132 + if (max(max_prompt_len, next_prompt_len) +
133 + max(max_output_len, next_output_len)) <= 2048:
134 # We can add more requests to the batch.
135 continue
136
137 # Generate the sequences.
141 - input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids
138 + input_ids = tokenizer(batch, return_tensors="pt",
139 + padding=True).input_ids
140 llm_outputs = llm.generate(
141 input_ids=input_ids.cuda(),
142 do_sample=not use_beam_search,
@@ -165,44 +163,58 @@ def main(args: argparse.Namespace):
163 random.seed(args.seed)
164
165 # Sample the requests.
168 - tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
166 + tokenizer = get_tokenizer(args.tokenizer,
167 + trust_remote_code=args.trust_remote_code)
168 requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
169
170 if args.backend == "vllm":
172 - elapsed_time = run_vllm(
173 - requests, args.model, args.tokenizer, args.tensor_parallel_size,
174 - args.seed, args.n, args.use_beam_search, args.trust_remote_code)
171 + elapsed_time = run_vllm(requests, args.model, args.tokenizer,
172 + args.quantization, args.tensor_parallel_size,
173 + args.seed, args.n, args.use_beam_search,
174 + args.trust_remote_code)
175 elif args.backend == "hf":
176 assert args.tensor_parallel_size == 1
177 - elapsed_time = run_hf(
178 - requests, args.model, tokenizer, args.n, args.use_beam_search,
179 - args.hf_max_batch_size, args.trust_remote_code)
177 + elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
178 + args.use_beam_search, args.hf_max_batch_size,
179 + args.trust_remote_code)
180 else:
181 raise ValueError(f"Unknown backend: {args.backend}")
182 - total_num_tokens = sum(
183 - prompt_len + output_len
184 - for _, prompt_len, output_len in requests
185 - )
182 + total_num_tokens = sum(prompt_len + output_len
183 + for _, prompt_len, output_len in requests)
184 print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
185 f"{total_num_tokens / elapsed_time:.2f} tokens/s")
186
187
188 if __name__ == "__main__":
189 parser = argparse.ArgumentParser(description="Benchmark the throughput.")
192 - parser.add_argument("--backend", type=str, choices=["vllm", "hf"],
190 + parser.add_argument("--backend",
191 + type=str,
192 + choices=["vllm", "hf"],
193 default="vllm")
194 - parser.add_argument("--dataset", type=str, required=True,
194 + parser.add_argument("--dataset",
195 + type=str,
196 + required=True,
197 help="Path to the dataset.")
198 parser.add_argument("--model", type=str, default="facebook/opt-125m")
199 parser.add_argument("--tokenizer", type=str, default=None)
200 + parser.add_argument('--quantization',
201 + '-q',
202 + choices=['awq', None],
203 + default=None)
204 parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
199 - parser.add_argument("--n", type=int, default=1,
205 + parser.add_argument("--n",
206 + type=int,
207 + default=1,
208 help="Number of generated sequences per prompt.")
209 parser.add_argument("--use-beam-search", action="store_true")
202 - parser.add_argument("--num-prompts", type=int, default=1000,
210 + parser.add_argument("--num-prompts",
211 + type=int,
212 + default=1000,
213 help="Number of prompts to process.")
214 parser.add_argument("--seed", type=int, default=0)
205 - parser.add_argument("--hf-max-batch-size", type=int, default=None,
215 + parser.add_argument("--hf-max-batch-size",
216 + type=int,
217 + default=None,
218 help="Maximum batch size for HF backend.")
219 parser.add_argument('--trust-remote-code',
220 action='store_true',
@@ -215,6 +227,8 @@ if __name__ == "__main__":
227 elif args.backend == "hf":
228 if args.hf_max_batch_size is None:
229 raise ValueError("HF max batch size is required for HF backend.")
230 + if args.quantization is not None:
231 + raise ValueError("Quantization is only for vLLM backend.")
232 if args.tokenizer is None:
233 args.tokenizer = args.model
234
csrc/quantization.cpp new
+15
@@ -0,0 +1,15 @@
1 +#include <torch/extension.h>
2 +
3 +torch::Tensor awq_gemm(
4 + torch::Tensor _in_feats,
5 + torch::Tensor _kernel,
6 + torch::Tensor _scaling_factors,
7 + torch::Tensor _zeros,
8 + int split_k_iters);
9 +
10 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
11 + m.def(
12 + "awq_gemm",
13 + &awq_gemm,
14 + "Quantized GEMM for AWQ");
15 +}
csrc/quantization/awq/dequantize.cuh new
+79
@@ -0,0 +1,79 @@
1 +/*
2 +Adapted from https://github.com/mit-han-lab/llm-awq
3 +Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
4 +@article{lin2023awq,
5 + title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
6 + author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
7 + journal={arXiv},
8 + year={2023}
9 +}
10 +*/
11 +
12 +#pragma once
13 +
14 +
15 +__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
16 +{
17 + uint4 result;
18 +
19 + uint32_t* h = reinterpret_cast<uint32_t*>(&result);
20 + uint32_t const i4s = reinterpret_cast<uint32_t const&>(source);
21 +
22 + // First, we extract the i4s and construct an intermediate fp16 number.
23 + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa;
24 + static constexpr uint32_t BOTTOM_MASK = 0x000f000f;
25 + static constexpr uint32_t TOP_MASK = 0x00f000f0;
26 + static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400;
27 +
28 + // Note that the entire sequence only requires 1 shift instruction. This is thanks to the register packing
29 + // format and the fact that we force our integers to be unsigned, and account for this in the fp16 subtractions.
30 + // In addition, I exploit the fact that sub and fma have the same throughput in order to convert elt_23 and
31 + // elt_67 to fp16 without having to shift them to the bottom bits before hand.
32 +
33 + // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW dependency if we issue
34 + // immediately before required.
35 + const uint32_t top_i4s = i4s >> 8;
36 + // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400
37 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
38 + : "=r"(h[0])
39 + : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
40 + // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400
41 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
42 + : "=r"(h[1])
43 + : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
44 + // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400
45 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
46 + : "=r"(h[2])
47 + : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
48 + // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400
49 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
50 + : "=r"(h[3])
51 + : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
52 +
53 + // I use inline PTX below because I am not sure if the compiler will emit float2half instructions if I use the
54 + // half2 ctor. In this case, I chose performance reliability over code readability.
55 +
56 + // This is the half2 {1032, 1032} represented as an integer.
57 + // static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408;
58 + // Haotian: subtract {1024, 1024} instead, we do not need to map to [-8, 7]
59 + static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400;
60 + // This is the half2 {1 / 16, 1 / 16} represented as an integer.
61 + static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00;
62 + // This is the half2 {-72, -72} represented as an integer.
63 + // static constexpr uint32_t NEG_72 = 0xd480d480;
64 + // Haotian: Let's use {-64, -64}.
65 + static constexpr uint32_t NEG_64 = 0xd400d400;
66 +
67 + // Finally, we construct the output numbers.
68 + // Convert elt_01
69 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM));
70 + // Convert elt_23
71 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
72 + // Convert elt_45
73 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM));
74 + // Convert elt_67
75 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
76 +
77 + return result;
78 +}
79 +
csrc/quantization/awq/gemm_kernels.cu new
+477
@@ -0,0 +1,477 @@
1 +/*
2 +Adapted from https://github.com/mit-han-lab/llm-awq
3 +@article{lin2023awq,
4 + title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
5 + author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
6 + journal={arXiv},
7 + year={2023}
8 +}
9 + */
10 +
11 +
12 +#include <torch/extension.h>
13 +#include <c10/cuda/CUDAGuard.h>
14 +
15 +#include "dequantize.cuh"
16 +
17 +#include <cuda_fp16.h>
18 +
19 +// Pack two half values.
20 +static inline __device__ __host__ unsigned
21 +__pack_half2(const half x, const half y) {
22 + unsigned v0 = *((unsigned short *)&x);
23 + unsigned v1 = *((unsigned short *)&y);
24 + return (v1 << 16) | v0;
25 +}
26 +
27 +__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
28 +{
29 + static constexpr uint32_t ZERO = 0x0;
30 + float C_warp[32];
31 + __shared__ half A_shared[16 * (32 + 8)];
32 + __shared__ half B_shared[32 * (128 + 8)];
33 +
34 + __shared__ half scaling_factors_shared[128];
35 + __shared__ half zeros_shared[128];
36 +
37 + int j_factors1 = ((OC + 128 - 1) / 128);
38 + int blockIdx_x = 0;
39 + int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
40 + int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
41 +
42 + half A_shared_warp[8];
43 + half B_shared_warp[32];
44 + for (int j_0_4_init = 0; j_0_4_init < 4; ++j_0_4_init) {
45 + for (int i = 0; i < 8; ++i) {
46 + C_warp[(j_0_4_init * 8) + i] = 0.0;
47 + }
48 + }
49 +
50 + static constexpr int row_stride_warp = 32 * 8 / 32;
51 + static constexpr int row_stride = 2 * 32 * 8 / 128;
52 + bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 128;
53 + // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
54 + bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
55 + // bool wb_C_flag = (threadIdx.x / 4) < M;
56 +
57 + half* A_ptr = A
58 + + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
59 + + (((int)threadIdx.x) % (32 / 8)) * 8;
60 +
61 + int* B_ptr = B
62 + + ((int)threadIdx.y) * (OC / 8) * 2
63 + + (((int)threadIdx.x) / (128 / 8)) * (OC / 8)
64 + + (((int)blockIdx_y) % j_factors1) * (128 / 8)
65 + + (((int)threadIdx.x) % (128 / 8)) * 1;
66 +// Why * 1 in the above line?
67 +
68 + half* A_shared_ptr = A_shared
69 + + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
70 + + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
71 + + (((int)threadIdx.x) % (32 / 8) ) * 8;
72 +
73 + half* B_shared_ptr = B_shared
74 + + ((int)threadIdx.y) * (row_stride / 2) * (128 + 8)
75 + + (((int)threadIdx.x) / (128 / 8)) * (128 + 8)
76 + + (((int)threadIdx.x) % (128 / 8)) * 8;
77 +
78 + int* zeros_ptr = zeros
79 + + (((int)blockIdx_y) % j_factors1) * (128 / 8)
80 + + ((int)threadIdx.x) % (128 / 8);
81 +
82 + half* scaling_factors_ptr = scaling_factors
83 + + (((int)blockIdx_y) % j_factors1) * (128)
84 + + (((int)threadIdx.x) % (128 / 8)) * 8;
85 +
86 + half* C_ptr = C
87 + + blockIdx_z * M * OC // blockIdz.x -> split_k dim
88 + + (((int)blockIdx_y) % j_factors1) * 128
89 + + ((int)threadIdx.y) * 64
90 + + (((int)threadIdx.x) % 4) * 2;
91 +
92 + // preload s.f. and zeros
93 + int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
94 + if ((k_bound - 1) * split_k_iters * 32 + blockIdx_z * 32 >= IC) k_bound -= 1;
95 + for (int _k_0_0 = 0; _k_0_0 < k_bound; ++_k_0_0) {
96 + int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
97 + __syncthreads();
98 + // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
99 + if (ld_A_flag)
100 + {
101 + *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
102 + }
103 + else
104 + {
105 + *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
106 + }
107 +
108 + // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
109 + uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
110 + uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
111 + uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
112 + /*
113 + if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
114 + printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
115 + }
116 + */
117 + // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
118 + int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
119 +
120 + for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 8; ++ax0_ax1_fused_0) {
121 +
122 + // B: 32 x 136 (128+8) float16
123 + // each warp: 32 x 4
124 + // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
125 + // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
126 + // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
127 + uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
128 + uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
129 + //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
130 +
131 + // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
132 + // - zero and * scale
133 + // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
134 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
135 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
136 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
137 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
138 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
139 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
140 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
141 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
142 + /*
143 + if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
144 + printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
145 + }
146 + */
147 +
148 + // write back
149 + *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (128 + 8)) = B_loaded_fp16;
150 + }
151 + __syncthreads();
152 +
153 + for (int k_0_1 = 0; k_0_1 < 2; ++k_0_1) {
154 + {
155 + unsigned int addr;
156 + __asm__ __volatile__(
157 + "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
158 + : "=r"(addr)
159 + : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
160 + );
161 +
162 +
163 + __asm__ __volatile__(
164 + "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
165 + "{%0, %1, %2, %3}, [%4];\n"
166 + : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
167 + : "r"(addr)
168 + );
169 + }
170 +
171 + for (int ax1_0 = 0; ax1_0 < 4; ++ax1_0) {
172 + {
173 + unsigned int addr;
174 + __asm__ __volatile__(
175 + "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
176 + : "=r"(addr)
177 + : "l"((void *)((&(B_shared[(((k_0_1 * 2176) + (((int)threadIdx.y) * 64)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 136) + ((((int)threadIdx.x) >> 4) * 8))))
178 + );
179 + __asm__ __volatile__(
180 + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
181 + "{%0, %1, %2, %3}, [%4];\n"
182 + : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
183 + : "r"(addr)
184 + );
185 + }
186 + }
187 + for (int j_0_4 = 0; j_0_4 < 4; ++j_0_4) {
188 + {
189 + __asm__ __volatile__(
190 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
191 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
192 + : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
193 + : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
194 + }
195 +
196 + {
197 + __asm__ __volatile__(
198 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
199 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
200 + : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
201 + : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
202 + }
203 + }
204 + }
205 + }
206 +
207 +// TODO: Shang: Hoist loop invariance.
208 + for (int ax1_0_1 = 0; ax1_0_1 < 4; ++ax1_0_1) {
209 + for (int local_id = 0; local_id < 8; ++local_id) {
210 + int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
211 + if (row_offset < M)
212 + {
213 + *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
214 + }
215 + }
216 + }
217 +}
218 +
219 +
220 +__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
221 +{
222 + static constexpr uint32_t ZERO = 0x0;
223 + float C_warp[32];
224 + __shared__ half A_shared[16 * (32 + 8)];
225 + __shared__ half B_shared[32 * (64 + 8)];
226 +
227 + __shared__ half scaling_factors_shared[64];
228 + __shared__ half zeros_shared[64];
229 +
230 + int j_factors1 = ((OC + 64 - 1) / 64);
231 +
232 + int blockIdx_x = 0;
233 + int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
234 + int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
235 +
236 + half A_shared_warp[8];
237 + half B_shared_warp[16];
238 + for (int j_0_4_init = 0; j_0_4_init < 2; ++j_0_4_init) {
239 + for (int i = 0; i < 8; ++i) {
240 + C_warp[(j_0_4_init * 8) + i] = 0.0;
241 + }
242 + }
243 +
244 + static constexpr int row_stride_warp = 32 * 8 / 32;
245 + static constexpr int row_stride = 2 * 32 * 8 / 64;
246 + bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 64;
247 + // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
248 + bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
249 + // bool wb_C_flag = (threadIdx.x / 4) < M;
250 +
251 + half* A_ptr = A
252 + + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
253 + + (((int)threadIdx.x) % (32 / 8)) * 8;
254 +
255 + int* B_ptr = B
256 + + ((int)threadIdx.y) * (OC / 8) * 4
257 + + (((int)threadIdx.x) / (64 / 8)) * (OC / 8)
258 + + (((int)blockIdx_y) % j_factors1) * (64 / 8)
259 + + (((int)threadIdx.x) % (64 / 8)) * 1;
260 +// Why * 1 in the above line?
261 +
262 + half* A_shared_ptr = A_shared
263 + + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
264 + + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
265 + + (((int)threadIdx.x) % (32 / 8) ) * 8;
266 +
267 + half* B_shared_ptr = B_shared
268 + + ((int)threadIdx.y) * (row_stride / 2) * (64 + 8)
269 + + (((int)threadIdx.x) / (64 / 8)) * (64 + 8)
270 + + (((int)threadIdx.x) % (64 / 8)) * 8;
271 +
272 + int* zeros_ptr = zeros
273 + + (((int)blockIdx_y) % j_factors1) * (64 / 8)
274 + + ((int)threadIdx.x) % (64 / 8);
275 +
276 + half* scaling_factors_ptr = scaling_factors
277 + + (((int)blockIdx_y) % j_factors1) * (64)
278 + + (((int)threadIdx.x) % (64 / 8)) * 8;
279 +
280 + half* C_ptr = C
281 + + blockIdx_z * M * OC // blockIdz.x -> split_k dim
282 + + (((int)blockIdx_y) % j_factors1) * 64
283 + + ((int)threadIdx.y) * 32
284 + + (((int)threadIdx.x) % 4) * 2;
285 +
286 + // preload s.f. and zeros
287 + int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
288 + if ((k_bound - 1) * split_k_iters * 32 + blockIdx_z * 32 >= IC) k_bound -= 1;
289 + for (int _k_0_0 = 0; _k_0_0 < k_bound; ++_k_0_0) {
290 + int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
291 + __syncthreads();
292 + // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
293 + if (ld_A_flag)
294 + {
295 + *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
296 + }
297 + else
298 + {
299 + *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
300 + }
301 +
302 + // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
303 + uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
304 + uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
305 + uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
306 + /*
307 + if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
308 + printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
309 + }
310 + */
311 + // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
312 + int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
313 +
314 + for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 4; ++ax0_ax1_fused_0) {
315 +
316 + // B: 32 x 136 (128+8) float16
317 + // each warp: 32 x 4
318 + // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
319 + // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
320 + // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
321 + uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
322 + uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
323 + //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
324 +
325 + // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
326 + // - zero and * scale
327 + // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
328 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
329 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
330 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
331 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
332 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
333 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
334 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
335 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
336 + /*
337 + if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
338 + printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
339 + }
340 + */
341 +
342 + // write back
343 + *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (64 + 8)) = B_loaded_fp16;
344 + }
345 + __syncthreads();
346 +
347 + for (int k_0_1 = 0; k_0_1 < 2; ++k_0_1)
348 + {
349 + {
350 + unsigned int addr;
351 + __asm__ __volatile__(
352 + "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
353 + : "=r"(addr)
354 + : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
355 + );
356 + __asm__ __volatile__(
357 + "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
358 + "{%0, %1, %2, %3}, [%4];\n"
359 + : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
360 + : "r"(addr)
361 + );
362 + }
363 +
364 +
365 + for (int ax1_0 = 0; ax1_0 < 2; ++ax1_0)
366 + {
367 + {
368 + unsigned int addr;
369 + __asm__ __volatile__(
370 + "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
371 + : "=r"(addr)
372 + : "l"((void *)((&(B_shared[(((k_0_1 * 1152) + (((int)threadIdx.y) * 32)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 72) + ((((int)threadIdx.x) >> 4) * 8))))
373 + );
374 + __asm__ __volatile__(
375 + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
376 + "{%0, %1, %2, %3}, [%4];\n"
377 + : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
378 + : "r"(addr)
379 + );
380 + }
381 + }
382 +
383 + for (int j_0_4 = 0; j_0_4 < 2; ++j_0_4)
384 + {
385 +
386 + {
387 + __asm__ __volatile__(
388 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
389 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
390 + : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
391 + : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
392 + }
393 +
394 + {
395 + __asm__ __volatile__(
396 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
397 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
398 + : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
399 + : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
400 + }
401 + }
402 + }
403 + }
404 +
405 +// TODO: Shang: Hoist loop invariance.
406 + for (int ax1_0_1 = 0; ax1_0_1 < 2; ++ax1_0_1) {
407 + for (int local_id = 0; local_id < 8; ++local_id) {
408 + int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
409 + if (row_offset < M)
410 + {
411 + *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
412 + }
413 + }
414 + }
415 +}
416 +
417 +// in_feats: M, IC [float16]
418 +// kernel: IC, OC // 8 [int32] -> cast to IC, OC [uint4b]
419 +// scaling_factors: IC // G, OC [float16]
420 +// zeros: IC // G, OC // 8 [int32] -> cast to IC // G, OC [uint4b]
421 +// assume that batch_size < 16 for now
422 +
423 +torch::Tensor awq_gemm(
424 + torch::Tensor _in_feats,
425 + torch::Tensor _kernel,
426 + torch::Tensor _scaling_factors,
427 + torch::Tensor _zeros,
428 + int split_k_iters)
429 +{
430 + int num_in_feats = _in_feats.size(0);
431 + int num_in_channels = _in_feats.size(1);
432 + const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
433 +
434 + auto options = torch::TensorOptions().dtype(_in_feats.dtype()).device(_in_feats.device());
435 + at::Tensor _out_feats = torch::empty({split_k_iters, num_in_feats, _kernel.size(1) * 8}, options);
436 + int num_out_feats = _out_feats.size(-2);
437 + int num_out_channels = _out_feats.size(-1);
438 +
439 + auto in_feats = reinterpret_cast<half*>(_in_feats.data_ptr<at::Half>());
440 + auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
441 + auto out_feats = reinterpret_cast<half*>(_out_feats.data_ptr<at::Half>());
442 + auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
443 + auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
444 + int group_size = num_in_channels / _scaling_factors.size(0);
445 +
446 + if (num_out_channels % 64 != 0)
447 + throw std::invalid_argument("OC is not multiple of cta_N = 64");
448 + if (num_out_channels % 8 != 0)
449 + throw std::invalid_argument("OC is not multiple of pack_num = 8");
450 + if (group_size % 32 != 0)
451 + throw std::invalid_argument("Group size should be a multiple of 32");
452 + if (num_out_channels % group_size != 0)
453 + throw std::invalid_argument("OC is not multiple of Group size");
454 +
455 + if (num_out_channels % 128 == 0)
456 + {
457 + int j_factors1 = num_out_channels / 128 / 1;
458 + dim3 num_blocks((num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
459 + // threadIdx.x: 32
460 + // threadIdx.y: i_factors[2] * j_factors[2]
461 + dim3 threads_per_block(32, 2);
462 + gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
463 + group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
464 + }
465 + else if (num_out_channels % 64 == 0)
466 + {
467 + int j_factors1 = num_out_channels / 64 / 1;
468 + dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
469 +
470 + // threadIdx.x: 32
471 + // threadIdx.y: i_factors[2] * j_factors[2]
472 + dim3 threads_per_block(32, 2);
473 + gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
474 + group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
475 + }
476 + return _out_feats.sum(0);
477 +}
setup.py
+14
@@ -146,6 +146,20 @@ activation_extension = CUDAExtension(
146 )
147 ext_modules.append(activation_extension)
148
149 +# Quantization kernels.
150 +quantization_extension = CUDAExtension(
151 + name="vllm.quantization_ops",
152 + sources=[
153 + "csrc/quantization.cpp",
154 + "csrc/quantization/awq/gemm_kernels.cu",
155 + ],
156 + extra_compile_args={
157 + "cxx": CXX_FLAGS,
158 + "nvcc": NVCC_FLAGS,
159 + },
160 +)
161 +ext_modules.append(quantization_extension)
162 +
163
164 def get_path(*filepath) -> str:
165 return os.path.join(ROOT_DIR, *filepath)
vllm/config.py
+16
@@ -43,6 +43,8 @@ class ModelConfig:
43 version.
44 max_model_len: Maximum length of a sequence (including prompt and
45 output). If None, will be derived from the model.
46 + quantization: Quantization method that was used to quantize the model
47 + weights. If None, we assume the model weights are not quantized.
48 """
49
50 def __init__(
@@ -57,6 +59,7 @@ class ModelConfig:
59 seed: int,
60 revision: Optional[str],
61 max_model_len: Optional[int] = None,
62 + quantization: Optional[str] = None,
63 ) -> None:
64 self.model = model
65 self.tokenizer = tokenizer
@@ -66,11 +69,13 @@ class ModelConfig:
69 self.load_format = load_format
70 self.seed = seed
71 self.revision = revision
72 + self.quantization = quantization
73
74 self.hf_config = get_config(model, trust_remote_code, revision)
75 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
76 self._verify_load_format()
77 self._verify_tokenizer_mode()
78 + self._verify_quantization()
79 self.max_model_len = None
80 if max_model_len is not None:
81 derived_max_model_len = self.get_max_model_len()
@@ -100,6 +105,17 @@ class ModelConfig:
105 "either 'auto' or 'slow'.")
106 self.tokenizer_mode = tokenizer_mode
107
108 + def _verify_quantization(self) -> None:
109 + supported_quantization = ["awq"]
110 + if self.quantization is None:
111 + return
112 + quantization = self.quantization.lower()
113 + if quantization not in supported_quantization:
114 + raise ValueError(
115 + f"Unknown quantization: {self.quantization}. Must be one of "
116 + f"{supported_quantization}.")
117 + self.quantization = quantization
118 +
119 def verify_with_parallel_config(
120 self,
121 parallel_config: "ParallelConfig",
vllm/engine/arg_utils.py
+9 -3
@@ -29,6 +29,7 @@ class EngineArgs:
29 max_num_seqs: int = 256
30 disable_log_stats: bool = False
31 revision: Optional[str] = None
32 + quantization: Optional[str] = None
33
34 def __post_init__(self):
35 if self.tokenizer is None:
@@ -88,7 +89,6 @@ class EngineArgs:
89 'a numpy cache to speed up the loading. '
90 '"dummy" will initialize the weights with random values, '
91 'which is mainly for profiling.')
91 - # TODO(woosuk): Support FP32.
92 parser.add_argument(
93 '--dtype',
94 type=str,
@@ -150,6 +150,13 @@ class EngineArgs:
150 parser.add_argument('--disable-log-stats',
151 action='store_true',
152 help='disable logging statistics')
153 + # Quantization settings.
154 + parser.add_argument('--quantization',
155 + '-q',
156 + type=str,
157 + choices=['awq', None],
158 + default=None,
159 + help='Method used to quantize the weights')
160 return parser
161
162 @classmethod
@@ -163,12 +170,11 @@ class EngineArgs:
170 def create_engine_configs(
171 self,
172 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
166 - # Initialize the configs.
173 model_config = ModelConfig(self.model, self.tokenizer,
174 self.tokenizer_mode, self.trust_remote_code,
175 self.download_dir, self.load_format,
176 self.dtype, self.seed, self.revision,
171 - self.max_model_len)
177 + self.max_model_len, self.quantization)
178 cache_config = CacheConfig(self.block_size,
179 self.gpu_memory_utilization,
180 self.swap_space)
vllm/engine/llm_engine.py
+1
@@ -80,6 +80,7 @@ class LLMEngine:
80 f"download_dir={model_config.download_dir!r}, "
81 f"load_format={model_config.load_format}, "
82 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
83 + f"quantization={model_config.quantization}, "
84 f"seed={model_config.seed})")
85 # TODO(woosuk): Print more configs in debug mode.
86
vllm/model_executor/layers/quantized_linear/__init__.py new
+37
@@ -0,0 +1,37 @@
1 +from vllm.model_executor.layers.quantized_linear.awq import (
2 + AWQColumnParallelLinear, AWQRowParallelLinear)
3 +from vllm.model_executor.parallel_utils.tensor_parallel import (
4 + ColumnParallelLinear, RowParallelLinear)
5 +
6 +_QUANTIZED_LINEAR_REGISTRY = {
7 + "awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
8 +}
9 +
10 +
11 +class ParallelLinear:
12 +
13 + @classmethod
14 + def column(cls, *args, **kwargs) -> ColumnParallelLinear:
15 + quant_config = kwargs.get("quant_config", None)
16 + if quant_config is None:
17 + return ColumnParallelLinear(*args, **kwargs)
18 +
19 + name = quant_config.get_name()
20 + if name not in _QUANTIZED_LINEAR_REGISTRY:
21 + raise ValueError(f"No quantized linear is found for {name}")
22 +
23 + quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][0]
24 + return quant_linear_cls(*args, **kwargs)
25 +
26 + @classmethod
27 + def row(cls, *args, **kwargs) -> RowParallelLinear:
28 + quant_config = kwargs.get("quant_config", None)
29 + if quant_config is None:
30 + return RowParallelLinear(*args, **kwargs)
31 +
32 + name = quant_config.get_name()
33 + if name not in _QUANTIZED_LINEAR_REGISTRY:
34 + raise ValueError(f"No quantized linear is found for {name}")
35 +
36 + quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][1]
37 + return quant_linear_cls(*args, **kwargs)
vllm/model_executor/layers/quantized_linear/awq.py new
+102
@@ -0,0 +1,102 @@
1 +from typing import Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm import quantization_ops
7 +from vllm.model_executor.parallel_utils.tensor_parallel.layers import (
8 + ColumnParallelLinear, RowParallelLinear)
9 +
10 +
11 +class AWQColumnParallelLinear(ColumnParallelLinear):
12 +
13 + def create_weights(self, dtype: torch.dtype) -> None:
14 + assert self.input_size % self.quant_config.weight_bits == 0
15 + assert (self.output_size_per_partition %
16 + self.quant_config.pack_factor == 0)
17 + self.qweight = Parameter(
18 + torch.empty(
19 + self.input_size,
20 + self.output_size_per_partition //
21 + self.quant_config.pack_factor,
22 + device="cuda",
23 + dtype=torch.int32,
24 + ),
25 + requires_grad=False,
26 + )
27 + self.qzeros = Parameter(
28 + torch.empty(
29 + self.input_size // self.quant_config.group_size,
30 + self.output_size_per_partition //
31 + self.quant_config.pack_factor,
32 + device="cuda",
33 + dtype=torch.int32,
34 + ),
35 + requires_grad=False,
36 + )
37 + self.scales = Parameter(
38 + torch.empty(
39 + self.input_size // self.quant_config.group_size,
40 + self.output_size_per_partition,
41 + device="cuda",
42 + dtype=dtype,
43 + ),
44 + requires_grad=False,
45 + )
46 +
47 + def apply_weights(
48 + self,
49 + x: torch.Tensor,
50 + bias: Optional[torch.Tensor],
51 + ) -> torch.Tensor:
52 + pack_factor = self.quant_config.pack_factor
53 + out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
54 + reshaped_x = x.reshape(-1, x.shape[-1])
55 + out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
56 + self.qzeros, pack_factor)
57 + if bias is not None:
58 + out = out + bias
59 + return out.reshape(out_shape)
60 +
61 +
62 +class AWQRowParallelLinear(RowParallelLinear):
63 +
64 + def create_weights(self, dtype: torch.dtype) -> None:
65 + assert (self.input_size_per_partition %
66 + self.quant_config.weight_bits == 0)
67 + assert self.output_size % self.quant_config.pack_factor == 0
68 + self.qweight = Parameter(
69 + torch.empty(
70 + self.input_size_per_partition,
71 + self.output_size // self.quant_config.pack_factor,
72 + device="cuda",
73 + dtype=torch.int32,
74 + ),
75 + requires_grad=False,
76 + )
77 + self.qzeros = Parameter(
78 + torch.empty(
79 + self.input_size_per_partition // self.quant_config.group_size,
80 + self.output_size // self.quant_config.pack_factor,
81 + device="cuda",
82 + dtype=torch.int32,
83 + ),
84 + requires_grad=False,
85 + )
86 + self.scales = Parameter(
87 + torch.empty(
88 + self.input_size_per_partition // self.quant_config.group_size,
89 + self.output_size,
90 + device="cuda",
91 + dtype=dtype,
92 + ),
93 + requires_grad=False,
94 + )
95 +
96 + def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
97 + pack_factor = self.quant_config.pack_factor
98 + out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
99 + reshaped_x = x.reshape(-1, x.shape[-1])
100 + out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
101 + self.qzeros, pack_factor)
102 + return out.reshape(out_shape)
vllm/model_executor/model_loader.py
+28 -2
@@ -8,7 +8,8 @@ from transformers import PretrainedConfig
8
9 from vllm.config import ModelConfig
10 from vllm.model_executor.models import * # pylint: disable=wildcard-import
11 -from vllm.model_executor.weight_utils import initialize_dummy_weights
11 +from vllm.model_executor.weight_utils import (get_quant_config,
12 + initialize_dummy_weights)
13
14 # TODO(woosuk): Lazy-load the model classes.
15 _MODEL_REGISTRY = {
@@ -30,6 +31,11 @@ _MODEL_REGISTRY = {
31 "RWForCausalLM": FalconForCausalLM,
32 }
33
34 +# FIXME(woosuk): Remove this once all models support quantization.
35 +_MODEL_CLASSES_SUPPORT_QUANTIZATION = [
36 + LlamaForCausalLM,
37 +]
38 +
39
40 @contextlib.contextmanager
41 def _set_default_torch_dtype(dtype: torch.dtype):
@@ -52,10 +58,30 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
58
59 def get_model(model_config: ModelConfig) -> nn.Module:
60 model_class = _get_model_architecture(model_config.hf_config)
61 +
62 + # Get the quantization config.
63 + quant_config = None
64 + if model_config.quantization is not None:
65 + if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
66 + raise ValueError(
67 + f"Quantization is not supported for {model_class}.")
68 + quant_config = get_quant_config(model_config.quantization,
69 + model_config.model,
70 + model_config.download_dir)
71 + supported_dtypes = quant_config.get_supported_act_dtypes()
72 + if model_config.dtype not in supported_dtypes:
73 + raise ValueError(
74 + f"{model_config.dtype} is not supported for quantization "
75 + f"method {model_config.quantization}. Supported dtypes: "
76 + f"{supported_dtypes}")
77 +
78 with _set_default_torch_dtype(model_config.dtype):
79 # Create a model instance.
80 # The weights will be initialized as empty tensors.
58 - model = model_class(model_config.hf_config)
81 + if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
82 + model = model_class(model_config.hf_config, quant_config)
83 + else:
84 + model = model_class(model_config.hf_config)
85 if model_config.load_format == "dummy":
86 model = model.cuda()
87 # NOTE(woosuk): For accurate performance evaluation, we assign
vllm/model_executor/models/llama.py
+91 -34
@@ -36,13 +36,15 @@ from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38 from vllm.model_executor.layers.sampler import Sampler
39 -from vllm.model_executor.weight_utils import (
40 - load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
41 - hf_model_weights_iterator)
39 +from vllm.model_executor.layers.quantized_linear import ParallelLinear
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 from vllm.model_executor.parallel_utils.tensor_parallel import (
45 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
43 + VocabParallelEmbedding)
44 +from vllm.model_executor.quantization_utils import QuantizationConfig
45 +from vllm.model_executor.weight_utils import (
46 + load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
47 + hf_model_weights_iterator)
48 from vllm.sequence import SamplerOutput
49
50 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -55,18 +57,21 @@ class LlamaMLP(nn.Module):
57 hidden_size: int,
58 intermediate_size: int,
59 hidden_act: str,
58 - ):
60 + quant_config: Optional[QuantizationConfig] = None,
61 + ) -> None:
62 super().__init__()
60 - self.gate_up_proj = ColumnParallelLinear(hidden_size,
61 - 2 * intermediate_size,
62 - bias=False,
63 - gather_output=False,
64 - perform_initialization=False)
65 - self.down_proj = RowParallelLinear(intermediate_size,
66 - hidden_size,
67 - bias=False,
68 - input_is_parallel=True,
69 - perform_initialization=False)
63 + self.gate_up_proj = ParallelLinear.column(hidden_size,
64 + 2 * intermediate_size,
65 + bias=False,
66 + gather_output=False,
67 + perform_initialization=False,
68 + quant_config=quant_config)
69 + self.down_proj = ParallelLinear.row(intermediate_size,
70 + hidden_size,
71 + bias=False,
72 + input_is_parallel=True,
73 + perform_initialization=False,
74 + quant_config=quant_config)
75 if hidden_act != "silu":
76 raise ValueError(f"Unsupported activation: {hidden_act}. "
77 "Only silu is supported for now.")
@@ -87,7 +92,8 @@ class LlamaAttention(nn.Module):
92 num_heads: int,
93 num_kv_heads: int,
94 rope_theta: float = 10000,
90 - ):
95 + quant_config: Optional[QuantizationConfig] = None,
96 + ) -> None:
97 super().__init__()
98 self.hidden_size = hidden_size
99 tp_size = get_tensor_model_parallel_world_size()
@@ -103,20 +109,22 @@ class LlamaAttention(nn.Module):
109 self.scaling = self.head_dim**-0.5
110 self.rope_theta = rope_theta
111
106 - self.qkv_proj = ColumnParallelLinear(
112 + self.qkv_proj = ParallelLinear.column(
113 hidden_size,
114 (self.total_num_heads + 2 * self.total_num_kv_heads) *
115 self.head_dim,
116 bias=False,
117 gather_output=False,
118 perform_initialization=False,
119 + quant_config=quant_config,
120 )
114 - self.o_proj = RowParallelLinear(
121 + self.o_proj = ParallelLinear.row(
122 self.total_num_heads * self.head_dim,
123 hidden_size,
124 bias=False,
125 input_is_parallel=True,
126 perform_initialization=False,
127 + quant_config=quant_config,
128 )
129 self.attn = PagedAttentionWithRoPE(self.num_heads,
130 self.head_dim,
@@ -144,7 +152,11 @@ class LlamaAttention(nn.Module):
152
153 class LlamaDecoderLayer(nn.Module):
154
147 - def __init__(self, config: LlamaConfig):
155 + def __init__(
156 + self,
157 + config: LlamaConfig,
158 + quant_config: Optional[QuantizationConfig] = None,
159 + ) -> None:
160 super().__init__()
161 self.hidden_size = config.hidden_size
162 # Requires transformers > 4.32.0
@@ -154,11 +166,13 @@ class LlamaDecoderLayer(nn.Module):
166 num_heads=config.num_attention_heads,
167 num_kv_heads=config.num_key_value_heads,
168 rope_theta=rope_theta,
169 + quant_config=quant_config,
170 )
171 self.mlp = LlamaMLP(
172 hidden_size=self.hidden_size,
173 intermediate_size=config.intermediate_size,
174 hidden_act=config.hidden_act,
175 + quant_config=quant_config,
176 )
177 self.input_layernorm = RMSNorm(config.hidden_size,
178 eps=config.rms_norm_eps)
@@ -195,7 +209,11 @@ class LlamaDecoderLayer(nn.Module):
209
210 class LlamaModel(nn.Module):
211
198 - def __init__(self, config: LlamaConfig):
212 + def __init__(
213 + self,
214 + config: LlamaConfig,
215 + quant_config: Optional[QuantizationConfig] = None,
216 + ) -> None:
217 super().__init__()
218 self.config = config
219 self.padding_idx = config.pad_token_id
@@ -205,7 +223,8 @@ class LlamaModel(nn.Module):
223 self.embed_tokens = VocabParallelEmbedding(
224 vocab_size, config.hidden_size, perform_initialization=False)
225 self.layers = nn.ModuleList([
208 - LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
226 + LlamaDecoderLayer(config, quant_config)
227 + for _ in range(config.num_hidden_layers)
228 ])
229 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
230
@@ -237,16 +256,23 @@ class LlamaModel(nn.Module):
256
257 class LlamaForCausalLM(nn.Module):
258
240 - def __init__(self, config):
259 + def __init__(
260 + self,
261 + config: LlamaConfig,
262 + quant_config: Optional[QuantizationConfig] = None,
263 + ) -> None:
264 super().__init__()
265 self.config = config
243 - self.model = LlamaModel(config)
266 + self.quant_config = quant_config
267 + self.model = LlamaModel(config, quant_config)
268 vocab_size = ((config.vocab_size + 63) // 64) * 64
245 - self.lm_head = ColumnParallelLinear(config.hidden_size,
246 - vocab_size,
247 - bias=False,
248 - gather_output=False,
249 - perform_initialization=False)
269 + # NOTE: The LM head is not quantized.
270 + self.lm_head = ParallelLinear.column(config.hidden_size,
271 + vocab_size,
272 + bias=False,
273 + gather_output=False,
274 + perform_initialization=False,
275 + quant_config=None)
276 self.sampler = Sampler(config.vocab_size)
277
278 def forward(
@@ -263,16 +289,28 @@ class LlamaForCausalLM(nn.Module):
289 input_metadata)
290 return next_tokens
291
266 - _column_parallel_weights = [
267 - "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
268 - ]
269 - _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
292 + _column_parallel_layers = []
293 + _row_parallel_layers = ["o_proj", "down_proj"]
294
295 def load_weights(self,
296 model_name_or_path: str,
297 cache_dir: Optional[str] = None,
298 load_format: str = "auto",
299 revision: Optional[str] = None):
300 + if self.quant_config is None:
301 + weight_suffixes = ["weight"]
302 + else:
303 + weight_suffixes = self.quant_config.get_tp_tensor_names()
304 +
305 + column_parallel_weights: List[str] = []
306 + for layer in self._column_parallel_layers:
307 + for suffix in weight_suffixes:
308 + column_parallel_weights.append(f"{layer}.{suffix}")
309 + row_parallel_weights: List[str] = []
310 + for layer in self._row_parallel_layers:
311 + for suffix in weight_suffixes:
312 + row_parallel_weights.append(f"{layer}.{suffix}")
313 +
314 tp_size = get_tensor_model_parallel_world_size()
315 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
316 q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -293,11 +331,25 @@ class LlamaForCausalLM(nn.Module):
331 if "rotary_emb.inv_freq" in name:
332 continue
333
334 + is_packed = False
335 + is_transposed = False
336 + if self.quant_config is not None:
337 + is_packed = self.quant_config.is_packed(name)
338 + is_transposed = self.quant_config.is_transposed(name)
339 + if is_transposed:
340 + loaded_weight = loaded_weight.T
341 +
342 is_attention_weight = False
343 for weight_name, shard_size, offset in attention_weight_specs:
344 if weight_name not in name:
345 continue
346 param = state_dict[name.replace(weight_name, "qkv_proj")]
347 + if is_transposed:
348 + param = param.T
349 +
350 + if is_packed:
351 + shard_size //= self.quant_config.pack_factor
352 + offset //= self.quant_config.pack_factor
353
354 loaded_weight = loaded_weight[
355 shard_size * tensor_model_parallel_rank:shard_size *
@@ -316,6 +368,9 @@ class LlamaForCausalLM(nn.Module):
368 if weight_name not in name:
369 continue
370 param = state_dict[name.replace(weight_name, "gate_up_proj")]
371 + if is_transposed:
372 + param = param.T
373 +
374 shard_size = param.shape[0] // 2
375 loaded_weight = loaded_weight[
376 shard_size * tensor_model_parallel_rank:shard_size *
@@ -330,6 +385,8 @@ class LlamaForCausalLM(nn.Module):
385 continue
386
387 param = state_dict[name]
388 + if is_transposed:
389 + param = param.T
390
391 if "embed_tokens" in name or "lm_head" in name:
392 load_padded_tensor_parallel_vocab(param, loaded_weight,
@@ -337,6 +394,6 @@ class LlamaForCausalLM(nn.Module):
394 continue
395
396 load_tensor_parallel_weights(param, loaded_weight, name,
340 - self._column_parallel_weights,
341 - self._row_parallel_weights,
397 + column_parallel_weights,
398 + row_parallel_weights,
399 tensor_model_parallel_rank)
vllm/model_executor/parallel_utils/tensor_parallel/layers.py
+47 -130
@@ -4,7 +4,7 @@
4
5 # Parts of the code here are adapted from PyTorch
6 # repo: https://github.com/pytorch/pytorch
7 -
7 +from typing import Optional
8
9 import torch
10 import torch.nn.functional as F
@@ -16,13 +16,11 @@ from vllm.model_executor.parallel_utils.parallel_state import (
16 get_tensor_model_parallel_world_size,
17 )
18 from .mappings import (
19 - copy_to_tensor_model_parallel_region,
19 gather_from_tensor_model_parallel_region,
20 reduce_from_tensor_model_parallel_region,
21 scatter_to_tensor_model_parallel_region,
22 )
23
25 -from .random import get_cuda_rng_tracker
24 from .utils import (
25 divide,
26 VocabUtility,
@@ -65,59 +63,6 @@ def copy_tensor_model_parallel_attributes(destination_tensor, source_tensor):
63 maybe_copy(attribute)
64
65
68 -def _initialize_affine_weight_gpu(weight, init_method,
69 - partition_dim, stride=1):
70 - """Initialize affine weight for model parallel on GPU."""
71 -
72 - set_tensor_model_parallel_attributes(tensor=weight,
73 - is_parallel=True,
74 - dim=partition_dim,
75 - stride=stride)
76 -
77 - with get_cuda_rng_tracker().fork():
78 - init_method(weight)
79 -
80 -
81 -def _initialize_affine_weight_cpu(weight, output_size, input_size,
82 - per_partition_size, partition_dim,
83 - init_method, stride=1,
84 - return_master_weight=False,
85 - *, params_dtype=None):
86 - """Initialize affine weight for model parallel.
87 -
88 - Build the master weight on all processes and scatter
89 - the relevant chunk."""
90 -
91 - set_tensor_model_parallel_attributes(tensor=weight,
92 - is_parallel=True,
93 - dim=partition_dim,
94 - stride=stride)
95 -
96 - if params_dtype is None:
97 - params_dtype = torch.get_default_dtype()
98 -
99 - # Initialize master weight
100 - master_weight = torch.empty(output_size, input_size,
101 - dtype=torch.float,
102 - requires_grad=False)
103 - init_method(master_weight)
104 - master_weight = master_weight.to(dtype=params_dtype)
105 -
106 - # Split and copy
107 - per_partition_per_stride_size = divide(per_partition_size, stride)
108 - weight_list = torch.split(master_weight, per_partition_per_stride_size,
109 - dim=partition_dim)
110 - rank = get_tensor_model_parallel_rank()
111 - world_size = get_tensor_model_parallel_world_size()
112 - my_weight_list = weight_list[rank::world_size]
113 -
114 - with torch.no_grad():
115 - torch.cat(my_weight_list, dim=partition_dim, out=weight)
116 - if return_master_weight:
117 - return master_weight
118 - return None
119 -
120 -
66 class VocabParallelEmbedding(torch.nn.Module):
67 """Embedding parallelized in the vocabulary dimension.
68
@@ -140,6 +85,9 @@ class VocabParallelEmbedding(torch.nn.Module):
85 use_cpu_initialization: bool=False,
86 perform_initialization: bool=True):
87 super(VocabParallelEmbedding, self).__init__()
88 + assert not perform_initialization
89 + assert not use_cpu_initialization
90 +
91 # Keep the input dimensions.
92 self.num_embeddings = num_embeddings
93 self.embedding_dim = embedding_dim
@@ -162,24 +110,10 @@ class VocabParallelEmbedding(torch.nn.Module):
110 self.num_embeddings_per_partition = self.vocab_end_index - \
111 self.vocab_start_index
112
165 - # Allocate weights and initialize.
166 - if use_cpu_initialization:
167 - self.weight = Parameter(torch.empty(
168 - self.num_embeddings_per_partition, self.embedding_dim,
169 - dtype=params_dtype))
170 - if perform_initialization:
171 - _initialize_affine_weight_cpu(
172 - self.weight, self.num_embeddings, self.embedding_dim,
173 - self.num_embeddings_per_partition, 0, init_method,
174 - params_dtype=params_dtype)
175 - else:
176 - self.weight = Parameter(torch.empty(
177 - self.num_embeddings_per_partition, self.embedding_dim,
178 - device=torch.cuda.current_device(), dtype=params_dtype))
179 - if perform_initialization:
180 - _initialize_affine_weight_gpu(self.weight, init_method,
181 - partition_dim=0, stride=1)
182 -
113 + self.weight = Parameter(torch.empty(
114 + self.num_embeddings_per_partition, self.embedding_dim,
115 + device=torch.cuda.current_device(), dtype=params_dtype))
116 +
117 def forward(self, input_):
118 if self.tensor_model_parallel_size > 1:
119 # Build the mask.
@@ -239,8 +173,11 @@ class ColumnParallelLinear(torch.nn.Module):
173 params_dtype=None,
174 use_cpu_initialization=False,
175 perform_initialization=True,
176 + quant_config=None,
177 ):
178 super(ColumnParallelLinear, self).__init__()
179 + assert not perform_initialization
180 + assert not use_cpu_initialization
181
182 # Keep input parameters
183 self.input_size = input_size
@@ -250,6 +187,7 @@ class ColumnParallelLinear(torch.nn.Module):
187 self.world_size = get_tensor_model_parallel_world_size()
188 self.output_size_per_partition = divide(output_size, self.world_size)
189 self.skip_bias_add = skip_bias_add
190 + self.quant_config = quant_config
191
192 if params_dtype is None:
193 params_dtype = torch.get_default_dtype()
@@ -257,33 +195,13 @@ class ColumnParallelLinear(torch.nn.Module):
195 # Parameters.
196 # Note: torch.nn.functional.linear performs XA^T + b and as a result
197 # we allocate the transpose.
260 - # Initialize weight.
261 - if use_cpu_initialization:
262 - self.weight = Parameter(torch.empty(self.output_size_per_partition,
263 - self.input_size,
264 - dtype=params_dtype))
265 - if perform_initialization:
266 - self.master_weight = _initialize_affine_weight_cpu(
267 - self.weight, self.output_size, self.input_size,
268 - self.output_size_per_partition, 0, init_method,
269 - stride=stride, return_master_weight=keep_master_weight_for_test)
270 - else:
271 - self.weight = Parameter(torch.empty(
272 - self.output_size_per_partition, self.input_size,
273 - device=torch.cuda.current_device(), dtype=params_dtype))
274 - if perform_initialization:
275 - _initialize_affine_weight_gpu(self.weight, init_method,
276 - partition_dim=0, stride=stride)
198 + self.create_weights(params_dtype)
199
200 if bias:
279 - if use_cpu_initialization:
280 - self.bias = Parameter(torch.empty(
281 - self.output_size_per_partition, dtype=params_dtype))
282 - else:
283 - self.bias = Parameter(torch.empty(
284 - self.output_size_per_partition,
285 - device=torch.cuda.current_device(),
286 - dtype=params_dtype))
201 + self.bias = Parameter(torch.empty(
202 + self.output_size_per_partition,
203 + device=torch.cuda.current_device(),
204 + dtype=params_dtype))
205 set_tensor_model_parallel_attributes(self.bias, True, 0, stride)
206 # Always initialize bias to zero.
207 with torch.no_grad():
@@ -291,6 +209,17 @@ class ColumnParallelLinear(torch.nn.Module):
209 else:
210 self.register_parameter('bias', None)
211
212 + def create_weights(self, dtype: torch.dtype) -> None:
213 + self.weight = Parameter(torch.empty(
214 + self.output_size_per_partition, self.input_size,
215 + device=torch.cuda.current_device(), dtype=dtype))
216 +
217 + def apply_weights(
218 + self,
219 + x: torch.Tensor,
220 + bias: Optional[torch.Tensor],
221 + ) -> torch.Tensor:
222 + return F.linear(x, self.weight, bias)
223
224 def forward(self, input_):
225 """Forward of ColumnParallelLinear
@@ -306,7 +235,7 @@ class ColumnParallelLinear(torch.nn.Module):
235
236 input_parallel = input_
237 # Matrix multiply.
309 - output_parallel = F.linear(input_parallel, self.weight, bias)
238 + output_parallel = self.apply_weights(input_parallel, bias)
239 if self.gather_output:
240 # All-gather across the partitions.
241 output = gather_from_tensor_model_parallel_region(output_parallel)
@@ -361,8 +290,11 @@ class RowParallelLinear(torch.nn.Module):
290 use_cpu_initialization=False,
291 perform_initialization=True,
292 reduce_results=True,
293 + quant_config=None,
294 ):
295 super(RowParallelLinear, self).__init__()
296 + assert not perform_initialization
297 + assert not use_cpu_initialization
298
299 # Keep input parameters
300 self.input_size = input_size
@@ -376,47 +308,32 @@ class RowParallelLinear(torch.nn.Module):
308 self.world_size = get_tensor_model_parallel_world_size()
309 self.input_size_per_partition = divide(input_size, self.world_size)
310 self.skip_bias_add = skip_bias_add
311 + self.quant_config = quant_config
312 +
313 + self.create_weights(params_dtype)
314
315 if not reduce_results and (bias and not skip_bias_add):
316 raise ValueError("When not reduce the results, adding bias to the "
317 "results can lead to incorrect results")
318
384 - # Parameters.
385 - # Note: torch.nn.functional.linear performs XA^T + b and as a result
386 - # we allocate the transpose.
387 - # Initialize weight.
388 - if use_cpu_initialization:
389 - self.weight = Parameter(torch.empty(self.output_size,
390 - self.input_size_per_partition,
391 - dtype=params_dtype))
392 - if perform_initialization:
393 - self.master_weight = _initialize_affine_weight_cpu(
394 - self.weight, self.output_size, self.input_size,
395 - self.input_size_per_partition, 1, init_method,
396 - stride=stride, return_master_weight=keep_master_weight_for_test,
397 - params_dtype=params_dtype)
398 - else:
399 - self.weight = Parameter(torch.empty(
400 - self.output_size, self.input_size_per_partition,
401 - device=torch.cuda.current_device(), dtype=params_dtype))
402 - if perform_initialization:
403 - _initialize_affine_weight_gpu(self.weight, init_method,
404 - partition_dim=1, stride=stride)
319 if bias:
406 - if use_cpu_initialization:
407 - self.bias = Parameter(torch.empty(self.output_size,
408 - dtype=params_dtype))
409 - else:
410 - self.bias = Parameter(torch.empty(
411 - self.output_size, device=torch.cuda.current_device(),
412 - dtype=params_dtype))
320 + self.bias = Parameter(torch.empty(
321 + self.output_size, device=torch.cuda.current_device(),
322 + dtype=params_dtype))
323
324 # Always initialize bias to zero.
325 with torch.no_grad():
326 self.bias.zero_()
327 else:
328 self.register_parameter('bias', None)
419 - self.weight_t = self.weight.t()
329 +
330 + def create_weights(self, dtype: torch.dtype) -> None:
331 + self.weight = Parameter(torch.empty(
332 + self.output_size, self.input_size_per_partition,
333 + device=torch.cuda.current_device(), dtype=dtype))
334 +
335 + def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
336 + return F.linear(x, self.weight)
337
338 def forward(self, input_):
339 """Forward of RowParallelLinear
@@ -434,7 +351,7 @@ class RowParallelLinear(torch.nn.Module):
351 else:
352 input_parallel = scatter_to_tensor_model_parallel_region(input_)
353 # Matrix multiply.
437 - output_parallel = F.linear(input_parallel, self.weight)
354 + output_parallel = self.apply_weights(input_parallel)
355 if self.reduce_results and self.world_size > 1:
356 output_ = reduce_from_tensor_model_parallel_region(output_parallel)
357 else:
vllm/model_executor/quantization_utils/__init__.py new
+20
@@ -0,0 +1,20 @@
1 +from typing import Type
2 +
3 +from vllm.model_executor.quantization_utils.awq import AWQConfig
4 +from vllm.model_executor.quantization_utils.base import QuantizationConfig
5 +
6 +_QUANTIZATION_REGISTRY = {
7 + "awq": AWQConfig,
8 +}
9 +
10 +
11 +def get_quant_class(quantization: str) -> Type[QuantizationConfig]:
12 + if quantization not in _QUANTIZATION_REGISTRY:
13 + raise ValueError(f"Invalid quantization method: {quantization}")
14 + return _QUANTIZATION_REGISTRY[quantization]
15 +
16 +
17 +__all__ = [
18 + "QuantizationConfig",
19 + "get_quant_class",
20 +]
vllm/model_executor/quantization_utils/awq.py new
+67
@@ -0,0 +1,67 @@
1 +from typing import Any, Dict, List
2 +
3 +import torch
4 +
5 +from vllm.model_executor.quantization_utils.base import QuantizationConfig
6 +
7 +
8 +class AWQConfig(QuantizationConfig):
9 + """Config class for AWQ.
10 +
11 + Reference: https://arxiv.org/abs/2306.00978
12 + """
13 +
14 + def __init__(
15 + self,
16 + weight_bits: int,
17 + group_size: int,
18 + zero_point: bool,
19 + ) -> None:
20 + self.weight_bits = weight_bits
21 + self.group_size = group_size
22 + self.zero_point = zero_point
23 +
24 + if self.weight_bits != 4:
25 + raise ValueError(
26 + "Currently, only 4-bit weight quantization is supported for "
27 + f"AWQ, but got {self.weight_bits} bits.")
28 + self.pack_factor = 32 // self.weight_bits
29 +
30 + def __repr__(self) -> str:
31 + return (f"AWQConfig(weight_bits={self.weight_bits}, "
32 + f"group_size={self.group_size}, "
33 + f"zero_point={self.zero_point})")
34 +
35 + @classmethod
36 + def get_name(cls) -> str:
37 + return "awq"
38 +
39 + @classmethod
40 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41 + return [torch.half]
42 +
43 + @classmethod
44 + def get_config_filenames(cls) -> List[str]:
45 + return [
46 + "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
47 + "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq # pylint: disable=line-too-long
48 + ]
49 +
50 + @classmethod
51 + def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
52 + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
53 + group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
54 + zero_point = cls.get_from_keys(config, ["zero_point"])
55 + return cls(weight_bits, group_size, zero_point)
56 +
57 + @classmethod
58 + def get_packed_tensor_names(cls) -> List[str]:
59 + return ["qweight", "qzeros"]
60 +
61 + @classmethod
62 + def get_transposed_tensor_names(cls) -> List[str]:
63 + return ["qweight", "qzeros", "scales"]
64 +
65 + @classmethod
66 + def get_tp_tensor_names(cls) -> List[str]:
67 + return ["qweight", "qzeros", "scales"]
vllm/model_executor/quantization_utils/base.py new
+65
@@ -0,0 +1,65 @@
1 +from typing import Any, Dict, List
2 +
3 +import torch
4 +
5 +
6 +class QuantizationConfig:
7 +
8 + @classmethod
9 + def get_name(cls) -> str:
10 + """Name of the quantization method."""
11 + raise NotImplementedError
12 +
13 + @classmethod
14 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
15 + """List of supported activation dtypes."""
16 + raise NotImplementedError
17 +
18 + @classmethod
19 + def get_config_filenames(cls) -> List[str]:
20 + """List of filenames to search for in the model directory."""
21 + raise NotImplementedError
22 +
23 + @classmethod
24 + def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
25 + """Create a config class from the model's quantization config."""
26 + raise NotImplementedError
27 +
28 + @staticmethod
29 + def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
30 + """Get a value from the model's quantization config."""
31 + for key in keys:
32 + if key in config:
33 + return config[key]
34 + raise ValueError(f"Cannot find any of {keys} in the model's "
35 + "quantization config.")
36 +
37 + @classmethod
38 + def get_packed_tensor_names(cls) -> List[str]:
39 + raise NotImplementedError
40 +
41 + @classmethod
42 + def is_packed(cls, tensor_name: str) -> bool:
43 + """Returns True if a tensor is packed.
44 +
45 + A tensor is considered packed if each element in the tensor is a
46 + packed representation of multiple elements in the original tensor.
47 + For example, an INT32 element in the tensor may represent 8 INT4
48 + elements in the original tensor.
49 + """
50 + return any(tag in tensor_name for tag in cls.get_packed_tensor_names())
51 +
52 + @classmethod
53 + def get_transposed_tensor_names(cls) -> List[str]:
54 + raise NotImplementedError
55 +
56 + @classmethod
57 + def is_transposed(cls, tensor_name: str) -> bool:
58 + """Returns True if a tensor is transposed relative to nn.Linear.weight.
59 + """
60 + return any(tag in tensor_name
61 + for tag in cls.get_transposed_tensor_names())
62 +
63 + @classmethod
64 + def get_tp_tensor_names(cls) -> List[str]:
65 + raise NotImplementedError
vllm/model_executor/weight_utils.py
+48 -5
@@ -4,7 +4,7 @@ import glob
4 import json
5 import os
6 from collections import defaultdict
7 -from typing import Iterator, List, Optional, Tuple, Any
7 +from typing import Any, Iterator, List, Optional, Tuple
8
9 from huggingface_hub import snapshot_download
10 from safetensors.torch import load_file, save_file, safe_open
@@ -13,6 +13,8 @@ import torch
13 from tqdm.auto import tqdm
14
15 from vllm.logger import init_logger
16 +from vllm.model_executor.quantization_utils import get_quant_class
17 +from vllm.model_executor.quantization_utils.base import QuantizationConfig
18
19 logger = init_logger(__name__)
20
@@ -44,7 +46,7 @@ def _shared_pointers(tensors):
46 def convert_bin_to_safetensor_file(
47 pt_filename: str,
48 sf_filename: str,
47 -):
49 +) -> None:
50 loaded = torch.load(pt_filename, map_location="cpu")
51 if "state_dict" in loaded:
52 loaded = loaded["state_dict"]
@@ -78,16 +80,55 @@ def convert_bin_to_safetensor_file(
80 raise RuntimeError(f"The output tensors do not match for key {k}")
81
82
83 +# TODO(woosuk): Move this to other place.
84 +def get_quant_config(
85 + quantization: str,
86 + model_name_or_path: str,
87 + cache_dir: Optional[str] = None,
88 +) -> QuantizationConfig:
89 + is_local = os.path.isdir(model_name_or_path)
90 + if not is_local:
91 + # Download the config files.
92 + with get_lock(model_name_or_path, cache_dir):
93 + hf_folder = snapshot_download(model_name_or_path,
94 + allow_patterns="*.json",
95 + cache_dir=cache_dir,
96 + tqdm_class=Disabledtqdm)
97 + else:
98 + hf_folder = model_name_or_path
99 + config_files = glob.glob(os.path.join(hf_folder, "*.json"))
100 +
101 + quant_cls = get_quant_class(quantization)
102 + quant_config_files = [
103 + f for f in config_files if any(
104 + f.endswith(x) for x in quant_cls.get_config_filenames())
105 + ]
106 + if len(quant_config_files) == 0:
107 + raise ValueError(f"Cannot find the config file for {quantization}")
108 + if len(quant_config_files) > 1:
109 + raise ValueError(f"Found multiple config files for {quantization}: "
110 + f"{quant_config_files}")
111 +
112 + quant_config_file = quant_config_files[0]
113 + with open(quant_config_file, "r") as f:
114 + config = json.load(f)
115 + return quant_cls.from_config(config)
116 +
117 +
118 def prepare_hf_model_weights(
119 model_name_or_path: str,
120 cache_dir: Optional[str] = None,
121 use_safetensors: bool = False,
122 fall_back_to_pt: bool = True,
123 revision: Optional[str] = None,
87 -):
124 +) -> Tuple[str, List[str], bool]:
125 # Download model weights from huggingface.
126 is_local = os.path.isdir(model_name_or_path)
90 - allow_patterns = "*.safetensors" if use_safetensors else "*.bin"
127 + if use_safetensors:
128 + allow_patterns = ["*.safetensors"]
129 + else:
130 + # Some quantized models use .pt files for storing the weights.
131 + allow_patterns = ["*.bin", "*.pt"]
132 if not is_local:
133 # Use file lock to prevent multiple processes from
134 # downloading the same model weights at the same time.
@@ -99,7 +140,9 @@ def prepare_hf_model_weights(
140 revision=revision)
141 else:
142 hf_folder = model_name_or_path
102 - hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
143 + hf_weights_files: List[str] = []
144 + for pattern in allow_patterns:
145 + hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
146 if not use_safetensors:
147 hf_weights_files = [
148 x for x in hf_weights_files if not x.endswith("training_args.bin")