A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel] fix types used in aqlm and ggml kernels to support dynamo (#7596)
bnellnm committed
Aug 16, 2024 at 17:00 UTC
37fd47e7803fedd9715abceee8bdb57070fc09f4
7 files changed
+39
-53
csrc/ops.h
+8
-8
@@ -63,12 +63,12 @@ void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
63
torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
64
const torch::Tensor& codebooks,
65
const torch::Tensor& scales,
66
- const torch::Tensor& codebook_partition_sizes,
66
+ const std::vector<int64_t>& codebook_partition_sizes,
67
const std::optional<torch::Tensor>& bias);
68
69
-torch::Tensor aqlm_dequant(const torch::Tensor& codes,
70
- const torch::Tensor& codebooks,
71
- const torch::Tensor& codebook_partition_sizes);
69
+torch::Tensor aqlm_dequant(
70
+ const torch::Tensor& codes, const torch::Tensor& codebooks,
71
+ const std::vector<int64_t>& codebook_partition_sizes);
72
73
torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
74
torch::Tensor _scaling_factors, torch::Tensor _zeros,
@@ -107,13 +107,13 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
107
torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
108
int64_t size_n, int64_t num_bits);
109
110
-torch::Tensor ggml_dequantize(torch::Tensor W, int8_t type, int64_t m,
110
+torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
111
int64_t n);
112
113
-torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X, int8_t type,
114
- int64_t row);
113
+torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
114
+ int64_t type, int64_t row);
115
116
-torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int8_t type,
116
+torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
117
int64_t row);
118
119
torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
csrc/quantization/aqlm/gemm_kernels.cu
+13
-12
@@ -496,14 +496,14 @@ torch::Tensor code2x8_matmat(const torch::Tensor& input,
496
}
497
498
// Accumulate the partition sizes.
499
-int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes) {
499
+int4 accumulate_sizes(const std::vector<int64_t>& codebook_partition_sizes) {
500
int4 cumulative_sizes;
501
auto cumulative_size = &cumulative_sizes.x;
502
- int i = 0;
502
+ size_t i = 0;
503
int last = 0;
504
- assert(codebook_partition_sizes.size(0) <= 4);
505
- for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size) {
506
- *cumulative_size = codebook_partition_sizes[i].item<int>() + last;
504
+ assert(codebook_partition_sizes.size() <= 4);
505
+ for (; i < codebook_partition_sizes.size(); ++i, ++cumulative_size) {
506
+ *cumulative_size = codebook_partition_sizes[i] + last;
507
last = *cumulative_size;
508
}
509
// fill in the rest with unreachable.
@@ -519,12 +519,12 @@ int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes) {
519
torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
520
const torch::Tensor& codebooks,
521
const torch::Tensor& scales,
522
- const torch::Tensor& codebook_partition_sizes,
522
+ const std::vector<int64_t>& codebook_partition_sizes,
523
const std::optional<torch::Tensor>& bias) {
524
int4 cumulative_sizes =
525
vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
526
527
- int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
527
+ int const nbooks = codebooks.size(0) / codebook_partition_sizes.size();
528
int const entries = codebooks.size(1);
529
530
if (nbooks == 1 && entries == (1 << 16)) {
@@ -541,13 +541,13 @@ torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
541
return {};
542
}
543
544
-torch::Tensor aqlm_dequant(const torch::Tensor& codes,
545
- const torch::Tensor& codebooks,
546
- const torch::Tensor& codebook_partition_sizes) {
544
+torch::Tensor aqlm_dequant(
545
+ const torch::Tensor& codes, const torch::Tensor& codebooks,
546
+ const std::vector<int64_t>& codebook_partition_sizes) {
547
int4 cumulative_sizes =
548
vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
549
550
- int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
550
+ int const nbooks = codebooks.size(0) / codebook_partition_sizes.size();
551
int const entries = codebooks.size(1);
552
553
const at::cuda::OptionalCUDAGuard device_guard(device_of(codes));
@@ -557,7 +557,8 @@ torch::Tensor aqlm_dequant(const torch::Tensor& codes,
557
auto in_features = codes.size(1) * 8;
558
auto out_features = codes.size(0);
559
560
- assert(out_features = codebook_partition_sizes.sum().item<int>());
560
+ assert(out_features == std::accumulate(codebook_partition_sizes.begin(),
561
+ codebook_partition_sizes.end(), 0));
562
563
auto weights = torch::empty({out_features, in_features},
564
torch::TensorOptions()
csrc/quantization/gguf/dequantize.cuh
+1
-1
@@ -487,7 +487,7 @@ static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k,
487
dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
488
}
489
490
-static to_fp16_cuda_t ggml_get_to_fp16_cuda(int type) {
490
+static to_fp16_cuda_t ggml_get_to_fp16_cuda(int64_t type) {
491
switch (type) {
492
case 2:
493
return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
csrc/quantization/gguf/gguf_kernel.cu
+4
-4
@@ -60,7 +60,7 @@ static void quantize_row_q8_1_cuda(const half* x, void* vy, const int kx,
60
}
61
62
torch::Tensor ggml_dequantize(torch::Tensor W, // quant weight
63
- int8_t type, int64_t m, int64_t n) {
63
+ int64_t type, int64_t m, int64_t n) {
64
const at::cuda::OptionalCUDAGuard device_guard(device_of(W));
65
auto options =
66
torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
@@ -73,7 +73,7 @@ torch::Tensor ggml_dequantize(torch::Tensor W, // quant weight
73
74
torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, // quant weight
75
torch::Tensor X, // input
76
- int8_t type, int64_t row) {
76
+ int64_t type, int64_t row) {
77
int col = X.sizes()[1];
78
const int padded = (col + 512 - 1) / 512 * 512;
79
const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
@@ -172,7 +172,7 @@ torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, // quant weight
172
173
torch::Tensor ggml_mul_mat_a8(torch::Tensor W, // quant weight
174
torch::Tensor X, // input
175
- int8_t type, int64_t row) {
175
+ int64_t type, int64_t row) {
176
int col = X.sizes()[1];
177
int padded = (col + 512 - 1) / 512 * 512;
178
int batch = X.sizes()[0];
@@ -239,4 +239,4 @@ torch::Tensor ggml_mul_mat_a8(torch::Tensor W, // quant weight
239
break;
240
}
241
return Y;
242
-}
\ No newline at end of file
242
+}
vllm/_custom_ops.py
+7
-21
@@ -17,13 +17,7 @@ if not current_platform.is_tpu():
17
logger.warning("Failed to import from vllm._C with %r", e)
18
19
with contextlib.suppress(ImportError):
20
- # ruff: noqa: F401
21
- import vllm._moe_C
22
-
23
-
24
-def is_custom_op_supported(op_name: str) -> bool:
25
- op, overloads = torch._C._jit_get_operation(op_name)
26
- return op is not None
20
+ import vllm._moe_C # noqa: F401
21
22
23
def hint_on_error(fn):
@@ -280,14 +274,14 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
274
# aqlm
275
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
276
codebooks: torch.Tensor, scales: torch.Tensor,
283
- codebook_partition_sizes: torch.Tensor,
277
+ codebook_partition_sizes: List[int],
278
bias: Optional[torch.Tensor]) -> torch.Tensor:
279
return torch.ops._C.aqlm_gemm(input, codes, codebooks, scales,
280
codebook_partition_sizes, bias)
281
282
283
def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
290
- codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
284
+ codebook_partition_sizes: List[int]) -> torch.Tensor:
285
return torch.ops._C.aqlm_dequant(codes, codebooks,
286
codebook_partition_sizes)
287
@@ -434,25 +428,17 @@ def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
428
429
430
# gguf
437
-def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int, n: int):
431
+def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int,
432
+ n: int) -> torch.Tensor:
433
return torch.ops._C.ggml_dequantize(W, quant_type, m, n)
434
435
441
-def ggml_mul_mat_vec(
442
- W: torch.Tensor,
443
- X: torch.Tensor,
444
- quant_type: int,
445
- row: int,
446
-):
447
- return torch.ops._C.ggml_mul_mat_vec(W, X, quant_type, row)
448
-
449
-
436
def ggml_mul_mat_vec_a8(
437
W: torch.Tensor,
438
X: torch.Tensor,
439
quant_type: int,
440
row: int,
455
-):
441
+) -> torch.Tensor:
442
return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
443
444
@@ -461,7 +447,7 @@ def ggml_mul_mat_a8(
447
X: torch.Tensor,
448
quant_type: int,
449
row: int,
464
-):
450
+) -> torch.Tensor:
451
return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
452
453
vllm/model_executor/layers/quantization/aqlm.py
+5
-7
@@ -95,7 +95,7 @@ def generic_dequantize_gemm(
95
codebooks: torch.
96
Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
97
scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
98
- output_partition_sizes: torch.IntTensor,
98
+ output_partition_sizes: List[int],
99
bias: Optional[torch.Tensor],
100
) -> torch.Tensor:
101
output_shape = input.shape[:-1] + (scales.shape[0], )
@@ -133,7 +133,7 @@ def optimized_dequantize_gemm(
133
codebooks: torch.
134
Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
135
scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
136
- output_partition_sizes: torch.IntTensor,
136
+ output_partition_sizes: List[int],
137
bias: Optional[torch.Tensor],
138
) -> torch.Tensor:
139
weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
@@ -288,10 +288,8 @@ class AQLMLinearMethod(LinearMethodBase):
288
codebooks,
289
{
290
# metadata indicates fixed size concatenated along dim 0
291
- "is_metadata":
292
- True,
293
- "output_partition_sizes":
294
- torch.tensor(output_partition_sizes, device='cpu'),
291
+ "is_metadata": True,
292
+ "output_partition_sizes": output_partition_sizes
293
},
294
)
295
@@ -334,7 +332,7 @@ class AQLMLinearMethod(LinearMethodBase):
332
codes = layer.codes
333
scales = layer.scales
334
output_partition_sizes = getattr(codebooks, "output_partition_sizes",
337
- None)
335
+ [])
336
337
nbooks = codes.shape[2]
338
ingroups = codebooks.shape[3]
vllm/model_executor/layers/quantization/gptq.py
+1
@@ -212,6 +212,7 @@ class GPTQLinearMethod(LinearMethodBase):
212
layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
213
else:
214
layer.g_idx.data = torch.empty((0, ),
215
+ dtype=torch.int,
216
device=layer.g_idx.device)
217
layer.exllama_state = ExllamaState.READY
218
ops.gptq_shuffle(layer.qweight, layer.g_idx,