@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 37fd47e780
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Kernel] fix types used in aqlm and ggml kernels to support dynamo (#7596)

bnellnm committed Aug 16, 2024 at 17:00 UTC 37fd47e7803fedd9715abceee8bdb57070fc09f4
7 files changed +39 -53
csrc/ops.h
+8 -8
@@ -63,12 +63,12 @@ void advance_step(int64_t num_seqs, int64_t num_queries, int64_t block_size,
63 torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
64 const torch::Tensor& codebooks,
65 const torch::Tensor& scales,
66 - const torch::Tensor& codebook_partition_sizes,
66 + const std::vector<int64_t>& codebook_partition_sizes,
67 const std::optional<torch::Tensor>& bias);
68
69 -torch::Tensor aqlm_dequant(const torch::Tensor& codes,
70 - const torch::Tensor& codebooks,
71 - const torch::Tensor& codebook_partition_sizes);
69 +torch::Tensor aqlm_dequant(
70 + const torch::Tensor& codes, const torch::Tensor& codebooks,
71 + const std::vector<int64_t>& codebook_partition_sizes);
72
73 torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
74 torch::Tensor _scaling_factors, torch::Tensor _zeros,
@@ -107,13 +107,13 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
107 torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
108 int64_t size_n, int64_t num_bits);
109
110 -torch::Tensor ggml_dequantize(torch::Tensor W, int8_t type, int64_t m,
110 +torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
111 int64_t n);
112
113 -torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X, int8_t type,
114 - int64_t row);
113 +torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
114 + int64_t type, int64_t row);
115
116 -torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int8_t type,
116 +torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
117 int64_t row);
118
119 torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
csrc/quantization/aqlm/gemm_kernels.cu
+13 -12
@@ -496,14 +496,14 @@ torch::Tensor code2x8_matmat(const torch::Tensor& input,
496 }
497
498 // Accumulate the partition sizes.
499 -int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes) {
499 +int4 accumulate_sizes(const std::vector<int64_t>& codebook_partition_sizes) {
500 int4 cumulative_sizes;
501 auto cumulative_size = &cumulative_sizes.x;
502 - int i = 0;
502 + size_t i = 0;
503 int last = 0;
504 - assert(codebook_partition_sizes.size(0) <= 4);
505 - for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size) {
506 - *cumulative_size = codebook_partition_sizes[i].item<int>() + last;
504 + assert(codebook_partition_sizes.size() <= 4);
505 + for (; i < codebook_partition_sizes.size(); ++i, ++cumulative_size) {
506 + *cumulative_size = codebook_partition_sizes[i] + last;
507 last = *cumulative_size;
508 }
509 // fill in the rest with unreachable.
@@ -519,12 +519,12 @@ int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes) {
519 torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
520 const torch::Tensor& codebooks,
521 const torch::Tensor& scales,
522 - const torch::Tensor& codebook_partition_sizes,
522 + const std::vector<int64_t>& codebook_partition_sizes,
523 const std::optional<torch::Tensor>& bias) {
524 int4 cumulative_sizes =
525 vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
526
527 - int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
527 + int const nbooks = codebooks.size(0) / codebook_partition_sizes.size();
528 int const entries = codebooks.size(1);
529
530 if (nbooks == 1 && entries == (1 << 16)) {
@@ -541,13 +541,13 @@ torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
541 return {};
542 }
543
544 -torch::Tensor aqlm_dequant(const torch::Tensor& codes,
545 - const torch::Tensor& codebooks,
546 - const torch::Tensor& codebook_partition_sizes) {
544 +torch::Tensor aqlm_dequant(
545 + const torch::Tensor& codes, const torch::Tensor& codebooks,
546 + const std::vector<int64_t>& codebook_partition_sizes) {
547 int4 cumulative_sizes =
548 vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
549
550 - int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
550 + int const nbooks = codebooks.size(0) / codebook_partition_sizes.size();
551 int const entries = codebooks.size(1);
552
553 const at::cuda::OptionalCUDAGuard device_guard(device_of(codes));
@@ -557,7 +557,8 @@ torch::Tensor aqlm_dequant(const torch::Tensor& codes,
557 auto in_features = codes.size(1) * 8;
558 auto out_features = codes.size(0);
559
560 - assert(out_features = codebook_partition_sizes.sum().item<int>());
560 + assert(out_features == std::accumulate(codebook_partition_sizes.begin(),
561 + codebook_partition_sizes.end(), 0));
562
563 auto weights = torch::empty({out_features, in_features},
564 torch::TensorOptions()
csrc/quantization/gguf/dequantize.cuh
+1 -1
@@ -487,7 +487,7 @@ static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k,
487 dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
488 }
489
490 -static to_fp16_cuda_t ggml_get_to_fp16_cuda(int type) {
490 +static to_fp16_cuda_t ggml_get_to_fp16_cuda(int64_t type) {
491 switch (type) {
492 case 2:
493 return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
csrc/quantization/gguf/gguf_kernel.cu
+4 -4
@@ -60,7 +60,7 @@ static void quantize_row_q8_1_cuda(const half* x, void* vy, const int kx,
60 }
61
62 torch::Tensor ggml_dequantize(torch::Tensor W, // quant weight
63 - int8_t type, int64_t m, int64_t n) {
63 + int64_t type, int64_t m, int64_t n) {
64 const at::cuda::OptionalCUDAGuard device_guard(device_of(W));
65 auto options =
66 torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
@@ -73,7 +73,7 @@ torch::Tensor ggml_dequantize(torch::Tensor W, // quant weight
73
74 torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, // quant weight
75 torch::Tensor X, // input
76 - int8_t type, int64_t row) {
76 + int64_t type, int64_t row) {
77 int col = X.sizes()[1];
78 const int padded = (col + 512 - 1) / 512 * 512;
79 const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
@@ -172,7 +172,7 @@ torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, // quant weight
172
173 torch::Tensor ggml_mul_mat_a8(torch::Tensor W, // quant weight
174 torch::Tensor X, // input
175 - int8_t type, int64_t row) {
175 + int64_t type, int64_t row) {
176 int col = X.sizes()[1];
177 int padded = (col + 512 - 1) / 512 * 512;
178 int batch = X.sizes()[0];
@@ -239,4 +239,4 @@ torch::Tensor ggml_mul_mat_a8(torch::Tensor W, // quant weight
239 break;
240 }
241 return Y;
242 -}
\ No newline at end of file
242 +}
vllm/_custom_ops.py
+7 -21
@@ -17,13 +17,7 @@ if not current_platform.is_tpu():
17 logger.warning("Failed to import from vllm._C with %r", e)
18
19 with contextlib.suppress(ImportError):
20 - # ruff: noqa: F401
21 - import vllm._moe_C
22 -
23 -
24 -def is_custom_op_supported(op_name: str) -> bool:
25 - op, overloads = torch._C._jit_get_operation(op_name)
26 - return op is not None
20 + import vllm._moe_C # noqa: F401
21
22
23 def hint_on_error(fn):
@@ -280,14 +274,14 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
274 # aqlm
275 def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
276 codebooks: torch.Tensor, scales: torch.Tensor,
283 - codebook_partition_sizes: torch.Tensor,
277 + codebook_partition_sizes: List[int],
278 bias: Optional[torch.Tensor]) -> torch.Tensor:
279 return torch.ops._C.aqlm_gemm(input, codes, codebooks, scales,
280 codebook_partition_sizes, bias)
281
282
283 def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
290 - codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
284 + codebook_partition_sizes: List[int]) -> torch.Tensor:
285 return torch.ops._C.aqlm_dequant(codes, codebooks,
286 codebook_partition_sizes)
287
@@ -434,25 +428,17 @@ def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
428
429
430 # gguf
437 -def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int, n: int):
431 +def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int,
432 + n: int) -> torch.Tensor:
433 return torch.ops._C.ggml_dequantize(W, quant_type, m, n)
434
435
441 -def ggml_mul_mat_vec(
442 - W: torch.Tensor,
443 - X: torch.Tensor,
444 - quant_type: int,
445 - row: int,
446 -):
447 - return torch.ops._C.ggml_mul_mat_vec(W, X, quant_type, row)
448 -
449 -
436 def ggml_mul_mat_vec_a8(
437 W: torch.Tensor,
438 X: torch.Tensor,
439 quant_type: int,
440 row: int,
455 -):
441 +) -> torch.Tensor:
442 return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
443
444
@@ -461,7 +447,7 @@ def ggml_mul_mat_a8(
447 X: torch.Tensor,
448 quant_type: int,
449 row: int,
464 -):
450 +) -> torch.Tensor:
451 return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
452
453
vllm/model_executor/layers/quantization/aqlm.py
+5 -7
@@ -95,7 +95,7 @@ def generic_dequantize_gemm(
95 codebooks: torch.
96 Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
97 scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
98 - output_partition_sizes: torch.IntTensor,
98 + output_partition_sizes: List[int],
99 bias: Optional[torch.Tensor],
100 ) -> torch.Tensor:
101 output_shape = input.shape[:-1] + (scales.shape[0], )
@@ -133,7 +133,7 @@ def optimized_dequantize_gemm(
133 codebooks: torch.
134 Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
135 scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
136 - output_partition_sizes: torch.IntTensor,
136 + output_partition_sizes: List[int],
137 bias: Optional[torch.Tensor],
138 ) -> torch.Tensor:
139 weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
@@ -288,10 +288,8 @@ class AQLMLinearMethod(LinearMethodBase):
288 codebooks,
289 {
290 # metadata indicates fixed size concatenated along dim 0
291 - "is_metadata":
292 - True,
293 - "output_partition_sizes":
294 - torch.tensor(output_partition_sizes, device='cpu'),
291 + "is_metadata": True,
292 + "output_partition_sizes": output_partition_sizes
293 },
294 )
295
@@ -334,7 +332,7 @@ class AQLMLinearMethod(LinearMethodBase):
332 codes = layer.codes
333 scales = layer.scales
334 output_partition_sizes = getattr(codebooks, "output_partition_sizes",
337 - None)
335 + [])
336
337 nbooks = codes.shape[2]
338 ingroups = codebooks.shape[3]
vllm/model_executor/layers/quantization/gptq.py
+1
@@ -212,6 +212,7 @@ class GPTQLinearMethod(LinearMethodBase):
212 layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
213 else:
214 layer.g_idx.data = torch.empty((0, ),
215 + dtype=torch.int,
216 device=layer.g_idx.device)
217 layer.exllama_state = ExllamaState.READY
218 ops.gptq_shuffle(layer.qweight, layer.g_idx,