@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 468d761b32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Reduce supported Punica dtypes (#4304)

Woosuk Kwon committed Apr 23, 2024 at 18:54 UTC 468d761b32e3b3c5d64eeaa797e54ab809b7e50c
16 files changed +66 -72
CMakeLists.txt
-12
@@ -212,23 +212,11 @@ define_gpu_extension_target(
212
213 set(VLLM_PUNICA_EXT_SRC
214 "csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu"
215 - "csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu"
216 - "csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu"
217 - "csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu"
215 "csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu"
219 - "csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu"
220 - "csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu"
221 - "csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu"
222 - "csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu"
216 "csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu"
224 - "csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu"
217 "csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
218 "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
227 - "csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu"
228 - "csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu"
219 "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
230 - "csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu"
231 - "csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu"
220 "csrc/punica/punica_ops.cc")
221
222 #
csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu deleted
-4
@@ -1,4 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
csrc/punica/bgmv/generator.py
+20
@@ -18,6 +18,26 @@ for input_dtype in DTYPES:
18 if weight_dtype == "fp32":
19 # FP32 weights are not supported.
20 continue
21 + if output_dtype == "fp32":
22 + # LoRA A matrix.
23 + if input_dtype != weight_dtype:
24 + # NOTE(woosuk): While Punica supports the case where the
25 + # input and weight dtypes are different, we only generate
26 + # the kernels the same dtypes to reduce the binary size.
27 + continue
28 + elif input_dtype == "fp32":
29 + # LoRA B matrix.
30 + if output_dtype != weight_dtype:
31 + # NOTE(woosuk): While Punica supports the case where the
32 + # output and weight dtypes are different, we only generate
33 + # the kernels the same dtypes to reduce the binary size.
34 + continue
35 + elif not (input_dtype == output_dtype == weight_dtype):
36 + # NOTE(woosuk): While Punica supports mixed data types for
37 + # input, output, and weight, we only generate the kernels with
38 + # the same data types to reduce the binary size.
39 + continue
40 +
41 kernel_definition = TEMPLATE.format(
42 input_dtype=DTYPE_MAP[input_dtype],
43 output_dtype=DTYPE_MAP[output_dtype],
csrc/punica/punica_ops.cc
+17
@@ -50,6 +50,23 @@ inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
50 int64_t y_offset, int64_t full_y_size,
51 int64_t batch_size, int64_t num_layers,
52 int64_t layer_idx, float scale) {
53 + // NOTE(woosuk): While Punica supports various combinations of input/output
54 + // data types, we limit the supported data types to reduce the binary size.
55 + constexpr bool is_input_float = std::is_same<in_T, float>::value;
56 + constexpr bool is_output_float = std::is_same<out_T, float>::value;
57 + if (is_input_float) {
58 + if (!std::is_same<out_T, W_T>::value) {
59 + return false;
60 + }
61 + } else if (is_output_float) {
62 + if (!std::is_same<in_T, W_T>::value) {
63 + return false;
64 + }
65 + } else if (!(std::is_same<in_T, W_T>::value &&
66 + std::is_same<out_T, W_T>::value)) {
67 + return false;
68 + }
69 +
70 switch (pack_u32(in_features, out_features)) {
71 #define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
72 case pack_u32(feat_in, feat_out): \
tests/lora/test_layers.py
+29 -12
@@ -413,7 +413,9 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
413
414 def _pretest():
415 linear = ParallelLMHead(vocab_size + lora_config.lora_extra_vocab_size,
416 - 1024, vocab_size)
416 + 1024,
417 + vocab_size,
418 + params_dtype=torch.float16)
419 linear.weight.data = torch.rand_like(linear.weight.data)
420 linear.weight.data[:, vocab_size:] = 0
421 logits_processor = LogitsProcessor(
@@ -445,7 +447,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
447 num_inputs=8 * num_loras, # * 3,
448 input_size=(1, 1024),
449 input_range=(0, 1),
448 - input_type=torch.float32,
450 + input_type=torch.float16,
451 )
452 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
453
@@ -494,7 +496,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
496 num_inputs=8 * num_loras * 3,
497 input_size=(1, 1024),
498 input_range=(0, 1),
497 - input_type=torch.float32,
499 + input_type=torch.float16,
500 )
501 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
502
@@ -533,11 +535,17 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
535
536 def create_random_linear_parallel_layer():
537 if orientation == "row":
536 - linear = RowParallelLinear(4096, 4096, bias=False)
538 + linear = RowParallelLinear(4096,
539 + 4096,
540 + bias=False,
541 + params_dtype=torch.float16)
542 linear.weight.data = torch.rand_like(linear.weight.data)
543 lora_linear = RowParallelLinearWithLoRA(linear)
544 else:
540 - linear = ColumnParallelLinear(4096, 4096, bias=False)
545 + linear = ColumnParallelLinear(4096,
546 + 4096,
547 + bias=False,
548 + params_dtype=torch.float16)
549 linear.weight.data = torch.rand_like(linear.weight.data)
550 lora_linear = ColumnParallelLinearWithLoRA(linear)
551 lora_linear.create_lora_weights(max_loras, lora_config)
@@ -561,7 +569,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
569 num_inputs=32 * num_loras,
570 input_size=(1, 4096),
571 input_range=(0, 1),
564 - input_type=torch.float32,
572 + input_type=torch.float16,
573 )
574 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
575
@@ -600,7 +608,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
608 num_inputs=32 * num_loras,
609 input_size=(1, 4096),
610 input_range=(0, 1),
603 - input_type=torch.float32,
611 + input_type=torch.float16,
612 )
613 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
614
@@ -633,15 +641,24 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
641 def create_column_parallel_packed_layer():
642 if repeats == 2:
643 linear = MergedColumnParallelLinear(4096, [4096] * repeats,
636 - bias=False)
644 + bias=False,
645 + params_dtype=torch.float16)
646 linear.weight.data = torch.rand_like(linear.weight.data)
647 lora_linear = MergedColumnParallelLinearWithLoRA(linear)
648 elif repeats == 3:
640 - linear = QKVParallelLinear(4096, 64, 32, bias=False)
649 + linear = QKVParallelLinear(4096,
650 + 64,
651 + 32,
652 + bias=False,
653 + params_dtype=torch.float16)
654 linear.weight.data = torch.rand_like(linear.weight.data)
655 lora_linear = MergedQKVParallelLinearWithLora(linear)
656 else:
644 - linear = QKVParallelLinear(4096, 64, 32, bias=False)
657 + linear = QKVParallelLinear(4096,
658 + 64,
659 + 32,
660 + bias=False,
661 + params_dtype=torch.float16)
662 linear.weight.data = torch.rand_like(linear.weight.data)
663 lora_linear = QKVParallelLinearWithLora(linear)
664
@@ -676,7 +693,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
693 num_inputs=32 * num_loras,
694 input_size=(1, 4096),
695 input_range=(0, 1),
679 - input_type=torch.float32,
696 + input_type=torch.float16,
697 )
698 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
699
@@ -716,7 +733,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
733 num_inputs=32 * num_loras,
734 input_size=(1, 4096),
735 input_range=(0, 1),
719 - input_type=torch.float32,
736 + input_type=torch.float16,
737 )
738 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
739