A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Reduce supported Punica dtypes (#4304)
Woosuk Kwon committed
Apr 23, 2024 at 18:54 UTC
468d761b32e3b3c5d64eeaa797e54ab809b7e50c
16 files changed
+66
-72
CMakeLists.txt
-12
@@ -212,23 +212,11 @@ define_gpu_extension_target(
212
213
set(VLLM_PUNICA_EXT_SRC
214
"csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu"
215
- "csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu"
216
- "csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu"
217
- "csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu"
215
"csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu"
219
- "csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu"
220
- "csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu"
221
- "csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu"
222
- "csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu"
216
"csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu"
224
- "csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu"
217
"csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
218
"csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
227
- "csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu"
228
- "csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu"
219
"csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
230
- "csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu"
231
- "csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu"
220
"csrc/punica/punica_ops.cc")
221
222
#
csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu
deleted
-4
@@ -1,4 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
csrc/punica/bgmv/generator.py
+20
@@ -18,6 +18,26 @@ for input_dtype in DTYPES:
18
if weight_dtype == "fp32":
19
# FP32 weights are not supported.
20
continue
21
+ if output_dtype == "fp32":
22
+ # LoRA A matrix.
23
+ if input_dtype != weight_dtype:
24
+ # NOTE(woosuk): While Punica supports the case where the
25
+ # input and weight dtypes are different, we only generate
26
+ # the kernels the same dtypes to reduce the binary size.
27
+ continue
28
+ elif input_dtype == "fp32":
29
+ # LoRA B matrix.
30
+ if output_dtype != weight_dtype:
31
+ # NOTE(woosuk): While Punica supports the case where the
32
+ # output and weight dtypes are different, we only generate
33
+ # the kernels the same dtypes to reduce the binary size.
34
+ continue
35
+ elif not (input_dtype == output_dtype == weight_dtype):
36
+ # NOTE(woosuk): While Punica supports mixed data types for
37
+ # input, output, and weight, we only generate the kernels with
38
+ # the same data types to reduce the binary size.
39
+ continue
40
+
41
kernel_definition = TEMPLATE.format(
42
input_dtype=DTYPE_MAP[input_dtype],
43
output_dtype=DTYPE_MAP[output_dtype],
csrc/punica/punica_ops.cc
+17
@@ -50,6 +50,23 @@ inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
50
int64_t y_offset, int64_t full_y_size,
51
int64_t batch_size, int64_t num_layers,
52
int64_t layer_idx, float scale) {
53
+ // NOTE(woosuk): While Punica supports various combinations of input/output
54
+ // data types, we limit the supported data types to reduce the binary size.
55
+ constexpr bool is_input_float = std::is_same<in_T, float>::value;
56
+ constexpr bool is_output_float = std::is_same<out_T, float>::value;
57
+ if (is_input_float) {
58
+ if (!std::is_same<out_T, W_T>::value) {
59
+ return false;
60
+ }
61
+ } else if (is_output_float) {
62
+ if (!std::is_same<in_T, W_T>::value) {
63
+ return false;
64
+ }
65
+ } else if (!(std::is_same<in_T, W_T>::value &&
66
+ std::is_same<out_T, W_T>::value)) {
67
+ return false;
68
+ }
69
+
70
switch (pack_u32(in_features, out_features)) {
71
#define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
72
case pack_u32(feat_in, feat_out): \
tests/lora/test_layers.py
+29
-12
@@ -413,7 +413,9 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
413
414
def _pretest():
415
linear = ParallelLMHead(vocab_size + lora_config.lora_extra_vocab_size,
416
- 1024, vocab_size)
416
+ 1024,
417
+ vocab_size,
418
+ params_dtype=torch.float16)
419
linear.weight.data = torch.rand_like(linear.weight.data)
420
linear.weight.data[:, vocab_size:] = 0
421
logits_processor = LogitsProcessor(
@@ -445,7 +447,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
447
num_inputs=8 * num_loras, # * 3,
448
input_size=(1, 1024),
449
input_range=(0, 1),
448
- input_type=torch.float32,
450
+ input_type=torch.float16,
451
)
452
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
453
@@ -494,7 +496,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
496
num_inputs=8 * num_loras * 3,
497
input_size=(1, 1024),
498
input_range=(0, 1),
497
- input_type=torch.float32,
499
+ input_type=torch.float16,
500
)
501
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
502
@@ -533,11 +535,17 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
535
536
def create_random_linear_parallel_layer():
537
if orientation == "row":
536
- linear = RowParallelLinear(4096, 4096, bias=False)
538
+ linear = RowParallelLinear(4096,
539
+ 4096,
540
+ bias=False,
541
+ params_dtype=torch.float16)
542
linear.weight.data = torch.rand_like(linear.weight.data)
543
lora_linear = RowParallelLinearWithLoRA(linear)
544
else:
540
- linear = ColumnParallelLinear(4096, 4096, bias=False)
545
+ linear = ColumnParallelLinear(4096,
546
+ 4096,
547
+ bias=False,
548
+ params_dtype=torch.float16)
549
linear.weight.data = torch.rand_like(linear.weight.data)
550
lora_linear = ColumnParallelLinearWithLoRA(linear)
551
lora_linear.create_lora_weights(max_loras, lora_config)
@@ -561,7 +569,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
569
num_inputs=32 * num_loras,
570
input_size=(1, 4096),
571
input_range=(0, 1),
564
- input_type=torch.float32,
572
+ input_type=torch.float16,
573
)
574
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
575
@@ -600,7 +608,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
608
num_inputs=32 * num_loras,
609
input_size=(1, 4096),
610
input_range=(0, 1),
603
- input_type=torch.float32,
611
+ input_type=torch.float16,
612
)
613
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
614
@@ -633,15 +641,24 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
641
def create_column_parallel_packed_layer():
642
if repeats == 2:
643
linear = MergedColumnParallelLinear(4096, [4096] * repeats,
636
- bias=False)
644
+ bias=False,
645
+ params_dtype=torch.float16)
646
linear.weight.data = torch.rand_like(linear.weight.data)
647
lora_linear = MergedColumnParallelLinearWithLoRA(linear)
648
elif repeats == 3:
640
- linear = QKVParallelLinear(4096, 64, 32, bias=False)
649
+ linear = QKVParallelLinear(4096,
650
+ 64,
651
+ 32,
652
+ bias=False,
653
+ params_dtype=torch.float16)
654
linear.weight.data = torch.rand_like(linear.weight.data)
655
lora_linear = MergedQKVParallelLinearWithLora(linear)
656
else:
644
- linear = QKVParallelLinear(4096, 64, 32, bias=False)
657
+ linear = QKVParallelLinear(4096,
658
+ 64,
659
+ 32,
660
+ bias=False,
661
+ params_dtype=torch.float16)
662
linear.weight.data = torch.rand_like(linear.weight.data)
663
lora_linear = QKVParallelLinearWithLora(linear)
664
@@ -676,7 +693,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
693
num_inputs=32 * num_loras,
694
input_size=(1, 4096),
695
input_range=(0, 1),
679
- input_type=torch.float32,
696
+ input_type=torch.float16,
697
)
698
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
699
@@ -716,7 +733,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
733
num_inputs=32 * num_loras,
734
input_size=(1, 4096),
735
input_range=(0, 1),
719
- input_type=torch.float32,
736
+ input_type=torch.float16,
737
)
738
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
739