A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Kernel] support non-zero cuda devices in punica kernels (#3636)
Jee Li committed
Mar 27, 2024 at 08:37 UTC
566b57c5c48a7ba8bbed3e45a07fae9e0bb58cac
2 files changed
+29
-62
csrc/punica/punica_ops.cc
+3
-1
@@ -1,7 +1,7 @@
1
#include <cuda_bf16.h>
2
#include <cuda_fp16.h>
3
#include <torch/extension.h>
4
-
4
+#include <c10/cuda/CUDAGuard.h>
5
#include <cstdint>
6
7
#include "bgmv/bgmv_config.h"
@@ -91,6 +91,7 @@ void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
91
CHECK_EQ(w.size(2), h_out);
92
CHECK_EQ(indicies.size(0), x.size(0));
93
CHECK_EQ(y.size(0), x.size(0));
94
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
95
bool ok = false;
96
if (h_in < 65536 && h_out < 65536) {
97
// TODO: See if we can get rid of this massive nested switch
@@ -322,6 +323,7 @@ void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
323
CHECK_EQ(w.size(2), h_out);
324
CHECK_EQ(indicies.size(0), x.size(0));
325
CHECK_EQ(y.size(0), x.size(0));
326
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
327
bool ok = false;
328
if (h_in < 65536 && h_out < 65536) {
329
// TODO: See if we can get rid of this massive nested switch
tests/lora/test_punica.py
+26
-61
@@ -49,14 +49,18 @@ H1 = H2 = [
49
32768, 33024
50
]
51
SEED = [0xabcdabcd987]
52
+CUDA_DEVICES = [
53
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
54
+]
55
56
57
@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
58
@pytest.mark.parametrize("h1", H1)
59
@pytest.mark.parametrize("h2", H2)
60
@pytest.mark.parametrize("seed", SEED)
61
+@pytest.mark.parametrize("device", CUDA_DEVICES)
62
@torch.inference_mode()
59
-def test_lora_correctness(dtype_str, h1, h2, seed):
63
+def test_lora_correctness(dtype_str, h1, h2, seed, device):
64
torch.manual_seed(seed)
65
num_loras = 4
66
num_layers = 1
@@ -64,25 +68,15 @@ def test_lora_correctness(dtype_str, h1, h2, seed):
68
bs = 32
69
scale = 0.123
70
dtype = getattr(torch, dtype_str)
67
- device = torch.device("cuda")
68
-
69
- wa_T_all = torch.randn(num_loras,
70
- num_layers,
71
- r,
72
- h1,
73
- dtype=dtype,
74
- device=device)
75
- wb_T_all = torch.randn(num_loras,
76
- num_layers,
77
- h2,
78
- r,
79
- dtype=dtype,
80
- device=device)
81
- indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
71
+ torch.set_default_device(device)
72
+
73
+ wa_T_all = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
74
+ wb_T_all = torch.randn(num_loras, num_layers, h2, r, dtype=dtype)
75
+ indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
76
77
for layer_idx in range(num_layers):
84
- x = torch.randn(bs, h1, dtype=dtype, device=device)
85
- y = torch.randn(bs, h2, dtype=dtype, device=device)
78
+ x = torch.randn(bs, h1, dtype=dtype)
79
+ y = torch.randn(bs, h2, dtype=dtype)
80
81
y_ref = y.clone()
82
_lora_ref_impl(y_ref, x, wa_T_all, wb_T_all, indices, layer_idx, scale)
@@ -98,8 +92,9 @@ def test_lora_correctness(dtype_str, h1, h2, seed):
92
@pytest.mark.parametrize("h1", H1)
93
@pytest.mark.parametrize("h2", H2)
94
@pytest.mark.parametrize("seed", SEED)
95
+@pytest.mark.parametrize("device", CUDA_DEVICES)
96
@torch.inference_mode()
102
-def test_lora_correctness_slice(dtype_str, h1, h2, seed):
97
+def test_lora_correctness_slice(dtype_str, h1, h2, seed, device):
98
if h2 % 3 != 0 or h2 // 3 not in H1:
99
pytest.skip("h2 must be divisible by 3 and in supported shapes")
100
torch.manual_seed(seed)
@@ -109,50 +104,20 @@ def test_lora_correctness_slice(dtype_str, h1, h2, seed):
104
bs = 32
105
scale = 0.123
106
dtype = getattr(torch, dtype_str)
112
- device = torch.device("cuda")
113
-
114
- wa_T_all_0 = torch.randn(num_loras,
115
- num_layers,
116
- r,
117
- h1,
118
- dtype=dtype,
119
- device=device)
120
- wa_T_all_1 = torch.randn(num_loras,
121
- num_layers,
122
- r,
123
- h1,
124
- dtype=dtype,
125
- device=device)
126
- wa_T_all_2 = torch.randn(num_loras,
127
- num_layers,
128
- r,
129
- h1,
130
- dtype=dtype,
131
- device=device)
132
- wb_T_all_0 = torch.randn(num_loras,
133
- num_layers,
134
- h2 // 3,
135
- r,
136
- dtype=dtype,
137
- device=device)
138
- wb_T_all_1 = torch.randn(num_loras,
139
- num_layers,
140
- h2 // 3,
141
- r,
142
- dtype=dtype,
143
- device=device)
144
- wb_T_all_2 = torch.randn(num_loras,
145
- num_layers,
146
- h2 // 3,
147
- r,
148
- dtype=dtype,
149
- device=device)
150
-
151
- indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
107
+ torch.set_default_device(device)
108
+
109
+ wa_T_all_0 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
110
+ wa_T_all_1 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
111
+ wa_T_all_2 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
112
+ wb_T_all_0 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
113
+ wb_T_all_1 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
114
+ wb_T_all_2 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
115
+
116
+ indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
117
118
for layer_idx in range(num_layers):
154
- x = torch.randn(bs, h1, dtype=dtype, device=device)
155
- y = torch.randn(bs, h2, dtype=dtype, device=device)
119
+ x = torch.randn(bs, h1, dtype=dtype)
120
+ y = torch.randn(bs, h2, dtype=dtype)
121
s = h2 // 3
122
123
y_ref = y.clone()