@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 566b57c5c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Kernel] support non-zero cuda devices in punica kernels (#3636)

Jee Li committed Mar 27, 2024 at 08:37 UTC 566b57c5c48a7ba8bbed3e45a07fae9e0bb58cac
2 files changed +29 -62
csrc/punica/punica_ops.cc
+3 -1
@@ -1,7 +1,7 @@
1 #include <cuda_bf16.h>
2 #include <cuda_fp16.h>
3 #include <torch/extension.h>
4 -
4 +#include <c10/cuda/CUDAGuard.h>
5 #include <cstdint>
6
7 #include "bgmv/bgmv_config.h"
@@ -91,6 +91,7 @@ void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
91 CHECK_EQ(w.size(2), h_out);
92 CHECK_EQ(indicies.size(0), x.size(0));
93 CHECK_EQ(y.size(0), x.size(0));
94 + const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
95 bool ok = false;
96 if (h_in < 65536 && h_out < 65536) {
97 // TODO: See if we can get rid of this massive nested switch
@@ -322,6 +323,7 @@ void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
323 CHECK_EQ(w.size(2), h_out);
324 CHECK_EQ(indicies.size(0), x.size(0));
325 CHECK_EQ(y.size(0), x.size(0));
326 + const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
327 bool ok = false;
328 if (h_in < 65536 && h_out < 65536) {
329 // TODO: See if we can get rid of this massive nested switch
tests/lora/test_punica.py
+26 -61
@@ -49,14 +49,18 @@ H1 = H2 = [
49 32768, 33024
50 ]
51 SEED = [0xabcdabcd987]
52 +CUDA_DEVICES = [
53 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
54 +]
55
56
57 @pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
58 @pytest.mark.parametrize("h1", H1)
59 @pytest.mark.parametrize("h2", H2)
60 @pytest.mark.parametrize("seed", SEED)
61 +@pytest.mark.parametrize("device", CUDA_DEVICES)
62 @torch.inference_mode()
59 -def test_lora_correctness(dtype_str, h1, h2, seed):
63 +def test_lora_correctness(dtype_str, h1, h2, seed, device):
64 torch.manual_seed(seed)
65 num_loras = 4
66 num_layers = 1
@@ -64,25 +68,15 @@ def test_lora_correctness(dtype_str, h1, h2, seed):
68 bs = 32
69 scale = 0.123
70 dtype = getattr(torch, dtype_str)
67 - device = torch.device("cuda")
68 -
69 - wa_T_all = torch.randn(num_loras,
70 - num_layers,
71 - r,
72 - h1,
73 - dtype=dtype,
74 - device=device)
75 - wb_T_all = torch.randn(num_loras,
76 - num_layers,
77 - h2,
78 - r,
79 - dtype=dtype,
80 - device=device)
81 - indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
71 + torch.set_default_device(device)
72 +
73 + wa_T_all = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
74 + wb_T_all = torch.randn(num_loras, num_layers, h2, r, dtype=dtype)
75 + indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
76
77 for layer_idx in range(num_layers):
84 - x = torch.randn(bs, h1, dtype=dtype, device=device)
85 - y = torch.randn(bs, h2, dtype=dtype, device=device)
78 + x = torch.randn(bs, h1, dtype=dtype)
79 + y = torch.randn(bs, h2, dtype=dtype)
80
81 y_ref = y.clone()
82 _lora_ref_impl(y_ref, x, wa_T_all, wb_T_all, indices, layer_idx, scale)
@@ -98,8 +92,9 @@ def test_lora_correctness(dtype_str, h1, h2, seed):
92 @pytest.mark.parametrize("h1", H1)
93 @pytest.mark.parametrize("h2", H2)
94 @pytest.mark.parametrize("seed", SEED)
95 +@pytest.mark.parametrize("device", CUDA_DEVICES)
96 @torch.inference_mode()
102 -def test_lora_correctness_slice(dtype_str, h1, h2, seed):
97 +def test_lora_correctness_slice(dtype_str, h1, h2, seed, device):
98 if h2 % 3 != 0 or h2 // 3 not in H1:
99 pytest.skip("h2 must be divisible by 3 and in supported shapes")
100 torch.manual_seed(seed)
@@ -109,50 +104,20 @@ def test_lora_correctness_slice(dtype_str, h1, h2, seed):
104 bs = 32
105 scale = 0.123
106 dtype = getattr(torch, dtype_str)
112 - device = torch.device("cuda")
113 -
114 - wa_T_all_0 = torch.randn(num_loras,
115 - num_layers,
116 - r,
117 - h1,
118 - dtype=dtype,
119 - device=device)
120 - wa_T_all_1 = torch.randn(num_loras,
121 - num_layers,
122 - r,
123 - h1,
124 - dtype=dtype,
125 - device=device)
126 - wa_T_all_2 = torch.randn(num_loras,
127 - num_layers,
128 - r,
129 - h1,
130 - dtype=dtype,
131 - device=device)
132 - wb_T_all_0 = torch.randn(num_loras,
133 - num_layers,
134 - h2 // 3,
135 - r,
136 - dtype=dtype,
137 - device=device)
138 - wb_T_all_1 = torch.randn(num_loras,
139 - num_layers,
140 - h2 // 3,
141 - r,
142 - dtype=dtype,
143 - device=device)
144 - wb_T_all_2 = torch.randn(num_loras,
145 - num_layers,
146 - h2 // 3,
147 - r,
148 - dtype=dtype,
149 - device=device)
150 -
151 - indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
107 + torch.set_default_device(device)
108 +
109 + wa_T_all_0 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
110 + wa_T_all_1 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
111 + wa_T_all_2 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
112 + wb_T_all_0 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
113 + wb_T_all_1 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
114 + wb_T_all_2 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
115 +
116 + indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
117
118 for layer_idx in range(num_layers):
154 - x = torch.randn(bs, h1, dtype=dtype, device=device)
155 - y = torch.randn(bs, h2, dtype=dtype, device=device)
119 + x = torch.randn(bs, h1, dtype=dtype)
120 + y = torch.randn(bs, h2, dtype=dtype)
121 s = h2 // 3
122
123 y_ref = y.clone()