A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Add extra punica sizes to support bigger vocabs (#4015)
Antoni Baum committed
Apr 11, 2024 at 15:18 UTC
1e96c3341a4e055ae392085fecc7a672295b71c2
5 files changed
+109
-48
csrc/punica/bgmv/bgmv_config.h
+11
-1
@@ -60,7 +60,17 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
60
f(in_T, out_T, W_T, narrow, 33024) \
61
f(in_T, out_T, W_T, narrow, 36864) \
62
f(in_T, out_T, W_T, narrow, 49152) \
63
-// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
63
+ f(in_T, out_T, W_T, narrow, 64000) \
64
+ f(in_T, out_T, W_T, narrow, 64256) \
65
+ f(in_T, out_T, W_T, narrow, 64512) \
66
+ f(in_T, out_T, W_T, narrow, 102400) \
67
+ f(in_T, out_T, W_T, narrow, 102656) \
68
+ f(in_T, out_T, W_T, narrow, 102912) \
69
+ f(in_T, out_T, W_T, narrow, 128000) \
70
+ f(in_T, out_T, W_T, narrow, 128256) \
71
+ f(in_T, out_T, W_T, narrow, 128512) \
72
+// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
73
+// and vllm/tests/lora/test_punica.py
74
75
// Keep this in sync with vllm/config::LoRAConfig
76
#define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
csrc/punica/punica_ops.cc
+7
-7
@@ -20,8 +20,8 @@ inline void check_shape(const torch::Tensor &a, const torch::Tensor &b,
20
}
21
}
22
23
-inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) {
24
- return (uint32_t(a) << 16) | uint32_t(b);
23
+inline constexpr uint64_t pack_u32(uint32_t a, uint32_t b) {
24
+ return (uint64_t(a) << 32) | uint64_t(b);
25
}
26
27
#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
@@ -46,13 +46,13 @@ inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) {
46
template <typename in_T, typename out_T, typename W_T>
47
inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
48
const int64_t *lora_indices,
49
- uint16_t in_features, uint16_t out_features,
49
+ uint32_t in_features, uint32_t out_features,
50
int64_t y_offset, int64_t full_y_size,
51
int64_t batch_size, int64_t num_layers,
52
int64_t layer_idx, float scale) {
53
- switch (pack_u16(in_features, out_features)) {
53
+ switch (pack_u32(in_features, out_features)) {
54
#define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
55
- case pack_u16(feat_in, feat_out): \
55
+ case pack_u32(feat_in, feat_out): \
56
bgmv_kernel<feat_in, feat_out>(Y, X, W, lora_indices, y_offset, \
57
full_y_size, batch_size, num_layers, \
58
layer_idx, scale); \
@@ -93,7 +93,7 @@ void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
93
CHECK_EQ(y.size(0), x.size(0));
94
const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
95
bool ok = false;
96
- if (h_in < 65536 && h_out < 65536) {
96
+ if (h_in <= 128512 && h_out <= 128512) {
97
// TODO: See if we can get rid of this massive nested switch
98
switch (x.scalar_type()) {
99
case at::ScalarType::Half:
@@ -325,7 +325,7 @@ void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
325
CHECK_EQ(y.size(0), x.size(0));
326
const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
327
bool ok = false;
328
- if (h_in < 65536 && h_out < 65536) {
328
+ if (h_in <= 128512 && h_out <= 128512) {
329
// TODO: See if we can get rid of this massive nested switch
330
switch (x.scalar_type()) {
331
case at::ScalarType::Half:
tests/lora/test_layers.py
+44
-34
@@ -170,7 +170,8 @@ def create_random_inputs(
170
@torch.inference_mode()
171
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
172
@pytest.mark.parametrize("device", CUDA_DEVICES)
173
-def test_embeddings(dist_init, num_loras, device) -> None:
173
+@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
174
+def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
175
176
torch.set_default_device(device)
177
max_loras = 8
@@ -179,9 +180,9 @@ def test_embeddings(dist_init, num_loras, device) -> None:
180
lora_dtype=torch.float16)
181
182
def create_random_embedding_layer():
182
- embedding = VocabParallelEmbedding(512, 256)
183
+ embedding = VocabParallelEmbedding(vocab_size, 256)
184
embedding.weight.data = torch.rand_like(embedding.weight.data)
184
- embedding.weight.data[512:, :] = 0
185
+ embedding.weight.data[vocab_size:, :] = 0
186
lora_embedding = VocabParallelEmbeddingWithLoRA(embedding)
187
lora_embedding.create_lora_weights(max_loras, lora_config)
188
@@ -203,12 +204,13 @@ def test_embeddings(dist_init, num_loras, device) -> None:
204
active_lora_ids=list(lora_dict.keys()),
205
num_inputs=num_loras * 3,
206
input_size=(200, ),
206
- input_range=(1, 512),
207
+ input_range=(1, vocab_size),
208
)
209
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
210
211
mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
211
- 512, lora_config.lora_extra_vocab_size)
212
+ vocab_size,
213
+ lora_config.lora_extra_vocab_size)
214
lora_embedding.set_mapping(*mapping_info)
215
216
lora_result = lora_embedding(torch.cat(inputs))
@@ -240,12 +242,13 @@ def test_embeddings(dist_init, num_loras, device) -> None:
242
active_lora_ids=[0],
243
num_inputs=num_loras * 3,
244
input_size=(200, ),
243
- input_range=(1, 512),
245
+ input_range=(1, vocab_size),
246
)
247
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
248
249
mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
248
- 512, lora_config.lora_extra_vocab_size)
250
+ vocab_size,
251
+ lora_config.lora_extra_vocab_size)
252
lora_embedding.set_mapping(*mapping_info, )
253
254
lora_result = lora_embedding(torch.cat(inputs))
@@ -263,7 +266,9 @@ def test_embeddings(dist_init, num_loras, device) -> None:
266
# reason="Fails when loras are in any slot other than the first.")
267
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
268
@pytest.mark.parametrize("device", CUDA_DEVICES)
266
-def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
269
+@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
270
+def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
271
+ vocab_size) -> None:
272
273
torch.set_default_device(device)
274
max_loras = 8
@@ -272,15 +277,15 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
277
lora_dtype=torch.float16)
278
279
def create_random_embedding_layer():
275
- embedding = VocabParallelEmbedding(512, 256)
280
+ embedding = VocabParallelEmbedding(vocab_size, 256)
281
embedding_data = torch.rand_like(embedding.weight.data)
282
embedding.weight.data = embedding_data
278
- embedding.weight.data[512:, :] = 0
283
+ embedding.weight.data[vocab_size:, :] = 0
284
expanded_embedding = VocabParallelEmbedding(
280
- 512 + lora_config.lora_extra_vocab_size * max_loras,
285
+ vocab_size + lora_config.lora_extra_vocab_size * max_loras,
286
256,
282
- org_num_embeddings=512)
283
- expanded_embedding.weight.data[:512, :] = embedding_data
287
+ org_num_embeddings=vocab_size)
288
+ expanded_embedding.weight.data[:vocab_size, :] = embedding_data
289
# We need to deepcopy the embedding as it will be modified
290
# in place
291
lora_embedding = VocabParallelEmbeddingWithLoRA(
@@ -298,7 +303,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
303
id_to_index,
304
layer=lora_embedding,
305
layer_weights=torch.zeros(
301
- (256, 512 + lora_config.lora_extra_vocab_size)),
306
+ (256, vocab_size + lora_config.lora_extra_vocab_size)),
307
generate_embeddings_tensor=256,
308
)
309
@@ -316,7 +321,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
321
active_lora_ids=list(lora_dict.keys()),
322
num_inputs=num_loras * 3,
323
input_size=(200, ),
319
- input_range=(1, 512),
324
+ input_range=(1, vocab_size),
325
)
326
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
327
@@ -327,16 +332,18 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
332
for input_, original_input_, lora_id in zip(inputs, original_inputs,
333
prompt_mapping):
334
embedding_id = lora_id - 1
330
- input_[-1] = 512 + (embedding_id * embeddings_tensor_len)
331
- original_input_[-1] = 512
332
- input_[-2] = 512 + ((embedding_id + 1) * embeddings_tensor_len - 1)
333
- original_input_[-2] = 512 + embeddings_tensor_len - 1
335
+ input_[-1] = vocab_size + (embedding_id * embeddings_tensor_len)
336
+ original_input_[-1] = vocab_size
337
+ input_[-2] = vocab_size + (
338
+ (embedding_id + 1) * embeddings_tensor_len - 1)
339
+ original_input_[-2] = vocab_size + embeddings_tensor_len - 1
340
341
mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
336
- 512, lora_config.lora_extra_vocab_size)
342
+ vocab_size,
343
+ lora_config.lora_extra_vocab_size)
344
lora_embedding.set_mapping(*mapping_info, )
345
339
- expanded_embedding.weight[512:512 +
346
+ expanded_embedding.weight[vocab_size:vocab_size +
347
(embeddings_tensor_len *
348
max_loras)] = torch.cat(embeddings_tensors)
349
@@ -370,14 +377,15 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
377
active_lora_ids=[0],
378
num_inputs=num_loras * 3,
379
input_size=(200, ),
373
- input_range=(1, 512),
380
+ input_range=(1, vocab_size),
381
)
382
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
383
384
original_inputs = deepcopy(inputs)
385
386
mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
380
- 512, lora_config.lora_extra_vocab_size)
387
+ vocab_size,
388
+ lora_config.lora_extra_vocab_size)
389
lora_embedding.set_mapping(*mapping_info, )
390
391
lora_result = lora_embedding(torch.cat(original_inputs))
@@ -393,7 +401,9 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
401
@torch.inference_mode()
402
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
403
@pytest.mark.parametrize("device", CUDA_DEVICES)
396
-def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
404
+@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
405
+def test_lm_head_logits_processor(dist_init, num_loras, device,
406
+ vocab_size) -> None:
407
408
torch.set_default_device(device)
409
max_loras = 8
@@ -402,12 +412,12 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
412
lora_dtype=torch.float16)
413
414
def _pretest():
405
- linear = ParallelLMHead(32000 + lora_config.lora_extra_vocab_size,
406
- 1024, 32000)
415
+ linear = ParallelLMHead(vocab_size + lora_config.lora_extra_vocab_size,
416
+ 1024, vocab_size)
417
linear.weight.data = torch.rand_like(linear.weight.data)
408
- linear.weight.data[:, 32000:] = 0
418
+ linear.weight.data[:, vocab_size:] = 0
419
logits_processor = LogitsProcessor(
410
- 32000 + lora_config.lora_extra_vocab_size, 32000)
420
+ vocab_size + lora_config.lora_extra_vocab_size, vocab_size)
421
lora_logits_processor = LogitsProcessorWithLoRA(
422
logits_processor, 1024, linear.weight.dtype, linear.weight.device)
423
lora_logits_processor.create_lora_weights(max_loras, lora_config)
@@ -444,7 +454,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
454
lora_mapping,
455
id_to_index,
456
max_loras,
447
- 32000,
457
+ vocab_size,
458
lora_config.lora_extra_vocab_size,
459
)
460
lora_logits_processor.set_mapping(*mapping_info, )
@@ -460,7 +470,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
470
org_vocab_size:logits_processor.org_vocab_size +
471
embeddings_tensor_len] = embeddings_tensor
472
463
- logits_processor.org_vocab_size = (32000 +
473
+ logits_processor.org_vocab_size = (vocab_size +
474
lora_config.lora_extra_vocab_size)
475
expected_results = []
476
for input_, lora_id in zip(inputs, prompt_mapping):
@@ -468,11 +478,11 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
478
result = logits_processor._get_logits(hidden_states=input_,
479
embedding=linear.weight,
480
embedding_bias=None)
471
- result[:, 32000 + embeddings_tensor_len:] = float("-inf")
481
+ result[:, vocab_size + embeddings_tensor_len:] = float("-inf")
482
result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
483
expected_results.append(result)
484
expected_result = torch.cat(expected_results)
475
- logits_processor.org_vocab_size = 32000
485
+ logits_processor.org_vocab_size = vocab_size
486
487
# Check that resetting the lora weights succeeds
488
@@ -489,14 +499,14 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
499
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
500
501
mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
492
- 32000,
502
+ vocab_size,
503
lora_config.lora_extra_vocab_size)
504
lora_logits_processor.set_mapping(*mapping_info, )
505
506
lora_result = lora_logits_processor._get_logits(
507
hidden_states=torch.cat(inputs),
508
embedding=original_weight,
499
- embedding_bias=None)[:, :32000]
509
+ embedding_bias=None)[:, :vocab_size]
510
expected_result = logits_processor._get_logits(
511
hidden_states=torch.cat(inputs),
512
embedding=original_weight,
tests/lora/test_punica.py
+45
-4
@@ -43,10 +43,51 @@ def _lora_ref_impl(
43
44
45
H1 = H2 = [
46
- 128, 256, 512, 1024, 1152, 1280, 1536, 2048, 2304, 2560, 2752, 3072, 3456,
47
- 3584, 4096, 4608, 5120, 5504, 5632, 6144, 6848, 6912, 7168, 8192, 9216,
48
- 10240, 11008, 13824, 14336, 22016, 24576, 27392, 32000, 32256, 32512,
49
- 32768, 33024
46
+ 128,
47
+ 256,
48
+ 512,
49
+ 1024,
50
+ 1152,
51
+ 1280,
52
+ 1536,
53
+ 2048,
54
+ 2304,
55
+ 2560,
56
+ 2752,
57
+ 3072,
58
+ 3456,
59
+ 3584,
60
+ 4096,
61
+ 4608,
62
+ 5120,
63
+ 5504,
64
+ 5632,
65
+ 6144,
66
+ 6848,
67
+ 6912,
68
+ 7168,
69
+ 8192,
70
+ 9216,
71
+ 10240,
72
+ 11008,
73
+ 13824,
74
+ 14336,
75
+ 22016,
76
+ 24576,
77
+ 27392,
78
+ 32000,
79
+ 32256,
80
+ 32512,
81
+ 32768,
82
+ 33024,
83
+ 36864,
84
+ 49152,
85
+ 64000,
86
+ 64256,
87
+ 102400,
88
+ 102656,
89
+ 128000,
90
+ 128256,
91
]
92
SEED = [0xabcdabcd987]
93
CUDA_DEVICES = [
vllm/lora/layers.py
+2
-2
@@ -935,9 +935,9 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
935
model_config: Optional[PretrainedConfig] = None,
936
) -> None:
937
# Keep this in sync with csrc/punica/bgmv/bgmv_config.h
938
- if 32000 < self.base_layer.vocab_size > 33024:
938
+ if 32000 < self.base_layer.vocab_size > 128512:
939
raise ValueError("When using LoRA, vocab size must be "
940
- "32000 >= vocab_size <= 33024")
940
+ "32000 >= vocab_size <= 128512")
941
self.lora_a_stacked = torch.zeros(
942
(
943
max_loras,