@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e96c3341a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Add extra punica sizes to support bigger vocabs (#4015)

Antoni Baum committed Apr 11, 2024 at 15:18 UTC 1e96c3341a4e055ae392085fecc7a672295b71c2
5 files changed +109 -48
csrc/punica/bgmv/bgmv_config.h
+11 -1
@@ -60,7 +60,17 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
60 f(in_T, out_T, W_T, narrow, 33024) \
61 f(in_T, out_T, W_T, narrow, 36864) \
62 f(in_T, out_T, W_T, narrow, 49152) \
63 -// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
63 + f(in_T, out_T, W_T, narrow, 64000) \
64 + f(in_T, out_T, W_T, narrow, 64256) \
65 + f(in_T, out_T, W_T, narrow, 64512) \
66 + f(in_T, out_T, W_T, narrow, 102400) \
67 + f(in_T, out_T, W_T, narrow, 102656) \
68 + f(in_T, out_T, W_T, narrow, 102912) \
69 + f(in_T, out_T, W_T, narrow, 128000) \
70 + f(in_T, out_T, W_T, narrow, 128256) \
71 + f(in_T, out_T, W_T, narrow, 128512) \
72 +// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
73 +// and vllm/tests/lora/test_punica.py
74
75 // Keep this in sync with vllm/config::LoRAConfig
76 #define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
csrc/punica/punica_ops.cc
+7 -7
@@ -20,8 +20,8 @@ inline void check_shape(const torch::Tensor &a, const torch::Tensor &b,
20 }
21 }
22
23 -inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) {
24 - return (uint32_t(a) << 16) | uint32_t(b);
23 +inline constexpr uint64_t pack_u32(uint32_t a, uint32_t b) {
24 + return (uint64_t(a) << 32) | uint64_t(b);
25 }
26
27 #define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
@@ -46,13 +46,13 @@ inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) {
46 template <typename in_T, typename out_T, typename W_T>
47 inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
48 const int64_t *lora_indices,
49 - uint16_t in_features, uint16_t out_features,
49 + uint32_t in_features, uint32_t out_features,
50 int64_t y_offset, int64_t full_y_size,
51 int64_t batch_size, int64_t num_layers,
52 int64_t layer_idx, float scale) {
53 - switch (pack_u16(in_features, out_features)) {
53 + switch (pack_u32(in_features, out_features)) {
54 #define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
55 - case pack_u16(feat_in, feat_out): \
55 + case pack_u32(feat_in, feat_out): \
56 bgmv_kernel<feat_in, feat_out>(Y, X, W, lora_indices, y_offset, \
57 full_y_size, batch_size, num_layers, \
58 layer_idx, scale); \
@@ -93,7 +93,7 @@ void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
93 CHECK_EQ(y.size(0), x.size(0));
94 const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
95 bool ok = false;
96 - if (h_in < 65536 && h_out < 65536) {
96 + if (h_in <= 128512 && h_out <= 128512) {
97 // TODO: See if we can get rid of this massive nested switch
98 switch (x.scalar_type()) {
99 case at::ScalarType::Half:
@@ -325,7 +325,7 @@ void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
325 CHECK_EQ(y.size(0), x.size(0));
326 const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
327 bool ok = false;
328 - if (h_in < 65536 && h_out < 65536) {
328 + if (h_in <= 128512 && h_out <= 128512) {
329 // TODO: See if we can get rid of this massive nested switch
330 switch (x.scalar_type()) {
331 case at::ScalarType::Half:
tests/lora/test_layers.py
+44 -34
@@ -170,7 +170,8 @@ def create_random_inputs(
170 @torch.inference_mode()
171 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
172 @pytest.mark.parametrize("device", CUDA_DEVICES)
173 -def test_embeddings(dist_init, num_loras, device) -> None:
173 +@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
174 +def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
175
176 torch.set_default_device(device)
177 max_loras = 8
@@ -179,9 +180,9 @@ def test_embeddings(dist_init, num_loras, device) -> None:
180 lora_dtype=torch.float16)
181
182 def create_random_embedding_layer():
182 - embedding = VocabParallelEmbedding(512, 256)
183 + embedding = VocabParallelEmbedding(vocab_size, 256)
184 embedding.weight.data = torch.rand_like(embedding.weight.data)
184 - embedding.weight.data[512:, :] = 0
185 + embedding.weight.data[vocab_size:, :] = 0
186 lora_embedding = VocabParallelEmbeddingWithLoRA(embedding)
187 lora_embedding.create_lora_weights(max_loras, lora_config)
188
@@ -203,12 +204,13 @@ def test_embeddings(dist_init, num_loras, device) -> None:
204 active_lora_ids=list(lora_dict.keys()),
205 num_inputs=num_loras * 3,
206 input_size=(200, ),
206 - input_range=(1, 512),
207 + input_range=(1, vocab_size),
208 )
209 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
210
211 mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
211 - 512, lora_config.lora_extra_vocab_size)
212 + vocab_size,
213 + lora_config.lora_extra_vocab_size)
214 lora_embedding.set_mapping(*mapping_info)
215
216 lora_result = lora_embedding(torch.cat(inputs))
@@ -240,12 +242,13 @@ def test_embeddings(dist_init, num_loras, device) -> None:
242 active_lora_ids=[0],
243 num_inputs=num_loras * 3,
244 input_size=(200, ),
243 - input_range=(1, 512),
245 + input_range=(1, vocab_size),
246 )
247 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
248
249 mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
248 - 512, lora_config.lora_extra_vocab_size)
250 + vocab_size,
251 + lora_config.lora_extra_vocab_size)
252 lora_embedding.set_mapping(*mapping_info, )
253
254 lora_result = lora_embedding(torch.cat(inputs))
@@ -263,7 +266,9 @@ def test_embeddings(dist_init, num_loras, device) -> None:
266 # reason="Fails when loras are in any slot other than the first.")
267 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
268 @pytest.mark.parametrize("device", CUDA_DEVICES)
266 -def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
269 +@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
270 +def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
271 + vocab_size) -> None:
272
273 torch.set_default_device(device)
274 max_loras = 8
@@ -272,15 +277,15 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
277 lora_dtype=torch.float16)
278
279 def create_random_embedding_layer():
275 - embedding = VocabParallelEmbedding(512, 256)
280 + embedding = VocabParallelEmbedding(vocab_size, 256)
281 embedding_data = torch.rand_like(embedding.weight.data)
282 embedding.weight.data = embedding_data
278 - embedding.weight.data[512:, :] = 0
283 + embedding.weight.data[vocab_size:, :] = 0
284 expanded_embedding = VocabParallelEmbedding(
280 - 512 + lora_config.lora_extra_vocab_size * max_loras,
285 + vocab_size + lora_config.lora_extra_vocab_size * max_loras,
286 256,
282 - org_num_embeddings=512)
283 - expanded_embedding.weight.data[:512, :] = embedding_data
287 + org_num_embeddings=vocab_size)
288 + expanded_embedding.weight.data[:vocab_size, :] = embedding_data
289 # We need to deepcopy the embedding as it will be modified
290 # in place
291 lora_embedding = VocabParallelEmbeddingWithLoRA(
@@ -298,7 +303,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
303 id_to_index,
304 layer=lora_embedding,
305 layer_weights=torch.zeros(
301 - (256, 512 + lora_config.lora_extra_vocab_size)),
306 + (256, vocab_size + lora_config.lora_extra_vocab_size)),
307 generate_embeddings_tensor=256,
308 )
309
@@ -316,7 +321,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
321 active_lora_ids=list(lora_dict.keys()),
322 num_inputs=num_loras * 3,
323 input_size=(200, ),
319 - input_range=(1, 512),
324 + input_range=(1, vocab_size),
325 )
326 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
327
@@ -327,16 +332,18 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
332 for input_, original_input_, lora_id in zip(inputs, original_inputs,
333 prompt_mapping):
334 embedding_id = lora_id - 1
330 - input_[-1] = 512 + (embedding_id * embeddings_tensor_len)
331 - original_input_[-1] = 512
332 - input_[-2] = 512 + ((embedding_id + 1) * embeddings_tensor_len - 1)
333 - original_input_[-2] = 512 + embeddings_tensor_len - 1
335 + input_[-1] = vocab_size + (embedding_id * embeddings_tensor_len)
336 + original_input_[-1] = vocab_size
337 + input_[-2] = vocab_size + (
338 + (embedding_id + 1) * embeddings_tensor_len - 1)
339 + original_input_[-2] = vocab_size + embeddings_tensor_len - 1
340
341 mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
336 - 512, lora_config.lora_extra_vocab_size)
342 + vocab_size,
343 + lora_config.lora_extra_vocab_size)
344 lora_embedding.set_mapping(*mapping_info, )
345
339 - expanded_embedding.weight[512:512 +
346 + expanded_embedding.weight[vocab_size:vocab_size +
347 (embeddings_tensor_len *
348 max_loras)] = torch.cat(embeddings_tensors)
349
@@ -370,14 +377,15 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
377 active_lora_ids=[0],
378 num_inputs=num_loras * 3,
379 input_size=(200, ),
373 - input_range=(1, 512),
380 + input_range=(1, vocab_size),
381 )
382 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
383
384 original_inputs = deepcopy(inputs)
385
386 mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
380 - 512, lora_config.lora_extra_vocab_size)
387 + vocab_size,
388 + lora_config.lora_extra_vocab_size)
389 lora_embedding.set_mapping(*mapping_info, )
390
391 lora_result = lora_embedding(torch.cat(original_inputs))
@@ -393,7 +401,9 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
401 @torch.inference_mode()
402 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
403 @pytest.mark.parametrize("device", CUDA_DEVICES)
396 -def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
404 +@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
405 +def test_lm_head_logits_processor(dist_init, num_loras, device,
406 + vocab_size) -> None:
407
408 torch.set_default_device(device)
409 max_loras = 8
@@ -402,12 +412,12 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
412 lora_dtype=torch.float16)
413
414 def _pretest():
405 - linear = ParallelLMHead(32000 + lora_config.lora_extra_vocab_size,
406 - 1024, 32000)
415 + linear = ParallelLMHead(vocab_size + lora_config.lora_extra_vocab_size,
416 + 1024, vocab_size)
417 linear.weight.data = torch.rand_like(linear.weight.data)
408 - linear.weight.data[:, 32000:] = 0
418 + linear.weight.data[:, vocab_size:] = 0
419 logits_processor = LogitsProcessor(
410 - 32000 + lora_config.lora_extra_vocab_size, 32000)
420 + vocab_size + lora_config.lora_extra_vocab_size, vocab_size)
421 lora_logits_processor = LogitsProcessorWithLoRA(
422 logits_processor, 1024, linear.weight.dtype, linear.weight.device)
423 lora_logits_processor.create_lora_weights(max_loras, lora_config)
@@ -444,7 +454,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
454 lora_mapping,
455 id_to_index,
456 max_loras,
447 - 32000,
457 + vocab_size,
458 lora_config.lora_extra_vocab_size,
459 )
460 lora_logits_processor.set_mapping(*mapping_info, )
@@ -460,7 +470,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
470 org_vocab_size:logits_processor.org_vocab_size +
471 embeddings_tensor_len] = embeddings_tensor
472
463 - logits_processor.org_vocab_size = (32000 +
473 + logits_processor.org_vocab_size = (vocab_size +
474 lora_config.lora_extra_vocab_size)
475 expected_results = []
476 for input_, lora_id in zip(inputs, prompt_mapping):
@@ -468,11 +478,11 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
478 result = logits_processor._get_logits(hidden_states=input_,
479 embedding=linear.weight,
480 embedding_bias=None)
471 - result[:, 32000 + embeddings_tensor_len:] = float("-inf")
481 + result[:, vocab_size + embeddings_tensor_len:] = float("-inf")
482 result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
483 expected_results.append(result)
484 expected_result = torch.cat(expected_results)
475 - logits_processor.org_vocab_size = 32000
485 + logits_processor.org_vocab_size = vocab_size
486
487 # Check that resetting the lora weights succeeds
488
@@ -489,14 +499,14 @@ def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
499 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
500
501 mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
492 - 32000,
502 + vocab_size,
503 lora_config.lora_extra_vocab_size)
504 lora_logits_processor.set_mapping(*mapping_info, )
505
506 lora_result = lora_logits_processor._get_logits(
507 hidden_states=torch.cat(inputs),
508 embedding=original_weight,
499 - embedding_bias=None)[:, :32000]
509 + embedding_bias=None)[:, :vocab_size]
510 expected_result = logits_processor._get_logits(
511 hidden_states=torch.cat(inputs),
512 embedding=original_weight,
tests/lora/test_punica.py
+45 -4
@@ -43,10 +43,51 @@ def _lora_ref_impl(
43
44
45 H1 = H2 = [
46 - 128, 256, 512, 1024, 1152, 1280, 1536, 2048, 2304, 2560, 2752, 3072, 3456,
47 - 3584, 4096, 4608, 5120, 5504, 5632, 6144, 6848, 6912, 7168, 8192, 9216,
48 - 10240, 11008, 13824, 14336, 22016, 24576, 27392, 32000, 32256, 32512,
49 - 32768, 33024
46 + 128,
47 + 256,
48 + 512,
49 + 1024,
50 + 1152,
51 + 1280,
52 + 1536,
53 + 2048,
54 + 2304,
55 + 2560,
56 + 2752,
57 + 3072,
58 + 3456,
59 + 3584,
60 + 4096,
61 + 4608,
62 + 5120,
63 + 5504,
64 + 5632,
65 + 6144,
66 + 6848,
67 + 6912,
68 + 7168,
69 + 8192,
70 + 9216,
71 + 10240,
72 + 11008,
73 + 13824,
74 + 14336,
75 + 22016,
76 + 24576,
77 + 27392,
78 + 32000,
79 + 32256,
80 + 32512,
81 + 32768,
82 + 33024,
83 + 36864,
84 + 49152,
85 + 64000,
86 + 64256,
87 + 102400,
88 + 102656,
89 + 128000,
90 + 128256,
91 ]
92 SEED = [0xabcdabcd987]
93 CUDA_DEVICES = [
vllm/lora/layers.py
+2 -2
@@ -935,9 +935,9 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
935 model_config: Optional[PretrainedConfig] = None,
936 ) -> None:
937 # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
938 - if 32000 < self.base_layer.vocab_size > 33024:
938 + if 32000 < self.base_layer.vocab_size > 128512:
939 raise ValueError("When using LoRA, vocab size must be "
940 - "32000 >= vocab_size <= 33024")
940 + "32000 >= vocab_size <= 128512")
941 self.lora_a_stacked = torch.zeros(
942 (
943 max_loras,