@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b9926f7f66
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support block size 32 (#35)

Woosuk Kwon committed Apr 9, 2023 at 23:07 UTC b9926f7f663b953c886a42b691cd7ba3e4325239
4 files changed +49 -5
cacheflow/master/block_manager.py
+2 -2
@@ -15,9 +15,9 @@ class BlockAllocator:
15 block_size: int,
16 num_blocks: int,
17 ) -> None:
18 - if block_size not in [8, 16]:
18 + if block_size not in [8, 16, 32]:
19 raise ValueError(f'Unsupported block size: {block_size}'
20 - 'The block size must be either 8 or 16.')
20 + 'The block size must be one of {8, 16, 32}.')
21 self.device = device
22 self.block_size = block_size
23 self.num_blocks = num_blocks
cacheflow/master/server.py
+1 -1
@@ -174,7 +174,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
174 parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
175 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
176 # KV cache arguments
177 - parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
177 + parser.add_argument('--block-size', type=int, default=8, choices=[8, 16, 32], help='token block size')
178 # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
179 parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
180 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
csrc/attention_kernels.cu
+44
@@ -654,6 +654,16 @@ void single_query_cached_kv_attention(
654 block_tables,
655 context_lens,
656 max_context_len);
657 + } else if (block_size == 32) {
658 + single_query_cached_kv_attention_launcher<uint16_t, 32>(
659 + out,
660 + query,
661 + key_cache,
662 + value_cache,
663 + scale,
664 + block_tables,
665 + context_lens,
666 + max_context_len);
667 } else {
668 assert(false);
669 }
@@ -679,6 +689,16 @@ void single_query_cached_kv_attention(
689 block_tables,
690 context_lens,
691 max_context_len);
692 + } else if (block_size == 32) {
693 + single_query_cached_kv_attention_launcher<float, 32>(
694 + out,
695 + query,
696 + key_cache,
697 + value_cache,
698 + scale,
699 + block_tables,
700 + context_lens,
701 + max_context_len);
702 } else {
703 assert(false);
704 }
@@ -834,6 +854,18 @@ void multi_query_cached_kv_attention(
854 block_tables,
855 context_lens,
856 max_context_len);
857 + } else if (block_size == 32) {
858 + multi_query_cached_kv_attention_launcher<uint16_t, 32>(
859 + cu_query_lens,
860 + seq_prompt_mapping,
861 + out,
862 + query,
863 + key_cache,
864 + value_cache,
865 + scale,
866 + block_tables,
867 + context_lens,
868 + max_context_len);
869 } else {
870 assert(false);
871 }
@@ -863,6 +895,18 @@ void multi_query_cached_kv_attention(
895 block_tables,
896 context_lens,
897 max_context_len);
898 + } else if (block_size == 32) {
899 + multi_query_cached_kv_attention_launcher<float, 32>(
900 + cu_query_lens,
901 + seq_prompt_mapping,
902 + out,
903 + query,
904 + key_cache,
905 + value_cache,
906 + scale,
907 + block_tables,
908 + context_lens,
909 + max_context_len);
910 } else {
911 assert(false);
912 }
tests/kernels/attention.py
+2 -2
@@ -350,7 +350,7 @@ def test_attention(seed: int) -> None:
350 torch.random.manual_seed(seed)
351 torch.cuda.manual_seed(seed)
352 for dtype in [torch.half, torch.float]:
353 - for block_size in [8, 16]:
353 + for block_size in [8, 16, 32]:
354 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
355 print(f'Testing single_query_cached_kv_attention with '
356 f'dtype={dtype}, block_size={block_size}, '
@@ -368,7 +368,7 @@ def test_attention(seed: int) -> None:
368 # note that the test is also more likely to fail due to the much
369 # larger amount of tokens in the input may increase the variance.
370 for dtype in [torch.half, torch.float]:
371 - for block_size in [8, 16]:
371 + for block_size in [8, 16, 32]:
372 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
373 print(f'Testing multi_query_cached_kv_attention with '
374 f'dtype={dtype}, block_size={block_size}, '