A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Support block size 32 (#35)
Woosuk Kwon committed
Apr 9, 2023 at 23:07 UTC
b9926f7f663b953c886a42b691cd7ba3e4325239
4 files changed
+49
-5
cacheflow/master/block_manager.py
+2
-2
@@ -15,9 +15,9 @@ class BlockAllocator:
15
block_size: int,
16
num_blocks: int,
17
) -> None:
18
- if block_size not in [8, 16]:
18
+ if block_size not in [8, 16, 32]:
19
raise ValueError(f'Unsupported block size: {block_size}'
20
- 'The block size must be either 8 or 16.')
20
+ 'The block size must be one of {8, 16, 32}.')
21
self.device = device
22
self.block_size = block_size
23
self.num_blocks = num_blocks
cacheflow/master/server.py
+1
-1
@@ -174,7 +174,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
174
parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
175
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
176
# KV cache arguments
177
- parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
177
+ parser.add_argument('--block-size', type=int, default=8, choices=[8, 16, 32], help='token block size')
178
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
179
parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
180
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
csrc/attention_kernels.cu
+44
@@ -654,6 +654,16 @@ void single_query_cached_kv_attention(
654
block_tables,
655
context_lens,
656
max_context_len);
657
+ } else if (block_size == 32) {
658
+ single_query_cached_kv_attention_launcher<uint16_t, 32>(
659
+ out,
660
+ query,
661
+ key_cache,
662
+ value_cache,
663
+ scale,
664
+ block_tables,
665
+ context_lens,
666
+ max_context_len);
667
} else {
668
assert(false);
669
}
@@ -679,6 +689,16 @@ void single_query_cached_kv_attention(
689
block_tables,
690
context_lens,
691
max_context_len);
692
+ } else if (block_size == 32) {
693
+ single_query_cached_kv_attention_launcher<float, 32>(
694
+ out,
695
+ query,
696
+ key_cache,
697
+ value_cache,
698
+ scale,
699
+ block_tables,
700
+ context_lens,
701
+ max_context_len);
702
} else {
703
assert(false);
704
}
@@ -834,6 +854,18 @@ void multi_query_cached_kv_attention(
854
block_tables,
855
context_lens,
856
max_context_len);
857
+ } else if (block_size == 32) {
858
+ multi_query_cached_kv_attention_launcher<uint16_t, 32>(
859
+ cu_query_lens,
860
+ seq_prompt_mapping,
861
+ out,
862
+ query,
863
+ key_cache,
864
+ value_cache,
865
+ scale,
866
+ block_tables,
867
+ context_lens,
868
+ max_context_len);
869
} else {
870
assert(false);
871
}
@@ -863,6 +895,18 @@ void multi_query_cached_kv_attention(
895
block_tables,
896
context_lens,
897
max_context_len);
898
+ } else if (block_size == 32) {
899
+ multi_query_cached_kv_attention_launcher<float, 32>(
900
+ cu_query_lens,
901
+ seq_prompt_mapping,
902
+ out,
903
+ query,
904
+ key_cache,
905
+ value_cache,
906
+ scale,
907
+ block_tables,
908
+ context_lens,
909
+ max_context_len);
910
} else {
911
assert(false);
912
}
tests/kernels/attention.py
+2
-2
@@ -350,7 +350,7 @@ def test_attention(seed: int) -> None:
350
torch.random.manual_seed(seed)
351
torch.cuda.manual_seed(seed)
352
for dtype in [torch.half, torch.float]:
353
- for block_size in [8, 16]:
353
+ for block_size in [8, 16, 32]:
354
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
355
print(f'Testing single_query_cached_kv_attention with '
356
f'dtype={dtype}, block_size={block_size}, '
@@ -368,7 +368,7 @@ def test_attention(seed: int) -> None:
368
# note that the test is also more likely to fail due to the much
369
# larger amount of tokens in the input may increase the variance.
370
for dtype in [torch.half, torch.float]:
371
- for block_size in [8, 16]:
371
+ for block_size in [8, 16, 32]:
372
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
373
print(f'Testing multi_query_cached_kv_attention with '
374
f'dtype={dtype}, block_size={block_size}, '