A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] Support MAP-NEO model (#5081)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
SnowDist committed
May 31, 2024 at 10:24 UTC
a22dea54d3e80bf069cfeed8002a193ef8b18e1b
8 files changed
+18
-6
benchmarks/kernels/benchmark_paged_attention.py
+1
-1
@@ -170,7 +170,7 @@ if __name__ == '__main__':
170
parser.add_argument("--num-kv-heads", type=int, default=8)
171
parser.add_argument("--head-size",
172
type=int,
173
- choices=[64, 80, 96, 112, 128, 256],
173
+ choices=[64, 80, 96, 112, 128, 192, 256],
174
default=128)
175
parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
176
parser.add_argument("--use-alibi", action="store_true")
benchmarks/kernels/benchmark_rope.py
+1
-1
@@ -93,7 +93,7 @@ if __name__ == '__main__':
93
parser.add_argument("--num-heads", type=int, default=8)
94
parser.add_argument("--head-size",
95
type=int,
96
- choices=[64, 80, 96, 112, 128, 256],
96
+ choices=[64, 80, 96, 112, 128, 192, 256],
97
default=128)
98
parser.add_argument("--rotary-dim", type=int, choices=[16, 32], default=32)
99
parser.add_argument("--dtype",
csrc/attention/attention_kernels.cu
+6
@@ -754,6 +754,9 @@ void paged_attention_v1_launcher(
754
case 128:
755
LAUNCH_PAGED_ATTENTION_V1(128);
756
break;
757
+ case 192:
758
+ LAUNCH_PAGED_ATTENTION_V1(192);
759
+ break;
760
case 256:
761
LAUNCH_PAGED_ATTENTION_V1(256);
762
break;
@@ -911,6 +914,9 @@ void paged_attention_v2_launcher(
914
case 128:
915
LAUNCH_PAGED_ATTENTION_V2(128);
916
break;
917
+ case 192:
918
+ LAUNCH_PAGED_ATTENTION_V2(192);
919
+ break;
920
case 256:
921
LAUNCH_PAGED_ATTENTION_V2(256);
922
break;
csrc/cpu/attention.cpp
+6
@@ -390,6 +390,9 @@ void paged_attention_v1_impl_launcher(
390
case 128:
391
LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
392
break;
393
+ case 192:
394
+ LAUNCH_V1_ATTENTION_KERNEL(T, 192, BLOCK_SIZE);
395
+ break;
396
case 256:
397
LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
398
break;
@@ -703,6 +706,9 @@ void paged_attention_v2_impl_launcher(
706
case 128:
707
LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
708
break;
709
+ case 192:
710
+ LAUNCH_V2_ATTENTION_KERNEL(T, 192, BLOCK_SIZE);
711
+ break;
712
case 256:
713
LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
714
break;
tests/kernels/test_attention.py
+1
-1
@@ -28,7 +28,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28
29
# FlashAttention forward only supports head dimension at most 128
30
# https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
31
-HEAD_SIZES = [64, 80, 96, 112, 128, 256
31
+HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256
32
] if not is_hip() else [64, 80, 96, 112, 128]
33
34
BLOCK_SIZES = [16, 32]
tests/kernels/test_cache.py
+1
-1
@@ -11,7 +11,7 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
11
NUM_TOKENS = [42] # Arbitrary values for testing
12
NUM_LAYERS = [1] # Arbitrary values for testing
13
NUM_HEADS = [8] # Arbitrary values for testing
14
-HEAD_SIZES = [64, 80, 96, 112, 128, 256]
14
+HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
15
BLOCK_SIZES = [8, 16, 32]
16
17
# Arbitrary values for testing
tests/kernels/test_pos_encoding.py
+1
-1
@@ -10,7 +10,7 @@ from .allclose_default import get_default_atol, get_default_rtol
10
11
IS_NEOX_STYLE = [True, False]
12
DTYPES = [torch.half, torch.bfloat16, torch.float]
13
-HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13
+HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
14
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
15
NUM_HEADS = [7, 17] # Arbitrary values for testing
16
BATCH_SIZES = [1, 5] # Arbitrary values for testing
vllm/attention/ops/paged_attn.py
+1
-1
@@ -31,7 +31,7 @@ class PagedAttention:
31
32
@staticmethod
33
def get_supported_head_sizes() -> List[int]:
34
- return [64, 80, 96, 112, 128, 256]
34
+ return [64, 80, 96, 112, 128, 192, 256]
35
36
@staticmethod
37
def get_kv_cache_shape(