@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a22dea54d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] Support MAP-NEO model (#5081)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

SnowDist committed May 31, 2024 at 10:24 UTC a22dea54d3e80bf069cfeed8002a193ef8b18e1b
8 files changed +18 -6
benchmarks/kernels/benchmark_paged_attention.py
+1 -1
@@ -170,7 +170,7 @@ if __name__ == '__main__':
170 parser.add_argument("--num-kv-heads", type=int, default=8)
171 parser.add_argument("--head-size",
172 type=int,
173 - choices=[64, 80, 96, 112, 128, 256],
173 + choices=[64, 80, 96, 112, 128, 192, 256],
174 default=128)
175 parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
176 parser.add_argument("--use-alibi", action="store_true")
benchmarks/kernels/benchmark_rope.py
+1 -1
@@ -93,7 +93,7 @@ if __name__ == '__main__':
93 parser.add_argument("--num-heads", type=int, default=8)
94 parser.add_argument("--head-size",
95 type=int,
96 - choices=[64, 80, 96, 112, 128, 256],
96 + choices=[64, 80, 96, 112, 128, 192, 256],
97 default=128)
98 parser.add_argument("--rotary-dim", type=int, choices=[16, 32], default=32)
99 parser.add_argument("--dtype",
csrc/attention/attention_kernels.cu
+6
@@ -754,6 +754,9 @@ void paged_attention_v1_launcher(
754 case 128:
755 LAUNCH_PAGED_ATTENTION_V1(128);
756 break;
757 + case 192:
758 + LAUNCH_PAGED_ATTENTION_V1(192);
759 + break;
760 case 256:
761 LAUNCH_PAGED_ATTENTION_V1(256);
762 break;
@@ -911,6 +914,9 @@ void paged_attention_v2_launcher(
914 case 128:
915 LAUNCH_PAGED_ATTENTION_V2(128);
916 break;
917 + case 192:
918 + LAUNCH_PAGED_ATTENTION_V2(192);
919 + break;
920 case 256:
921 LAUNCH_PAGED_ATTENTION_V2(256);
922 break;
csrc/cpu/attention.cpp
+6
@@ -390,6 +390,9 @@ void paged_attention_v1_impl_launcher(
390 case 128:
391 LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
392 break;
393 + case 192:
394 + LAUNCH_V1_ATTENTION_KERNEL(T, 192, BLOCK_SIZE);
395 + break;
396 case 256:
397 LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
398 break;
@@ -703,6 +706,9 @@ void paged_attention_v2_impl_launcher(
706 case 128:
707 LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
708 break;
709 + case 192:
710 + LAUNCH_V2_ATTENTION_KERNEL(T, 192, BLOCK_SIZE);
711 + break;
712 case 256:
713 LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
714 break;
tests/kernels/test_attention.py
+1 -1
@@ -28,7 +28,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28
29 # FlashAttention forward only supports head dimension at most 128
30 # https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
31 -HEAD_SIZES = [64, 80, 96, 112, 128, 256
31 +HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256
32 ] if not is_hip() else [64, 80, 96, 112, 128]
33
34 BLOCK_SIZES = [16, 32]
tests/kernels/test_cache.py
+1 -1
@@ -11,7 +11,7 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
11 NUM_TOKENS = [42] # Arbitrary values for testing
12 NUM_LAYERS = [1] # Arbitrary values for testing
13 NUM_HEADS = [8] # Arbitrary values for testing
14 -HEAD_SIZES = [64, 80, 96, 112, 128, 256]
14 +HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
15 BLOCK_SIZES = [8, 16, 32]
16
17 # Arbitrary values for testing
tests/kernels/test_pos_encoding.py
+1 -1
@@ -10,7 +10,7 @@ from .allclose_default import get_default_atol, get_default_rtol
10
11 IS_NEOX_STYLE = [True, False]
12 DTYPES = [torch.half, torch.bfloat16, torch.float]
13 -HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13 +HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
14 ROTARY_DIMS = [None, 32] # None means rotary dim == head size
15 NUM_HEADS = [7, 17] # Arbitrary values for testing
16 BATCH_SIZES = [1, 5] # Arbitrary values for testing
vllm/attention/ops/paged_attn.py
+1 -1
@@ -31,7 +31,7 @@ class PagedAttention:
31
32 @staticmethod
33 def get_supported_head_sizes() -> List[int]:
34 - return [64, 80, 96, 112, 128, 256]
34 + return [64, 80, 96, 112, 128, 192, 256]
35
36 @staticmethod
37 def get_kv_cache_shape(