A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 02:15 UTC
[Perf] Add tuned selective_state_update configs for H200 and RTX PRO … (#44251)
Signed-off-by: Majid Taheri Andani <tahemaji@amazon.com> Co-authored-by: Majid Taheri Andani <tahemaji@amazon.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Majid committed
Jun 2, 2026 at 23:59 UTC
9af53a3c1316ed50507b835864cd00fa9ebc0578
4 files changed
+348
vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_H200,cache_dtype=float16.json
new
+87
@@ -0,0 +1,87 @@
1
+{
2
+ "triton_version": "3.6.0",
3
+ "8": {
4
+ "BLOCK_SIZE_M": 4,
5
+ "num_warps": 2
6
+ },
7
+ "16": {
8
+ "BLOCK_SIZE_M": 4,
9
+ "num_warps": 1
10
+ },
11
+ "32": {
12
+ "BLOCK_SIZE_M": 4,
13
+ "num_warps": 1
14
+ },
15
+ "64": {
16
+ "BLOCK_SIZE_M": 16,
17
+ "num_warps": 4
18
+ },
19
+ "128": {
20
+ "BLOCK_SIZE_M": 8,
21
+ "num_warps": 2
22
+ },
23
+ "256": {
24
+ "BLOCK_SIZE_M": 8,
25
+ "num_warps": 2
26
+ },
27
+ "512": {
28
+ "BLOCK_SIZE_M": 16,
29
+ "num_warps": 1
30
+ },
31
+ "1024": {
32
+ "BLOCK_SIZE_M": 16,
33
+ "num_warps": 1
34
+ },
35
+ "2048": {
36
+ "BLOCK_SIZE_M": 8,
37
+ "num_warps": 2
38
+ },
39
+ "4096": {
40
+ "BLOCK_SIZE_M": 16,
41
+ "num_warps": 2
42
+ },
43
+ "8192": {
44
+ "BLOCK_SIZE_M": 32,
45
+ "num_warps": 2
46
+ },
47
+ "12288": {
48
+ "BLOCK_SIZE_M": 32,
49
+ "num_warps": 4
50
+ },
51
+ "16384": {
52
+ "BLOCK_SIZE_M": 16,
53
+ "num_warps": 2
54
+ },
55
+ "24576": {
56
+ "BLOCK_SIZE_M": 32,
57
+ "num_warps": 4
58
+ },
59
+ "32768": {
60
+ "BLOCK_SIZE_M": 32,
61
+ "num_warps": 2
62
+ },
63
+ "49152": {
64
+ "BLOCK_SIZE_M": 16,
65
+ "num_warps": 2
66
+ },
67
+ "65536": {
68
+ "BLOCK_SIZE_M": 16,
69
+ "num_warps": 2
70
+ },
71
+ "98304": {
72
+ "BLOCK_SIZE_M": 16,
73
+ "num_warps": 2
74
+ },
75
+ "131072": {
76
+ "BLOCK_SIZE_M": 16,
77
+ "num_warps": 2
78
+ },
79
+ "196608": {
80
+ "BLOCK_SIZE_M": 16,
81
+ "num_warps": 2
82
+ },
83
+ "262144": {
84
+ "BLOCK_SIZE_M": 16,
85
+ "num_warps": 2
86
+ }
87
+}
\ No newline at end of file
vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_H200,cache_dtype=float32.json
new
+87
@@ -0,0 +1,87 @@
1
+{
2
+ "triton_version": "3.6.0",
3
+ "8": {
4
+ "BLOCK_SIZE_M": 8,
5
+ "num_warps": 4
6
+ },
7
+ "16": {
8
+ "BLOCK_SIZE_M": 4,
9
+ "num_warps": 1
10
+ },
11
+ "32": {
12
+ "BLOCK_SIZE_M": 4,
13
+ "num_warps": 1
14
+ },
15
+ "64": {
16
+ "BLOCK_SIZE_M": 4,
17
+ "num_warps": 1
18
+ },
19
+ "128": {
20
+ "BLOCK_SIZE_M": 4,
21
+ "num_warps": 1
22
+ },
23
+ "256": {
24
+ "BLOCK_SIZE_M": 4,
25
+ "num_warps": 1
26
+ },
27
+ "512": {
28
+ "BLOCK_SIZE_M": 4,
29
+ "num_warps": 1
30
+ },
31
+ "1024": {
32
+ "BLOCK_SIZE_M": 4,
33
+ "num_warps": 1
34
+ },
35
+ "2048": {
36
+ "BLOCK_SIZE_M": 4,
37
+ "num_warps": 1
38
+ },
39
+ "4096": {
40
+ "BLOCK_SIZE_M": 4,
41
+ "num_warps": 1
42
+ },
43
+ "8192": {
44
+ "BLOCK_SIZE_M": 4,
45
+ "num_warps": 1
46
+ },
47
+ "12288": {
48
+ "BLOCK_SIZE_M": 8,
49
+ "num_warps": 1
50
+ },
51
+ "16384": {
52
+ "BLOCK_SIZE_M": 8,
53
+ "num_warps": 1
54
+ },
55
+ "24576": {
56
+ "BLOCK_SIZE_M": 8,
57
+ "num_warps": 1
58
+ },
59
+ "32768": {
60
+ "BLOCK_SIZE_M": 4,
61
+ "num_warps": 1
62
+ },
63
+ "49152": {
64
+ "BLOCK_SIZE_M": 4,
65
+ "num_warps": 1
66
+ },
67
+ "65536": {
68
+ "BLOCK_SIZE_M": 8,
69
+ "num_warps": 2
70
+ },
71
+ "98304": {
72
+ "BLOCK_SIZE_M": 4,
73
+ "num_warps": 1
74
+ },
75
+ "131072": {
76
+ "BLOCK_SIZE_M": 32,
77
+ "num_warps": 4
78
+ },
79
+ "196608": {
80
+ "BLOCK_SIZE_M": 16,
81
+ "num_warps": 1
82
+ },
83
+ "262144": {
84
+ "BLOCK_SIZE_M": 16,
85
+ "num_warps": 1
86
+ }
87
+}
\ No newline at end of file
vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,cache_dtype=float16.json
new
+87
@@ -0,0 +1,87 @@
1
+{
2
+ "triton_version": "3.6.0",
3
+ "8": {
4
+ "BLOCK_SIZE_M": 4,
5
+ "num_warps": 4
6
+ },
7
+ "16": {
8
+ "BLOCK_SIZE_M": 4,
9
+ "num_warps": 1
10
+ },
11
+ "32": {
12
+ "BLOCK_SIZE_M": 4,
13
+ "num_warps": 1
14
+ },
15
+ "64": {
16
+ "BLOCK_SIZE_M": 8,
17
+ "num_warps": 8
18
+ },
19
+ "128": {
20
+ "BLOCK_SIZE_M": 16,
21
+ "num_warps": 8
22
+ },
23
+ "256": {
24
+ "BLOCK_SIZE_M": 16,
25
+ "num_warps": 8
26
+ },
27
+ "512": {
28
+ "BLOCK_SIZE_M": 16,
29
+ "num_warps": 8
30
+ },
31
+ "1024": {
32
+ "BLOCK_SIZE_M": 16,
33
+ "num_warps": 8
34
+ },
35
+ "2048": {
36
+ "BLOCK_SIZE_M": 16,
37
+ "num_warps": 8
38
+ },
39
+ "4096": {
40
+ "BLOCK_SIZE_M": 16,
41
+ "num_warps": 8
42
+ },
43
+ "8192": {
44
+ "BLOCK_SIZE_M": 32,
45
+ "num_warps": 1
46
+ },
47
+ "12288": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "num_warps": 1
50
+ },
51
+ "16384": {
52
+ "BLOCK_SIZE_M": 32,
53
+ "num_warps": 4
54
+ },
55
+ "24576": {
56
+ "BLOCK_SIZE_M": 32,
57
+ "num_warps": 4
58
+ },
59
+ "32768": {
60
+ "BLOCK_SIZE_M": 4,
61
+ "num_warps": 1
62
+ },
63
+ "49152": {
64
+ "BLOCK_SIZE_M": 32,
65
+ "num_warps": 2
66
+ },
67
+ "65536": {
68
+ "BLOCK_SIZE_M": 32,
69
+ "num_warps": 1
70
+ },
71
+ "98304": {
72
+ "BLOCK_SIZE_M": 4,
73
+ "num_warps": 1
74
+ },
75
+ "131072": {
76
+ "BLOCK_SIZE_M": 4,
77
+ "num_warps": 1
78
+ },
79
+ "196608": {
80
+ "BLOCK_SIZE_M": 32,
81
+ "num_warps": 1
82
+ },
83
+ "262144": {
84
+ "BLOCK_SIZE_M": 16,
85
+ "num_warps": 1
86
+ }
87
+}
\ No newline at end of file
vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,cache_dtype=float32.json
new
+87
@@ -0,0 +1,87 @@
1
+{
2
+ "triton_version": "3.6.0",
3
+ "8": {
4
+ "BLOCK_SIZE_M": 8,
5
+ "num_warps": 8
6
+ },
7
+ "16": {
8
+ "BLOCK_SIZE_M": 8,
9
+ "num_warps": 8
10
+ },
11
+ "32": {
12
+ "BLOCK_SIZE_M": 8,
13
+ "num_warps": 8
14
+ },
15
+ "64": {
16
+ "BLOCK_SIZE_M": 8,
17
+ "num_warps": 8
18
+ },
19
+ "128": {
20
+ "BLOCK_SIZE_M": 8,
21
+ "num_warps": 8
22
+ },
23
+ "256": {
24
+ "BLOCK_SIZE_M": 8,
25
+ "num_warps": 8
26
+ },
27
+ "512": {
28
+ "BLOCK_SIZE_M": 8,
29
+ "num_warps": 8
30
+ },
31
+ "1024": {
32
+ "BLOCK_SIZE_M": 16,
33
+ "num_warps": 8
34
+ },
35
+ "2048": {
36
+ "BLOCK_SIZE_M": 16,
37
+ "num_warps": 8
38
+ },
39
+ "4096": {
40
+ "BLOCK_SIZE_M": 16,
41
+ "num_warps": 1
42
+ },
43
+ "8192": {
44
+ "BLOCK_SIZE_M": 4,
45
+ "num_warps": 8
46
+ },
47
+ "12288": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "num_warps": 1
50
+ },
51
+ "16384": {
52
+ "BLOCK_SIZE_M": 4,
53
+ "num_warps": 1
54
+ },
55
+ "24576": {
56
+ "BLOCK_SIZE_M": 4,
57
+ "num_warps": 1
58
+ },
59
+ "32768": {
60
+ "BLOCK_SIZE_M": 4,
61
+ "num_warps": 4
62
+ },
63
+ "49152": {
64
+ "BLOCK_SIZE_M": 16,
65
+ "num_warps": 4
66
+ },
67
+ "65536": {
68
+ "BLOCK_SIZE_M": 64,
69
+ "num_warps": 8
70
+ },
71
+ "98304": {
72
+ "BLOCK_SIZE_M": 16,
73
+ "num_warps": 1
74
+ },
75
+ "131072": {
76
+ "BLOCK_SIZE_M": 8,
77
+ "num_warps": 1
78
+ },
79
+ "196608": {
80
+ "BLOCK_SIZE_M": 64,
81
+ "num_warps": 8
82
+ },
83
+ "262144": {
84
+ "BLOCK_SIZE_M": 64,
85
+ "num_warps": 4
86
+ }
87
+}
\ No newline at end of file