@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b0c4bab36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Kernel] Triton Configs for Fp8 Block Quantization (#11589)

Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: simon-mo <xmo@berkeley.edu>

Robert Shaw committed Jan 30, 2025 at 14:53 UTC 9b0c4bab36c8f355f562d58521650ee8d5b6095d
43 files changed +5972 -42
setup.py
+5 -1
@@ -608,7 +608,11 @@ if _build_custom_ops():
608 ext_modules.append(CMakeExtension(name="vllm._C"))
609
610 package_data = {
611 - "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
611 + "vllm": [
612 + "py.typed",
613 + "model_executor/layers/fused_moe/configs/*.json",
614 + "model_executor/layers/quantization/utils/configs/*.json",
615 + ]
616 }
617
618 if _no_device():
vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/fused_moe/fused_moe.py
+66 -25
@@ -598,15 +598,27 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
598 )
599
600
601 -def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
601 +# Adapted from: https://github.com/sgl-project/sglang/pull/2628
602 +def get_config_file_name(E: int,
603 + N: int,
604 + dtype: Optional[str],
605 + block_shape: Optional[List[int]] = None) -> str:
606 device_name = current_platform.get_device_name().replace(" ", "_")
607 dtype_selector = "" if not dtype else f",dtype={dtype}"
604 - return f"E={E},N={N},device_name={device_name}{dtype_selector}.json"
608 + block_shape_selector = ("" if not block_shape or not all(block_shape) else
609 + f",block_shape={block_shape}")
610 + return f"E={E},N={N},device_name={device_name}{dtype_selector}{block_shape_selector}.json" # noqa: E501
611
612
613 +# Adapted from: https://github.com/sgl-project/sglang/pull/2628
614 @functools.lru_cache
608 -def get_moe_configs(E: int, N: int,
609 - dtype: Optional[str]) -> Optional[Dict[int, Any]]:
615 +def get_moe_configs(
616 + E: int,
617 + N: int,
618 + dtype: Optional[str],
619 + block_n: Optional[int] = None,
620 + block_k: Optional[int] = None,
621 +) -> Optional[Dict[int, Any]]:
622 """
623 Return optimized configurations for the fused MoE kernel.
624
@@ -618,7 +630,8 @@ def get_moe_configs(E: int, N: int,
630
631 # First look up if an optimized configuration is available in the configs
632 # directory
621 - json_file_name = get_config_file_name(E, N, dtype)
633 + block_shape = [block_n, block_k] if block_n and block_k else None
634 + json_file_name = get_config_file_name(E, N, dtype, block_shape)
635
636 config_file_path = os.path.join(
637 os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
@@ -645,21 +658,53 @@ def get_default_config(
658 topk: int,
659 dtype: Optional[str],
660 is_marlin: bool,
661 + block_shape: Optional[List[int]] = None,
662 ) -> Dict[str, int]:
649 - config = {
650 - 'BLOCK_SIZE_M': 64,
651 - 'BLOCK_SIZE_N': 64,
652 - 'BLOCK_SIZE_K': 32,
653 - 'GROUP_SIZE_M': 8
654 - }
655 - # A heuristic: fused marlin works faster with this config for small M
656 - if M <= E or (is_marlin and M <= 32):
663 + if dtype == "fp8_w8a8":
664 + if block_shape is None:
665 + config = {
666 + "BLOCK_SIZE_M": 128,
667 + "BLOCK_SIZE_N": 256,
668 + "BLOCK_SIZE_K": 128,
669 + "GROUP_SIZE_M": 32,
670 + "num_warps": 8,
671 + "num_stages": 4,
672 + }
673 + if M <= E:
674 + config = {
675 + "BLOCK_SIZE_M": 64,
676 + "BLOCK_SIZE_N": 128,
677 + "BLOCK_SIZE_K": 128,
678 + "GROUP_SIZE_M": 1,
679 + "num_warps": 4,
680 + "num_stages": 4,
681 + }
682 + else:
683 + # Block-wise quant: BLOCK_SIZE_N must be divisible by block_shape[0]
684 + # BLOCK_SIZE_K must be divisible by block_shape[1]
685 + config = {
686 + "BLOCK_SIZE_M": 64,
687 + "BLOCK_SIZE_N": block_shape[0],
688 + "BLOCK_SIZE_K": block_shape[1],
689 + "GROUP_SIZE_M": 32,
690 + "num_warps": 4,
691 + "num_stages": 3,
692 + }
693 + else:
694 config = {
658 - 'BLOCK_SIZE_M': 16,
659 - 'BLOCK_SIZE_N': 32,
660 - 'BLOCK_SIZE_K': 64,
661 - 'GROUP_SIZE_M': 1
695 + "BLOCK_SIZE_M": 64,
696 + "BLOCK_SIZE_N": 64,
697 + "BLOCK_SIZE_K": 32,
698 + "GROUP_SIZE_M": 8,
699 }
700 + # A heuristic: fused marlin works faster with this config for small M
701 + if M <= E or (is_marlin and M <= 32):
702 + config = {
703 + "BLOCK_SIZE_M": 16,
704 + "BLOCK_SIZE_N": 32,
705 + "BLOCK_SIZE_K": 64,
706 + "GROUP_SIZE_M": 1,
707 + }
708 return config
709
710
@@ -679,7 +724,9 @@ def try_get_optimal_moe_config(
724 else:
725 # First try to load optimal config from the file
726 E, _, N = w2_shape
682 - configs = get_moe_configs(E, N, dtype)
727 + block_n = block_shape[0] if block_shape else 0
728 + block_k = block_shape[1] if block_shape else 0
729 + configs = get_moe_configs(E, N, dtype, block_n, block_k)
730
731 if configs:
732 # If an optimal configuration map has been found, look up the
@@ -688,13 +735,7 @@ def try_get_optimal_moe_config(
735 else:
736 # Else use the default config
737 config = get_default_config(M, E, N, w1_shape[2], top_k, dtype,
691 - is_marlin)
692 - # NOTE: For block-wise quant,
693 - # BLOCK_K must be divisible by block_shape[1]
694 - # BLOCK_N and BLOCK_M has no requirements
695 - if block_shape is not None and block_shape[0] != 0:
696 - config["BLOCK_SIZE_N"] = block_shape[0]
697 - config["BLOCK_SIZE_K"] = block_shape[1]
738 + is_marlin, block_shape)
739 return config
740
741
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 32,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 5
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 32,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 8,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 8,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 8,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 32,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 5
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 32,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 32,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 5
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 2
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 8,
24 + "num_stages": 2
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 128,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 8,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 8,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+61 -16
@@ -1,12 +1,18 @@
1 # Adapted from https://github.com/sgl-project/sglang/pull/2575
2 -from typing import List, Optional, Tuple
2 +import functools
3 +import json
4 +import os
5 +from typing import Any, Dict, List, Optional, Tuple
6
7 import torch
8 import triton
9 import triton.language as tl
10
11 +from vllm.logger import init_logger
12 from vllm.platforms import current_platform
13
14 +logger = init_logger(__name__)
15 +
16
17 def apply_w8a8_block_fp8_linear(
18 input: torch.Tensor,
@@ -277,6 +283,43 @@ def _w8a8_block_fp8_matmul(
283 tl.store(c_ptrs, c, mask=c_mask)
284
285
286 +@functools.lru_cache
287 +def get_w8a8_block_fp8_configs(N: int, K: int, block_n: int,
288 + block_k: int) -> Optional[Dict[int, Any]]:
289 + """
290 + Return optimized configurations for the w8a8 block fp8 kernel.
291 + The return value will be a dictionary that maps an irregular grid of
292 + batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the
293 + kernel on a given batch size bs, the closest batch size in the grid should
294 + be picked and the associated configuration chosen to invoke the kernel.
295 + """
296 +
297 + # First look up if an optimized configuration is available in the configs
298 + # directory
299 + device_name = current_platform.get_device_name().replace(" ", "_")
300 + json_file_name = f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8,block_shape=[{block_n}, {block_k}].json" # noqa: E501
301 +
302 + config_file_path = os.path.join(
303 + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
304 + if os.path.exists(config_file_path):
305 + with open(config_file_path) as f:
306 + logger.info(
307 + "Using configuration from %s for W8A8 Block FP8 kernel.",
308 + config_file_path,
309 + )
310 + # If a configuration has been found, return it
311 + return {int(key): val for key, val in json.load(f).items()}
312 +
313 + # If no optimized configuration is available, we will use the default
314 + # configuration
315 + logger.warning(
316 + "Using default W8A8 Block FP8 kernel config. Performance might "
317 + "be sub-optimal! Config file not found at %s",
318 + config_file_path,
319 + )
320 + return None
321 +
322 +
323 def w8a8_block_fp8_matmul(
324 A: torch.Tensor,
325 B: torch.Tensor,
@@ -316,17 +359,22 @@ def w8a8_block_fp8_matmul(
359 C_shape = A.shape[:-1] + (N, )
360 C = A.new_empty(C_shape, dtype=output_dtype)
361
319 - # TODO:
320 - # BLOCK_SIZE_M, BLOCK_SIZE_K, BLOCK_SIZE_N can be optimized.
321 - # BLOCK_SIZE_K must be divisible by block_k
322 - # BLOCK_SIZE_N and BLOCK_SIZE_M has no requirements
323 - BLOCK_SIZE_M = 128
324 - if M < BLOCK_SIZE_M:
325 - BLOCK_SIZE_M = triton.next_power_of_2(M)
326 - BLOCK_SIZE_M = max(BLOCK_SIZE_M, 16)
327 - BLOCK_SIZE_K = block_k
328 - assert block_k % BLOCK_SIZE_K == 0
329 - BLOCK_SIZE_N = block_n
362 + configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
363 + if configs:
364 + # Get the optimal config if there is one
365 + config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
366 + else:
367 + # Default config
368 + # Block-wise quant: BLOCK_SIZE_N must be divisible by block_size[0]
369 + # BLOCK_SIZE_K must be divisible by block_size[1]
370 + config = {
371 + "BLOCK_SIZE_M": 64,
372 + "BLOCK_SIZE_N": block_size[0],
373 + "BLOCK_SIZE_K": block_size[1],
374 + "GROUP_SIZE_M": 32,
375 + "num_warps": 4,
376 + "num_stages": 2,
377 + }
378
379 def grid(META):
380 return (triton.cdiv(M, META["BLOCK_SIZE_M"]) *
@@ -353,10 +401,7 @@ def w8a8_block_fp8_matmul(
401 As.stride(-1),
402 Bs.stride(1),
403 Bs.stride(0),
356 - BLOCK_SIZE_M=BLOCK_SIZE_M,
357 - BLOCK_SIZE_N=BLOCK_SIZE_N,
358 - BLOCK_SIZE_K=BLOCK_SIZE_K,
359 - GROUP_SIZE_M=8,
404 + **config,
405 )
406
407 return C