@tduncan / Nvidia-Optimization-Stack-VLLM / commits / d891b9bd51

[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>

Jinzhen Lin committed Jul 7, 2026 at 04:36 UTC d891b9bd51ce726a1bff24a72d03090057718789
73 files changed +1336 -122
.buildkite/test_areas/lm_eval.yaml
+60 -12
@@ -150,9 +150,9 @@ steps:
150 commands:
151 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
152
153 -- label: LM Eval Humming (A100 - TEMPORARY)
154 - key: lm-eval-humming-a100
155 - timeout_in_minutes: 30
153 +- label: LM Eval Humming f16 (A100 - TEMPORARY)
154 + key: lm-eval-humming-f16-a100
155 + timeout_in_minutes: 120
156 device: a100
157 optional: true
158 num_devices: 1
@@ -160,13 +160,29 @@ steps:
160 - vllm/model_executor/layers/quantization/humming.py
161 - vllm/model_executor/layers/quantization/utils/humming_utils.py
162 - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
163 - - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
163 + - vllm/model_executor/layers/fused_moe/oracle/
164 + - vllm/model_executor/kernels/linear/
165 commands:
166 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
167
167 -- label: LM Eval Humming (H100 - TEMPORARY)
168 - key: lm-eval-humming-h100
169 - timeout_in_minutes: 30
168 +- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
169 + key: lm-eval-humming-act-a100
170 + timeout_in_minutes: 120
171 + device: a100
172 + optional: true
173 + num_devices: 1
174 + source_file_dependencies:
175 + - vllm/model_executor/layers/quantization/humming.py
176 + - vllm/model_executor/layers/quantization/utils/humming_utils.py
177 + - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
178 + - vllm/model_executor/layers/fused_moe/oracle/
179 + - vllm/model_executor/kernels/linear/
180 + commands:
181 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
182 +
183 +- label: LM Eval Humming f16 (H100 - TEMPORARY)
184 + key: lm-eval-humming-f16-h100
185 + timeout_in_minutes: 120
186 device: h100
187 optional: true
188 num_devices: 1
@@ -174,14 +190,30 @@ steps:
190 - vllm/model_executor/layers/quantization/humming.py
191 - vllm/model_executor/layers/quantization/utils/humming_utils.py
192 - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
177 - - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
193 + - vllm/model_executor/layers/fused_moe/oracle/
194 + - vllm/model_executor/kernels/linear/
195 commands:
196 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
197 +
198 +- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
199 + key: lm-eval-humming-act-h100
200 + timeout_in_minutes: 120
201 + device: h100
202 + optional: true
203 + num_devices: 1
204 + source_file_dependencies:
205 + - vllm/model_executor/layers/quantization/humming.py
206 + - vllm/model_executor/layers/quantization/utils/humming_utils.py
207 + - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
208 + - vllm/model_executor/layers/fused_moe/oracle/
209 + - vllm/model_executor/kernels/linear/
210 + commands:
211 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
212 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
213
182 -- label: LM Eval Humming (B200 - TEMPORARY)
183 - key: lm-eval-humming-b200
184 - timeout_in_minutes: 30
214 +- label: LM Eval Humming f16 (B200 - TEMPORARY)
215 + key: lm-eval-humming-f16-b200
216 + timeout_in_minutes: 120
217 device: b200-k8s
218 optional: true
219 num_devices: 1
@@ -189,10 +221,26 @@ steps:
221 - vllm/model_executor/layers/quantization/humming.py
222 - vllm/model_executor/layers/quantization/utils/humming_utils.py
223 - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
192 - - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
224 + - vllm/model_executor/layers/fused_moe/oracle/
225 + - vllm/model_executor/kernels/linear/
226 commands:
227 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
228 +
229 +- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY)
230 + key: lm-eval-humming-act-b200
231 + timeout_in_minutes: 120
232 + device: b200-k8s
233 + optional: true
234 + num_devices: 1
235 + source_file_dependencies:
236 + - vllm/model_executor/layers/quantization/humming.py
237 + - vllm/model_executor/layers/quantization/utils/humming_utils.py
238 + - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
239 + - vllm/model_executor/layers/fused_moe/oracle/
240 + - vllm/model_executor/kernels/linear/
241 + commands:
242 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
243 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
244
245 - label: LM Eval TurboQuant KV Cache
246 key: lm-eval-turboquant-kv-cache
requirements/cuda.txt
+1 -1
@@ -28,4 +28,4 @@ quack-kernels>=0.3.3
28 tokenspeed-mla==0.1.2; platform_system == "Linux"
29
30 # Humming kernels for quantization gemm
31 -humming-kernels[cu13]==0.1.6
31 +humming-kernels[cu13]==0.1.10
tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml new
+10
@@ -0,0 +1,10 @@
1 +model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
2 +accuracy_threshold: 0.55
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --linear-backend humming
9 +env:
10 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml new
+8
@@ -0,0 +1,8 @@
1 +model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
2 +accuracy_threshold: 0.55
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml new
+10
@@ -0,0 +1,10 @@
1 +model_name: "mgoin/Qwen3-0.6B-MXFP8"
2 +accuracy_threshold: 0.39
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --linear-backend humming
9 +env:
10 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml new
+8
@@ -0,0 +1,8 @@
1 +model_name: "mgoin/Qwen3-0.6B-MXFP8"
2 +accuracy_threshold: 0.39
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml new
+13
@@ -0,0 +1,13 @@
1 +model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --linear-backend humming
11 + --dtype bfloat16
12 +env:
13 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml new
+13
@@ -0,0 +1,13 @@
1 +model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --linear-backend humming
11 + --dtype bfloat16
12 +env:
13 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --linear-backend humming
11 + --dtype bfloat16
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
10 +env:
11 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml new
+9
@@ -0,0 +1,9 @@
1 +model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
10 +env:
11 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml new
+9
@@ -0,0 +1,9 @@
1 +model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
10 +env:
11 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
10 +env:
11 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml new
+9
@@ -0,0 +1,9 @@
1 +model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
10 +env:
11 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml new
+9
@@ -0,0 +1,9 @@
1 +model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
+1 -2
@@ -5,8 +5,7 @@ num_fewshot: 5
5 server_args: >-
6 --enforce-eager
7 --max-model-len 8192
8 - --tensor-parallel-size 1
8 --quantization humming
10 - --kernel-config.enable_flashinfer_autotune=False
9 + --linear-backend humming
10 env:
11 VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml
+1 -2
@@ -5,6 +5,5 @@ num_fewshot: 5
5 server_args: >-
6 --enforce-eager
7 --max-model-len 8192
8 - --tensor-parallel-size 1
8 --quantization humming
10 - --kernel-config.enable_flashinfer_autotune=False
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml new
+9
@@ -0,0 +1,9 @@
1 +model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2 +accuracy_threshold: 0.86
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --moe-backend humming
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml new
+13
@@ -0,0 +1,13 @@
1 +model_name: "Qwen/Qwen3-30B-A3B"
2 +accuracy_threshold: 0.80
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --tensor-parallel-size 1
9 + --quantization humming
10 + --kernel-config.enable_flashinfer_autotune=False
11 +env:
12 + VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
13 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml new
+13
@@ -0,0 +1,13 @@
1 +model_name: "Qwen/Qwen3-30B-A3B"
2 +accuracy_threshold: 0.80
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --tensor-parallel-size 1
9 + --quantization humming
10 + --kernel-config.enable_flashinfer_autotune=False
11 +env:
12 + VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
13 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml new
+12
@@ -0,0 +1,12 @@
1 +model_name: "Qwen/Qwen3-30B-A3B"
2 +accuracy_threshold: 0.80
3 +num_questions: 1319
4 +num_fewshot: 5
5 +server_args: >-
6 + --enforce-eager
7 + --max-model-len 8192
8 + --tensor-parallel-size 1
9 + --quantization humming
10 + --kernel-config.enable_flashinfer_autotune=False
11 +env:
12 + VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml new
+12
@@ -0,0 +1,12 @@
1 +model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --linear-backend humming
11 +env:
12 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml new
+10
@@ -0,0 +1,10 @@
1 +model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml new
+12
@@ -0,0 +1,12 @@
1 +model_name: "Qwen/Qwen3.5-35B-A3B"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --quantization experts_int8
11 +env:
12 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml new
+10
@@ -0,0 +1,10 @@
1 +model_name: "Qwen/Qwen3.5-35B-A3B"
2 +accuracy_threshold: 0.90
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --quantization experts_int8
tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml new
+10
@@ -0,0 +1,10 @@
1 +model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
2 +accuracy_threshold: 0.82
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 +env:
10 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml new
+11
@@ -0,0 +1,11 @@
1 +model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
2 +accuracy_threshold: 0.82
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --linear-backend humming
10 +env:
11 + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml new
+9
@@ -0,0 +1,9 @@
1 +model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
2 +accuracy_threshold: 0.82
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --linear-backend humming
tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml new
+10
@@ -0,0 +1,10 @@
1 +model_name: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
2 +accuracy_threshold: 0.91
3 +num_questions: 1319
4 +num_fewshot: 5
5 +gen_prefix: " <think>\n\n</think>\n"
6 +server_args: >-
7 + --enforce-eager
8 + --max-model-len 8192
9 + --moe-backend humming
10 + --linear-backend humming
tests/evals/gsm8k/configs/humming/config-act-fp8.txt
+7
@@ -1,2 +1,9 @@
1 gpt-oss-20b-humming-act-fp8.yaml
2 Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
3 +Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
4 +Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
5 +Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
6 +Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
7 +Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
8 +Qwen3-30B-A3B-AWQ-humming.yaml
9 +Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
tests/evals/gsm8k/configs/humming/config-act-int8.txt new
+4
@@ -0,0 +1,4 @@
1 +Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
2 +Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
3 +Qwen3-30B-A3B-AWQ-humming.yaml
4 +Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt new
+3
@@ -0,0 +1,3 @@
1 +Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
2 +Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
3 +Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
tests/evals/gsm8k/configs/humming/config.txt
+11
@@ -1,2 +1,13 @@
1 gpt-oss-20b-humming.yaml
2 Qwen3-30B-A3B-MXFP4A16-humming.yaml
3 +Qwen2-1.5B-Instruct-FP8W8-humming.yaml
4 +Qwen3-0.6B-MXFP8-humming.yaml
5 +Qwen3.6-35B-A3B-NVFP4-humming.yaml
6 +Qwen3-30B-A3B-Fp8-v1-humming.yaml
7 +Qwen3-30B-A3B-FP8-block-humming.yaml
8 +Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
9 +Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
10 +Qwen3-30B-A3B-NVFP4-humming.yaml
11 +Qwen3-30B-A3B-AWQ-humming.yaml
12 +Qwen3.5-35B-A3B-FP8-humming.yaml
13 +Qwen3.5-35B-A3B-experts-int8-humming.yaml
tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml
-1
@@ -5,7 +5,6 @@ num_fewshot: 5
5 server_args: >-
6 --enforce-eager
7 --max-model-len 8192
8 - --tensor-parallel-size 1
8 --moe-backend humming
9 env:
10 VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml
-1
@@ -5,5 +5,4 @@ num_fewshot: 5
5 server_args: >-
6 --enforce-eager
7 --max-model-len 8192
8 - --tensor-parallel-size 1
8 --moe-backend humming
tests/evals/gsm8k/gsm8k_eval.py
+8 -4
@@ -146,6 +146,7 @@ async def call_vllm_chat_api(
146 def _build_gsm8k_prompts(
147 num_questions: int = 1319,
148 num_shots: int = 5,
149 + gen_prefix: str = "",
150 ) -> tuple[list[str], list[int]]:
151 """Build few-shot GSM8K completion prompts and ground-truth labels."""
152 if num_questions == 0:
@@ -157,14 +158,15 @@ def _build_gsm8k_prompts(
158 for i in range(num_shots):
159 few_shot_examples += (
160 f"Question: {train_data[i]['question']}\n"
160 - f"Answer: {train_data[i]['answer']}\n\n"
161 + f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
162 )
163
164 prompts = []
165 labels = []
166 for i in range(num_questions):
167 prompts.append(
167 - few_shot_examples + f"Question: {test_data[i]['question']}\nAnswer:"
168 + few_shot_examples
169 + + f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
170 )
171 labels.append(get_answer_value(test_data[i]["answer"]))
172
@@ -213,6 +215,7 @@ def evaluate_gsm8k(
215 temperature: float = 0.0,
216 seed: int | None = 42,
217 request_timeout_seconds: float = 600,
218 + gen_prefix: str = "",
219 ) -> dict[str, float | int]:
220 """
221 Evaluate GSM8K accuracy using vLLM serve endpoint.
@@ -220,7 +223,7 @@ def evaluate_gsm8k(
223 Returns dict with accuracy, invalid_rate, latency, etc.
224 """
225 base_url = f"{host}:{port}"
223 - prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
226 + prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
227 num_questions = len(prompts)
228
229 async def run_async_evaluation():
@@ -278,6 +281,7 @@ def evaluate_gsm8k_offline(
281 num_shots: int = 5,
282 max_tokens: int = 256,
283 temperature: float = 0.0,
284 + gen_prefix: str = "",
285 ) -> dict[str, float | int]:
286 """Evaluate GSM8K accuracy using an offline vllm.LLM object.
287
@@ -286,7 +290,7 @@ def evaluate_gsm8k_offline(
290 """
291 from vllm import SamplingParams
292
289 - prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
293 + prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
294
295 sampling_params = SamplingParams(
296 temperature=temperature,
tests/evals/gsm8k/test_gsm8k_correctness.py
+1
@@ -70,6 +70,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
70 host=host,
71 port=port,
72 request_timeout_seconds=request_timeout_seconds,
73 + gen_prefix=eval_config.get("gen_prefix", ""),
74 )
75
76 return results
vllm/config/kernel.py
+1
@@ -147,6 +147,7 @@ LinearBackend = Literal[
147 "flashinfer_cudnn",
148 "flashinfer_b12x",
149 "marlin",
150 + "humming",
151 "triton",
152 "deep_gemm",
153 "torch",
vllm/config/model.py
+1
@@ -999,6 +999,7 @@ class ModelConfig:
999 "modelopt",
1000 "modelopt_fp4",
1001 "modelopt_mxfp8",
1002 + "mxfp8",
1003 "modelopt_mixed",
1004 # Ensure heavy backends are probed last to avoid unnecessary
1005 # imports during override detection (e.g., MXFP4 imports Triton)
vllm/model_executor/kernels/linear/__init__.py
+29 -1
@@ -74,6 +74,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import (
74 from vllm.model_executor.kernels.linear.mxfp4.flashinfer import (
75 FlashInferMxFp4LinearKernel,
76 )
77 +from vllm.model_executor.kernels.linear.mxfp4.humming import (
78 + HummingMxFp4LinearKernel,
79 +)
80 from vllm.model_executor.kernels.linear.mxfp4.marlin import (
81 MarlinMxFp4LinearKernel,
82 )
@@ -91,6 +94,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
94 FlashInferCutedslMxfp8LinearKernel,
95 FlashInferCutlassMxfp8LinearKernel,
96 )
97 +from vllm.model_executor.kernels.linear.mxfp8.humming import (
98 + HummingMxfp8LinearKernel,
99 +)
100 from vllm.model_executor.kernels.linear.mxfp8.marlin import (
101 MarlinMxfp8LinearKernel,
102 )
@@ -120,6 +126,9 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
126 FlashInferCutlassNvFp4LinearKernel,
127 FlashInferTrtllmNvFp4LinearKernel,
128 )
129 +from vllm.model_executor.kernels.linear.nvfp4.humming import (
130 + HummingNvFp4LinearKernel,
131 +)
132 from vllm.model_executor.kernels.linear.nvfp4.marlin import (
133 MarlinNvFp4LinearKernel,
134 )
@@ -154,6 +163,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import (
163 FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
164 FlashInferFP8ScaledMMLinearKernel,
165 )
166 +from vllm.model_executor.kernels.linear.scaled_mm.humming import (
167 + HummingFP8ScaledMMLinearKernel,
168 + HummingInt8ScaledMMLinearKernel,
169 +)
170 from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
171 MarlinFP8ScaledMMLinearKernel,
172 )
@@ -225,6 +238,14 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
238 "flashinfer_b12x": {
239 FlashInferB12xNvFp4LinearKernel,
240 },
241 + "humming": {
242 + HummingFP8ScaledMMLinearKernel,
243 + HummingInt8ScaledMMLinearKernel,
244 + HummingLinearKernel,
245 + HummingMxfp8LinearKernel,
246 + HummingMxFp4LinearKernel,
247 + HummingNvFp4LinearKernel,
248 + },
249 "marlin": {
250 MarlinFP8ScaledMMLinearKernel,
251 MarlinLinearKernel,
@@ -292,6 +313,7 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
313 PlatformEnum.CUDA: [
314 CutlassInt8ScaledMMLinearKernel,
315 TritonInt8ScaledMMLinearKernel,
316 + HummingInt8ScaledMMLinearKernel,
317 ],
318 PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
319 }
@@ -304,6 +326,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
326 CutlassFP8ScaledMMLinearKernel,
327 PerTensorTorchFP8ScaledMMLinearKernel,
328 ChannelWiseTorchFP8ScaledMMLinearKernel,
329 + HummingFP8ScaledMMLinearKernel,
330 ],
331 PlatformEnum.ROCM: [
332 AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
@@ -335,6 +358,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
358 CutlassFp8BlockScaledMMKernel,
359 MarlinFP8ScaledMMLinearKernel,
360 TritonFp8BlockScaledMMKernel,
361 + HummingFP8ScaledMMLinearKernel,
362 ],
363 PlatformEnum.ROCM: [
364 AiterFp8BlockScaledMMKernel,
@@ -351,6 +375,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
375
376 _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
377 PlatformEnum.CUDA: [
378 + HummingFP8ScaledMMLinearKernel,
379 MarlinFP8ScaledMMLinearKernel,
380 ],
381 PlatformEnum.ROCM: [
@@ -371,10 +396,10 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
396 MacheteLinearKernel,
397 AllSparkLinearKernel,
398 MarlinLinearKernel,
374 - HummingLinearKernel,
399 ConchLinearKernel,
400 ExllamaLinearKernel,
401 TritonW4A16LinearKernel,
402 + HummingLinearKernel,
403 ],
404 PlatformEnum.ROCM: [
405 RDNA3W4A16LinearKernel,
@@ -400,6 +425,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
425 FlashInferCutlassMxfp8LinearKernel,
426 MarlinMxfp8LinearKernel,
427 EmulationMxfp8LinearKernel,
428 + HummingMxfp8LinearKernel,
429 ],
430 PlatformEnum.ROCM: [
431 # Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -426,6 +452,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
452 FlashInferCudnnNvFp4LinearKernel,
453 FbgemmNvFp4LinearKernel,
454 EmulationNvFp4LinearKernel,
455 + HummingNvFp4LinearKernel,
456 ],
457 PlatformEnum.ROCM: [
458 EmulationNvFp4LinearKernel,
@@ -436,6 +463,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
463 PlatformEnum.CUDA: [
464 FlashInferMxFp4LinearKernel,
465 MarlinMxFp4LinearKernel,
466 + HummingMxFp4LinearKernel,
467 ],
468 PlatformEnum.XPU: [
469 XPUMxFp4LinearKernel,
vllm/model_executor/kernels/linear/mixed_precision/__init__.py
+4
@@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import (
19 from vllm.model_executor.kernels.linear.mixed_precision.exllama import (
20 ExllamaLinearKernel,
21 )
22 +from vllm.model_executor.kernels.linear.mixed_precision.humming import (
23 + HummingLinearKernel,
24 +)
25 from vllm.model_executor.kernels.linear.mixed_precision.machete import (
26 MacheteLinearKernel,
27 )
@@ -52,6 +55,7 @@ __all__ = [
55 "CutlassW4A8LinearKernel",
56 "Dynamic4bitLinearKernel",
57 "ExllamaLinearKernel",
58 + "HummingLinearKernel",
59 "MacheteLinearKernel",
60 "MarlinLinearKernel",
61 "RDNA3W4A16LinearKernel",
vllm/model_executor/kernels/linear/mixed_precision/humming.py
+5 -2
@@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel):
23 return False, "Humming is not installed"
24 if c.has_g_idx:
25 return False, "Humming does not support act-order (g_idx)"
26 - if c.zero_points:
27 - return False, "Humming linear kernel only supports symmetric weights"
26 return True, None
27
28 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel):
39 "group_size": 0 if group_size == -1 else group_size,
40 }
41
42 + if self.config.zero_points:
43 + assert self.w_zp_name is not None
44 + name_map["zero_point"] = self.w_zp_name
45 + quant_config["has_zero_point"] = True
46 +
47 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
48 prepare_humming_layer(layer, quant_config)
49
vllm/model_executor/kernels/linear/mxfp4/humming.py new
+63
@@ -0,0 +1,63 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import torch
5 +
6 +from vllm.model_executor.layers.quantization.utils.humming_utils import (
7 + convert_linear_layer_to_humming_standard,
8 + prepare_humming_layer,
9 +)
10 +from vllm.platforms import current_platform
11 +
12 +from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
13 +
14 +
15 +class HummingMxFp4LinearKernel(MxFp4LinearKernel):
16 + """Humming GEMM Kernel for MXFP4."""
17 +
18 + @classmethod
19 + def is_supported(
20 + cls, compute_capability: int | None = None
21 + ) -> tuple[bool, str | None]:
22 + if not current_platform.is_cuda():
23 + return False, "Humming only supported on CUDA"
24 +
25 + if not current_platform.has_device_capability(75):
26 + return False, "Humming only supported on SM75+"
27 +
28 + return True, None
29 +
30 + @classmethod
31 + def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
32 + return True, None
33 +
34 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
35 + layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
36 + name_map = {"weight": "weight", "weight_scale": "weight_scale"}
37 +
38 + quant_config = {
39 + "quant_method": "humming",
40 + "dtype": "float4e2m1",
41 + "scale_dtype": "float8e8m0",
42 + "group_size": 32,
43 + "weight_scale_type": "group",
44 + }
45 +
46 + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
47 + prepare_humming_layer(layer, quant_config)
48 +
49 + def apply_weights(
50 + self,
51 + layer: torch.nn.Module,
52 + x: torch.Tensor,
53 + bias: torch.Tensor | None = None,
54 + ) -> torch.Tensor:
55 + from vllm.utils.humming import HummingMethod
56 +
57 + flatten_inputs = x.view(-1, x.size(-1))
58 + output = HummingMethod.forward_layer(
59 + layer=layer,
60 + inputs=flatten_inputs,
61 + compute_config=layer.compute_config,
62 + )
63 + return output.view(*x.shape[:-1], output.size(-1))
vllm/model_executor/kernels/linear/mxfp8/humming.py new
+63
@@ -0,0 +1,63 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import torch
5 +
6 +from vllm.model_executor.layers.quantization.utils.humming_utils import (
7 + convert_linear_layer_to_humming_standard,
8 + prepare_humming_layer,
9 +)
10 +from vllm.platforms import current_platform
11 +
12 +from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
13 +
14 +
15 +class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
16 + """Humming GEMM Kernel for MXFP8."""
17 +
18 + @classmethod
19 + def is_supported(
20 + cls, compute_capability: int | None = None
21 + ) -> tuple[bool, str | None]:
22 + if not current_platform.is_cuda():
23 + return False, "Humming only supported on CUDA"
24 +
25 + if not current_platform.has_device_capability(75):
26 + return False, "Humming only supported on SM75+"
27 +
28 + return True, None
29 +
30 + @classmethod
31 + def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
32 + return True, None
33 +
34 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
35 + layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
36 + name_map = {"weight": "weight", "weight_scale": "weight_scale"}
37 +
38 + quant_config = {
39 + "quant_method": "humming",
40 + "dtype": "float8e4m3",
41 + "scale_dtype": "float8e8m0",
42 + "group_size": 32,
43 + "weight_scale_type": "group",
44 + }
45 +
46 + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
47 + prepare_humming_layer(layer, quant_config)
48 +
49 + def apply_weights(
50 + self,
51 + layer: torch.nn.Module,
52 + x: torch.Tensor,
53 + bias: torch.Tensor | None = None,
54 + ) -> torch.Tensor:
55 + from vllm.utils.humming import HummingMethod
56 +
57 + flatten_inputs = x.view(-1, x.size(-1))
58 + output = HummingMethod.forward_layer(
59 + layer=layer,
60 + inputs=flatten_inputs,
61 + compute_config=layer.compute_config,
62 + )
63 + return output.view(*x.shape[:-1], output.size(-1))
vllm/model_executor/kernels/linear/nvfp4/humming.py new
+73
@@ -0,0 +1,73 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import torch
5 +
6 +from vllm.logger import init_logger
7 +from vllm.model_executor.layers.quantization.utils.humming_utils import (
8 + prepare_humming_layer,
9 +)
10 +from vllm.platforms import current_platform
11 +
12 +from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class HummingNvFp4LinearKernel(NvFp4LinearKernel):
18 + """Humming GEMM Kernel for NVFP4."""
19 +
20 + @classmethod
21 + def is_supported(
22 + cls, compute_capability: int | None = None
23 + ) -> tuple[bool, str | None]:
24 + if not current_platform.is_cuda():
25 + return False, "Humming only supported on CUDA"
26 +
27 + if not current_platform.has_device_capability(75):
28 + return False, "Humming only supported on SM75+"
29 +
30 + return True, None
31 +
32 + @classmethod
33 + def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
34 + return True, None
35 +
36 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
37 + # Route through humming's compressed-tensors nvfp4 loader (same path as
38 + # the MoE oracle); the native group_tensor schema mishandles a scalar
39 + # global scale.
40 + quant_config = {
41 + "quant_method": "compressed-tensors",
42 + "format": "nvfp4-pack-quantized",
43 + "type": "float",
44 + "num_bits": 4,
45 + "strategy": "group",
46 + "group_size": 16,
47 + }
48 + # CT pack-quantized reads `weight_packed`; the scheme renamed it to `weight`.
49 + if not hasattr(layer, "weight_packed"):
50 + layer.weight_packed = layer.weight
51 + del layer.weight
52 + # The CT linear scheme inverts the global scale (1/scale) for
53 + # marlin/cutlass; humming wants the original.
54 + layer.weight_global_scale = torch.nn.Parameter(
55 + 1.0 / layer.weight_global_scale, requires_grad=False
56 + )
57 + prepare_humming_layer(layer, quant_config)
58 +
59 + def apply_weights(
60 + self,
61 + layer: torch.nn.Module,
62 + x: torch.Tensor,
63 + bias: torch.Tensor | None = None,
64 + ) -> torch.Tensor:
65 + from vllm.utils.humming import HummingMethod
66 +
67 + flatten_inputs = x.view(-1, x.size(-1))
68 + output = HummingMethod.forward_layer(
69 + layer=layer,
70 + inputs=flatten_inputs,
71 + compute_config=layer.compute_config,
72 + )
73 + return output.view(*x.shape[:-1], output.size(-1))
vllm/model_executor/kernels/linear/scaled_mm/humming.py new
+156
@@ -0,0 +1,156 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import torch
5 +
6 +from vllm.logger import init_logger
7 +from vllm.model_executor.layers.quantization.utils.humming_utils import (
8 + convert_linear_layer_to_humming_standard,
9 + prepare_humming_layer,
10 +)
11 +from vllm.platforms import current_platform
12 +
13 +from .ScaledMMLinearKernel import (
14 + FP8ScaledMMLinearKernel,
15 + FP8ScaledMMLinearLayerConfig,
16 + Int8ScaledMMLinearKernel,
17 + Int8ScaledMMLinearLayerConfig,
18 +)
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
24 + """Humming GEMM Kernel for FP8."""
25 +
26 + @classmethod
27 + def is_supported(
28 + cls, compute_capability: int | None = None
29 + ) -> tuple[bool, str | None]:
30 + if not current_platform.is_cuda():
31 + return False, "Humming only supported on CUDA"
32 +
33 + if not current_platform.has_device_capability(75):
34 + return False, "Humming only supported on SM75+"
35 +
36 + return True, None
37 +
38 + @classmethod
39 + def can_implement(
40 + cls, config: FP8ScaledMMLinearLayerConfig
41 + ) -> tuple[bool, str | None]:
42 + return True, None
43 +
44 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
45 + from vllm.utils.humming import dtypes
46 +
47 + name_map = {"weight": "weight", "weight_scale": "weight_scale"}
48 + scale_torch_dtype = self.config.weight_quant_key.scale.dtype
49 + scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype)
50 +
51 + quant_config = {
52 + "quant_method": "humming",
53 + "dtype": "float8e4m3",
54 + "scale_dtype": scale_dtype,
55 + }
56 +
57 + assert self.config.weight_quant_key.scale2 is None
58 + scale_group_shape = self.config.weight_quant_key.scale.group_shape
59 + if scale_group_shape.is_per_tensor():
60 + quant_config["weight_scale_type"] = "tensor"
61 + if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
62 + del name_map["weight_scale"]
63 + name_map["global_scale"] = "weight_scale"
64 + elif scale_group_shape.is_per_channel():
65 + quant_config["weight_scale_type"] = "channel"
66 + elif scale_group_shape.is_per_group():
67 + quant_config["weight_scale_type"] = "group"
68 + quant_config["group_size"] = scale_group_shape.col
69 + else:
70 + assert scale_group_shape.row > 0 and scale_group_shape.col > 0
71 + quant_config["weight_scale_type"] = "block"
72 + quant_config["weight_scale_group_size_n"] = scale_group_shape.row
73 + quant_config["weight_scale_group_size"] = scale_group_shape.col
74 +
75 + if hasattr(layer, "weight_scale_inv"):
76 + name_map["weight_scale"] = "weight_scale_inv"
77 +
78 + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
79 + prepare_humming_layer(layer, quant_config)
80 +
81 + def apply_weights(
82 + self,
83 + layer: torch.nn.Module,
84 + x: torch.Tensor,
85 + bias: torch.Tensor | None = None,
86 + ) -> torch.Tensor:
87 + from vllm.utils.humming import HummingMethod
88 +
89 + flatten_inputs = x.view(-1, x.size(-1))
90 + output = HummingMethod.forward_layer(
91 + layer=layer,
92 + inputs=flatten_inputs,
93 + compute_config=layer.compute_config,
94 + )
95 + return output.view(*x.shape[:-1], output.size(-1))
96 +
97 + def apply_scaled_mm(
98 + self,
99 + *,
100 + A: torch.Tensor,
101 + B: torch.Tensor,
102 + out_dtype: torch.dtype,
103 + As: torch.Tensor,
104 + Bs: torch.Tensor,
105 + bias: torch.Tensor | None,
106 + output_shape: list,
107 + ) -> torch.Tensor:
108 + pass
109 +
110 +
111 +class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
112 + """Humming GEMM Kernel for INT8."""
113 +
114 + @classmethod
115 + def is_supported(
116 + cls, compute_capability: int | None = None
117 + ) -> tuple[bool, str | None]:
118 + if not current_platform.is_cuda():
119 + return False, "Humming only supported on CUDA"
120 +
121 + if not current_platform.has_device_capability(75):
122 + return False, "Humming only supported on SM75+"
123 +
124 + return True, None
125 +
126 + @classmethod
127 + def can_implement(
128 + cls, config: Int8ScaledMMLinearLayerConfig
129 + ) -> tuple[bool, str | None]:
130 + return True, None
131 +
132 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
133 + weight_name, weight_scale_name, *_ = self.layer_param_names
134 + name_map = {"weight": weight_name, "weight_scale": weight_scale_name}
135 + quant_config = {"quant_method": "humming", "dtype": "int8"}
136 + weight = getattr(layer, weight_name)
137 + weight.data = weight.data + 128
138 +
139 + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
140 + prepare_humming_layer(layer, quant_config)
141 +
142 + def apply_weights(
143 + self,
144 + layer: torch.nn.Module,
145 + x: torch.Tensor,
146 + bias: torch.Tensor | None = None,
147 + ) -> torch.Tensor:
148 + from vllm.utils.humming import HummingMethod
149 +
150 + flatten_inputs = x.view(-1, x.size(-1))
151 + output = HummingMethod.forward_layer(
152 + layer=layer,
153 + inputs=flatten_inputs,
154 + compute_config=layer.compute_config,
155 + )
156 + return output.view(*x.shape[:-1], output.size(-1))
vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+32 -10
@@ -38,15 +38,20 @@ from vllm.model_executor.layers.fused_moe.utils import (
38 )
39 from vllm.model_executor.layers.quantization.utils.quant_utils import (
40 QuantKey,
41 + kFp8Dynamic128Sym,
42 kFp8DynamicTokenSym,
43 kFp8Static128BlockSym,
44 kFp8StaticChannelSym,
45 + kFp8StaticTensorSym,
46 kInt4Static,
47 + kInt8DynamicTokenSym,
48 kInt8Static,
49 + kInt8StaticChannelSym,
50 kMxfp4Dynamic,
51 kMxfp4Static,
52 kMxfp8Dynamic,
53 kMxfp8Static,
54 + kNvfp4Dynamic,
55 kNvfp4Static,
56 )
57 from vllm.platforms import current_platform
@@ -61,9 +66,9 @@ if TYPE_CHECKING:
66 logger = init_logger(__name__)
67
68
64 -def get_humming_moe_gemm_type() -> str | None:
69 +def get_humming_moe_gemm_type() -> str:
70 env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
66 - gemm_type = None
71 + gemm_type = "indexed"
72 if env_gemm_type is not None:
73 env_gemm_type = env_gemm_type.lower()
74 if env_gemm_type == "indexed":
@@ -87,7 +92,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
92 num_dispatchers: int | None = None,
93 ):
94 self.layer = layer
90 - self.num_experts = self.layer.num_experts
95 + self.num_experts = self.layer.local_num_experts
96 self.global_num_experts = self.layer.global_num_experts
97 self.init_humming_moe()
98
@@ -186,6 +191,24 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
191 (kInt4Static, kFp8DynamicTokenSym),
192 (kInt8Static, None),
193 (kInt8Static, kFp8DynamicTokenSym),
194 + # Checkpoint-driven (weight, activation) pairs the dense/MoE oracles
195 + # pass. Humming defers input quant (see expects_unquantized_inputs),
196 + # so the activation key does not constrain support.
197 + # fp8 (compressed-tensors / native / modelopt)
198 + (kFp8StaticChannelSym, kFp8StaticTensorSym),
199 + (kFp8StaticChannelSym, kFp8Dynamic128Sym),
200 + (kFp8StaticTensorSym, None),
201 + (kFp8StaticTensorSym, kFp8DynamicTokenSym),
202 + (kFp8StaticTensorSym, kFp8StaticTensorSym),
203 + (kFp8StaticTensorSym, kFp8Dynamic128Sym),
204 + (kFp8Static128BlockSym, kFp8Dynamic128Sym),
205 + # int8 (compressed-tensors w8a8 / experts_int8)
206 + (kInt8StaticChannelSym, None),
207 + (kInt8StaticChannelSym, kInt8DynamicTokenSym),
208 + # nvfp4 (compressed-tensors / modelopt / quark)
209 + (kNvfp4Static, kNvfp4Dynamic),
210 + # mxfp8 (compressed-tensors / modelopt / online)
211 + (kMxfp8Static, kMxfp8Dynamic),
212 ]
213 return (weight_key, activation_key) in SUPPORTED_W_A
214
@@ -463,13 +486,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
486 assert hasattr(cls, "humming_gemm_type")
487 gemm_type = cls.humming_gemm_type().value.lower()
488 preferred_gemm_type = get_humming_moe_gemm_type()
466 - if preferred_gemm_type is not None:
467 - supported = preferred_gemm_type.lower() == gemm_type
468 - if not supported:
469 - reason = (
470 - f"preferred gemm type {preferred_gemm_type} != "
471 - f"supported gemm type {gemm_type}"
472 - )
489 + supported = preferred_gemm_type.lower() == gemm_type
490 + if not supported:
491 + reason = (
492 + f"preferred gemm type {preferred_gemm_type} != "
493 + f"supported gemm type {gemm_type}"
494 + )
495
496 return supported, reason
497
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+81
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from enum import Enum
4 +from typing import Any
5
6 import torch
7
@@ -45,6 +46,7 @@ class Fp8MoeBackend(Enum):
46 DEEPGEMM = "DEEPGEMM"
47 BATCHED_DEEPGEMM = "BATCHED_DEEPGEMM"
48 MARLIN = "MARLIN"
49 + HUMMING = "HUMMING"
50 TRITON = "TRITON"
51 BATCHED_TRITON = "BATCHED_TRITON"
52 AITER = "AITER"
@@ -83,6 +85,7 @@ def _get_priority_backends(
85 Fp8MoeBackend.VLLM_CUTLASS,
86 Fp8MoeBackend.TRITON,
87 Fp8MoeBackend.MARLIN,
88 + Fp8MoeBackend.HUMMING,
89 Fp8MoeBackend.BATCHED_DEEPGEMM,
90 Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
91 Fp8MoeBackend.BATCHED_TRITON,
@@ -162,6 +165,19 @@ def backend_to_kernel_cls(
165
166 return [BatchedDeepGemmExperts]
167
168 + elif backend == Fp8MoeBackend.HUMMING:
169 + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
170 + BatchedHummingGroupedExperts,
171 + HummingGroupedExperts,
172 + HummingIndexedExperts,
173 + )
174 +
175 + return [
176 + BatchedHummingGroupedExperts,
177 + HummingGroupedExperts,
178 + HummingIndexedExperts,
179 + ]
180 +
181 elif backend == Fp8MoeBackend.MARLIN:
182 from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
183 MarlinExperts,
@@ -240,6 +256,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
256 "flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
257 "flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS,
258 "marlin": Fp8MoeBackend.MARLIN,
259 + "humming": Fp8MoeBackend.HUMMING,
260 "aiter": Fp8MoeBackend.AITER,
261 "hpc": Fp8MoeBackend.HPC,
262 }
@@ -402,6 +419,41 @@ def select_fp8_moe_backend(
419 return Fp8MoeBackend.NONE, None
420
421
422 +def _humming_fp8_weight_schema(
423 + layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
424 +) -> dict[str, Any]:
425 + """Build the humming weight schema from the canonical on-device fp8/mxfp8
426 + tensors (scale dtype/shape, block size), not the producing quant method."""
427 + # mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
428 + # no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
429 + if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
430 + return {"quant_method": "modelopt", "quant_algo": "mxfp8"}
431 +
432 + if hasattr(layer, "w13_weight_scale_inv"):
433 + assert hasattr(layer, "weight_block_size")
434 + return {"quant_method": "fp8", "weight_block_size": layer.weight_block_size}
435 +
436 + # fp8 (e4m3): recover the strategy from the scale layout (block from
437 + # weight_block_size, else channel vs tensor by per-expert scale count).
438 + config: dict[str, Any] = {
439 + "quant_method": "compressed-tensors",
440 + "format": "float-quantized",
441 + "type": "float",
442 + "num_bits": 8,
443 + "symmetric": True,
444 + }
445 + weight_block_size = getattr(layer, "weight_block_size", None)
446 + num_experts, num_output = weight.shape[0], weight.shape[-2]
447 + if weight_block_size is not None:
448 + config["strategy"] = "block"
449 + config["block_structure"] = list(weight_block_size)
450 + elif weight_scale.numel() >= num_experts * num_output:
451 + config["strategy"] = "channel"
452 + else:
453 + config["strategy"] = "tensor"
454 + return config
455 +
456 +
457 def convert_to_fp8_moe_kernel_format(
458 fp8_backend: Fp8MoeBackend,
459 # TODO(bnell): replace layer with weight_block_size
@@ -429,6 +481,18 @@ def convert_to_fp8_moe_kernel_format(
481 w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
482 w13, w2, w13_scale, w2_scale
483 )
484 + elif fp8_backend == Fp8MoeBackend.HUMMING:
485 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
486 + convert_to_humming_moe_kernel_format,
487 + )
488 +
489 + convert_to_humming_moe_kernel_format(
490 + layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
491 + )
492 + w13 = layer.w13_weight
493 + w2 = layer.w2_weight
494 + w13_scale = layer.w13_weight_scale
495 + w2_scale = layer.w2_weight_scale
496 elif fp8_backend == Fp8MoeBackend.MARLIN:
497 weight_block_size = getattr(layer, "weight_block_size", None)
498 if weight_block_size == [1, 32]:
@@ -511,6 +575,7 @@ def make_fp8_moe_quant_config(
575 swiglu_limit: float | None = None,
576 gemm1_alpha: float | None = None,
577 gemm1_beta: float | None = None,
578 + layer: torch.nn.Module | None = None,
579 ) -> FusedMoEQuantConfig:
580 """
581 Create FusedMoEQuantConfig for the specified FP8 Backend.
@@ -537,6 +602,14 @@ def make_fp8_moe_quant_config(
602 gemm1_beta=gemm1_beta,
603 gemm1_clamp_limit=swiglu_limit,
604 )
605 + elif fp8_backend == Fp8MoeBackend.HUMMING:
606 + from vllm.model_executor.layers.fused_moe import RoutedExperts
607 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
608 + get_humming_moe_quant_config,
609 + )
610 +
611 + assert isinstance(layer, RoutedExperts)
612 + return get_humming_moe_quant_config(layer)
613
614 # Flashinfer CUTLASS or HPC per-tensor uses single dq scale
615 # (alpha = w_scale * a_scale) and inverse a2 scale.
@@ -600,6 +673,7 @@ def make_fp8_moe_kernel(
673 experts_cls: type[mk.FusedMoEExperts],
674 fp8_backend: Fp8MoeBackend,
675 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
676 + layer: torch.nn.Module | None = None,
677 ) -> mk.FusedMoEKernel:
678 # Create Prepare/Finalize.
679 prepare_finalize = maybe_make_prepare_finalize(
@@ -613,6 +687,11 @@ def make_fp8_moe_kernel(
687
688 logger.info_once("Using %s", prepare_finalize.__class__.__name__)
689
690 + extra_kwargs = {}
691 + if fp8_backend == Fp8MoeBackend.HUMMING:
692 + assert layer is not None
693 + extra_kwargs = {"layer": layer}
694 +
695 # Create Experts.
696 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
697 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -622,11 +701,13 @@ def make_fp8_moe_kernel(
701 quant_config=moe_quant_config,
702 max_num_tokens=max_num_tokens,
703 num_dispatchers=prepare_finalize.num_dispatchers(),
704 + **extra_kwargs,
705 )
706 else:
707 experts = experts_cls(
708 moe_config=moe_config,
709 quant_config=moe_quant_config,
710 + **extra_kwargs,
711 )
712
713 kernel = mk.FusedMoEKernel(
vllm/model_executor/layers/fused_moe/oracle/int8.py
+82 -1
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from enum import Enum
5 +from typing import Any
6
7 import torch
8
@@ -22,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
23 kInt8DynamicTokenSym,
24 kInt8StaticChannelSym,
25 )
26 +from vllm.model_executor.utils import replace_parameter
27 from vllm.platforms import current_platform
28
29 logger = init_logger(__name__)
@@ -29,6 +31,7 @@ logger = init_logger(__name__)
31
32 class Int8MoeBackend(Enum):
33 TRITON = "TRITON"
34 + HUMMING = "HUMMING"
35 CPU = "CPU"
36
37
@@ -40,6 +43,7 @@ def _get_priority_backends(
43 """
44 _AVAILABLE_BACKENDS = [
45 Int8MoeBackend.TRITON,
46 + Int8MoeBackend.HUMMING,
47 Int8MoeBackend.CPU,
48 ]
49
@@ -62,6 +66,19 @@ def backend_to_kernel_cls(
66
67 return [TritonExperts]
68
69 + elif backend == Int8MoeBackend.HUMMING:
70 + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
71 + BatchedHummingGroupedExperts,
72 + HummingGroupedExperts,
73 + HummingIndexedExperts,
74 + )
75 +
76 + return [
77 + BatchedHummingGroupedExperts,
78 + HummingGroupedExperts,
79 + HummingIndexedExperts,
80 + ]
81 +
82 elif backend == Int8MoeBackend.CPU:
83 from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
84 CPUExpertsInt8,
@@ -77,6 +94,7 @@ def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend:
94 """Map user's MoEBackend to Int8MoeBackend."""
95 mapping = {
96 "triton": Int8MoeBackend.TRITON,
97 + "humming": Int8MoeBackend.HUMMING,
98 }
99 if backend := mapping.get(runner_backend):
100 return backend
@@ -163,6 +181,7 @@ def select_int8_moe_backend(
181
182
183 def make_int8_moe_quant_config(
184 + int8_backend: Int8MoeBackend,
185 w1_scale: torch.Tensor,
186 w2_scale: torch.Tensor,
187 a1_scale: torch.Tensor | None = None,
@@ -170,11 +189,21 @@ def make_int8_moe_quant_config(
189 w1_bias: torch.Tensor | None = None,
190 w2_bias: torch.Tensor | None = None,
191 per_act_token_quant: bool = False,
192 + layer: torch.nn.Module | None = None,
193 ) -> FusedMoEQuantConfig:
194 assert (a1_scale is None and a2_scale is None) or (
195 a1_scale is not None and a2_scale is not None
196 ), "a1_scale and a2_scale must both be provided or both be None"
197
198 + if int8_backend == Int8MoeBackend.HUMMING:
199 + from vllm.model_executor.layers.fused_moe import RoutedExperts
200 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
201 + get_humming_moe_quant_config,
202 + )
203 +
204 + assert isinstance(layer, RoutedExperts)
205 + return get_humming_moe_quant_config(layer)
206 +
207 if a1_scale is None or a2_scale is None:
208 return int8_w8a16_moe_quant_config(
209 w1_scale=w1_scale,
@@ -196,13 +225,56 @@ def make_int8_moe_quant_config(
225 )
226
227
228 +def _humming_int8_weight_schema(
229 + weight: torch.Tensor, weight_scale: torch.Tensor
230 +) -> dict[str, Any]:
231 + """Build the humming compressed-tensors int8 schema from the canonical
232 + on-device tensors; humming does the signed-int8 -> native conversion."""
233 + config: dict[str, Any] = {
234 + "quant_method": "compressed-tensors",
235 + "format": "int-quantized",
236 + "type": "int",
237 + "num_bits": 8,
238 + "symmetric": True,
239 + "strategy": "channel",
240 + }
241 + num_experts, num_output = weight.shape[0], weight.shape[-2]
242 + if weight_scale.numel() < num_experts * num_output:
243 + config["strategy"] = "tensor"
244 + return config
245 +
246 +
247 def convert_to_int8_moe_kernel_format(
248 int8_backend: Int8MoeBackend,
249 w13: torch.Tensor,
250 w2: torch.Tensor,
251 + layer: torch.nn.Module | None = None,
252 + w13_scale: torch.Tensor | None = None,
253 ) -> tuple[torch.Tensor, torch.Tensor]:
254 """Convert INT8 MoE weights to backend-specific kernel format."""
205 - if int8_backend == Int8MoeBackend.CPU:
255 + if int8_backend == Int8MoeBackend.HUMMING:
256 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
257 + convert_to_humming_moe_kernel_format,
258 + )
259 +
260 + assert layer is not None
261 + # Humming reads canonical CT scales (w*_weight_scale) from the layer.
262 + # Online int8 produces per-channel (E, N) w*_scale; expose them as the
263 + # (E, N, 1) w*_weight_scale humming's loader expects.
264 + for sub in ("w13", "w2"):
265 + if hasattr(layer, f"{sub}_weight_scale"):
266 + continue
267 + scale = getattr(layer, f"{sub}_scale").data
268 + if scale.dim() < 3:
269 + scale = scale.unsqueeze(-1)
270 + replace_parameter(layer, f"{sub}_weight_scale", scale)
271 + delattr(layer, f"{sub}_scale")
272 + convert_to_humming_moe_kernel_format(
273 + layer,
274 + quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale),
275 + )
276 + return layer.w13_weight, layer.w2_weight
277 + elif int8_backend == Int8MoeBackend.CPU:
278 from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
279 prepare_int8_moe_layer_for_cpu,
280 )
@@ -215,10 +287,12 @@ def convert_to_int8_moe_kernel_format(
287
288
289 def make_int8_moe_kernel(
290 + int8_backend: Int8MoeBackend,
291 moe_quant_config: FusedMoEQuantConfig,
292 moe_config: FusedMoEConfig,
293 experts_cls: type[mk.FusedMoEExperts],
294 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
295 + layer: torch.nn.Module | None = None,
296 ) -> mk.FusedMoEKernel:
297 # Create Prepare/Finalize.
298 prepare_finalize = maybe_make_prepare_finalize(
@@ -232,6 +306,11 @@ def make_int8_moe_kernel(
306
307 logger.info_once("Using %s", prepare_finalize.__class__.__name__)
308
309 + extra_kwargs = {}
310 + if int8_backend == Int8MoeBackend.HUMMING:
311 + assert layer is not None
312 + extra_kwargs = {"layer": layer}
313 +
314 # Create Experts.
315 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
316 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -241,11 +320,13 @@ def make_int8_moe_kernel(
320 quant_config=moe_quant_config,
321 max_num_tokens=max_num_tokens,
322 num_dispatchers=prepare_finalize.num_dispatchers(),
323 + **extra_kwargs,
324 )
325 else:
326 experts = experts_cls(
327 moe_config=moe_config,
328 quant_config=moe_quant_config,
329 + **extra_kwargs,
330 )
331
332 kernel = mk.FusedMoEKernel(
vllm/model_executor/layers/fused_moe/oracle/int_wna16.py
+111 -21
@@ -11,6 +11,7 @@ from compressed_tensors.quantization import (
11
12 import vllm._custom_ops as ops
13 import vllm.model_executor.layers.fused_moe.modular_kernel as mk
14 +from vllm.config.kernel import MoEBackend
15 from vllm.logger import init_logger
16 from vllm.model_executor.layers.fused_moe.config import (
17 FusedMoEConfig,
@@ -46,6 +47,7 @@ logger = init_logger(__name__)
47 class WNA16MoEBackend(Enum):
48 MARLIN = "MARLIN"
49 BATCHED_MARLIN = "BATCHED_MARLIN"
50 + HUMMING = "HUMMING"
51 CPU = "CPU"
52 FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
53 XPU = "XPU"
@@ -55,7 +57,19 @@ def backend_to_kernel_cls(
57 backend: WNA16MoEBackend,
58 ) -> list[type[mk.FusedMoEExperts]]:
59 """Return the experts class for the given backend, or None for NONE."""
58 - if backend == WNA16MoEBackend.MARLIN:
60 + if backend == WNA16MoEBackend.HUMMING:
61 + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
62 + BatchedHummingGroupedExperts,
63 + HummingGroupedExperts,
64 + HummingIndexedExperts,
65 + )
66 +
67 + return [
68 + BatchedHummingGroupedExperts,
69 + HummingGroupedExperts,
70 + HummingIndexedExperts,
71 + ]
72 + elif backend == WNA16MoEBackend.MARLIN:
73 return [MarlinExperts]
74 elif backend == WNA16MoEBackend.BATCHED_MARLIN:
75 return [BatchedMarlinExperts]
@@ -90,10 +104,26 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
104 WNA16MoEBackend.FLASHINFER_TRTLLM,
105 WNA16MoEBackend.MARLIN,
106 WNA16MoEBackend.BATCHED_MARLIN,
107 + WNA16MoEBackend.HUMMING,
108 ]
109 return _AVAILABLE_BACKENDS
110
111
112 +def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
113 + """Map user's MoEBackend to WNA16MoEBackend."""
114 + mapping = {
115 + "marlin": WNA16MoEBackend.MARLIN,
116 + "humming": WNA16MoEBackend.HUMMING,
117 + "flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
118 + }
119 + if backend := mapping.get(runner_backend):
120 + return backend
121 + raise ValueError(
122 + f"moe_backend='{runner_backend}' is not supported for WNA16 MoE. "
123 + f"Expected one of {list(mapping.keys())}."
124 + )
125 +
126 +
127 def select_wna16_moe_backend(
128 config: FusedMoEConfig,
129 weight_key: QuantKey,
@@ -146,6 +176,14 @@ def select_wna16_moe_backend(
176 return backend, k_cls
177 raise ValueError(_make_log_unsupported(backend, reason))
178
179 + # Handle explicit moe_backend from user.
180 + runner_backend = config.moe_backend
181 + if runner_backend != "auto":
182 + requested_backend = map_wna16_backend(runner_backend)
183 + return _return_or_raise(
184 + requested_backend, config, weight_key, None, activation_format
185 + )
186 +
187 # Select kernels in order of backend.
188 AVAILABLE_BACKENDS = _get_priority_backends()
189
@@ -210,6 +248,8 @@ def make_wna16_moe_kernel(
248 moe_quant_config: FusedMoEQuantConfig,
249 moe_config: FusedMoEConfig,
250 experts_cls: type[mk.FusedMoEExperts],
251 + backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
252 + layer: torch.nn.Module | None = None,
253 is_k_full: bool = False,
254 w13_g_idx: torch.Tensor | None = None,
255 w2_g_idx: torch.Tensor | None = None,
@@ -228,14 +268,18 @@ def make_wna16_moe_kernel(
268 )
269
270 # Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
231 - # BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
232 - assert experts_cls in (
271 + # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming
272 + # grouped/indexed experts.
273 + allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
274 MarlinExperts,
275 BatchedMarlinExperts,
276 TrtLlmMxint4ExpertsMonolithic,
277 XPUExpertsWNA16,
278 CPUExpertsInt4,
279 )
280 + if backend == WNA16MoEBackend.HUMMING:
281 + allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING))
282 + assert experts_cls in allowed_experts
283
284 is_monolithic = experts_cls.is_monolithic()
285
@@ -251,7 +295,10 @@ def make_wna16_moe_kernel(
295 logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local")
296
297 extra_args: dict[str, Any] = {}
254 - if issubclass(experts_cls, MarlinExpertsBase):
298 + if backend == WNA16MoEBackend.HUMMING:
299 + assert layer is not None
300 + extra_args = {"layer": layer}
301 + elif issubclass(experts_cls, MarlinExpertsBase):
302 extra_args = {
303 "w13_g_idx": w13_g_idx,
304 "w2_g_idx": w2_g_idx,
@@ -941,6 +988,35 @@ def _process_weights_xpu(
988 )
989
990
991 +def _humming_wna16_weight_schema(
992 + quant_config: QuantizationConfig | QuantizationArgs | None,
993 +) -> dict[str, Any]:
994 + """Humming weight schema for a WNA16 checkpoint, derived from the quant
995 + config rather than the running kernel."""
996 + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
997 + from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
998 +
999 + if isinstance(quant_config, AutoAWQConfig):
1000 + return {
1001 + "quant_method": "awq",
1002 + "bits": quant_config.weight_bits,
1003 + "group_size": quant_config.group_size,
1004 + "zero_point": quant_config.zero_point,
1005 + }
1006 + if isinstance(quant_config, AutoGPTQConfig):
1007 + return {
1008 + "quant_method": "gptq",
1009 + "bits": quant_config.weight_bits,
1010 + "group_size": quant_config.group_size,
1011 + "desc_act": quant_config.desc_act,
1012 + "sym": quant_config.is_sym,
1013 + }
1014 + raise TypeError(
1015 + "Humming WNA16 MoE requires AutoAWQConfig or AutoGPTQConfig, "
1016 + f"got {type(quant_config).__name__}."
1017 + )
1018 +
1019 +
1020 def convert_to_wna16_moe_kernel_format(
1021 backend: WNA16MoEBackend,
1022 layer: torch.nn.Module,
@@ -956,26 +1032,30 @@ def convert_to_wna16_moe_kernel_format(
1032 w2_qzeros: torch.Tensor | None = None,
1033 w13_bias: torch.Tensor | None = None,
1034 w2_bias: torch.Tensor | None = None,
959 -) -> tuple[
960 - torch.Tensor, # w13_qweight
961 - torch.Tensor, # w2_qweight
962 - torch.Tensor, # w13_scales
963 - torch.Tensor, # w2_scales
964 - torch.Tensor | None, # w13_g_idx
965 - torch.Tensor | None, # w2_g_idx
966 - torch.Tensor | None, # w13_g_idx_sort_indices
967 - torch.Tensor | None, # w2_g_idx_sort_indices
968 - torch.Tensor | None, # w13_qzeros
969 - torch.Tensor | None, # w2_qzeros
970 - torch.Tensor | None, # w13_input_global_scale
971 - torch.Tensor | None, # w2_input_global_scale
972 - torch.Tensor | None, # w13_bias
973 - torch.Tensor | None, # w2_bias
974 -]:
1035 +) -> (
1036 + tuple[
1037 + torch.Tensor, # w13_qweight
1038 + torch.Tensor, # w2_qweight
1039 + torch.Tensor, # w13_scales
1040 + torch.Tensor, # w2_scales
1041 + torch.Tensor | None, # w13_g_idx
1042 + torch.Tensor | None, # w2_g_idx
1043 + torch.Tensor | None, # w13_g_idx_sort_indices
1044 + torch.Tensor | None, # w2_g_idx_sort_indices
1045 + torch.Tensor | None, # w13_qzeros
1046 + torch.Tensor | None, # w2_qzeros
1047 + torch.Tensor | None, # w13_input_global_scale
1048 + torch.Tensor | None, # w2_input_global_scale
1049 + torch.Tensor | None, # w13_bias
1050 + torch.Tensor | None, # w2_bias
1051 + ]
1052 + | None
1053 +):
1054 """Dispatch weight post-processing to the appropriate per-backend handler.
1055
1056 To add a new backend, implement a ``_process_weights_<name>`` helper and
978 - add a branch here.
1057 + add a branch here. Backends that rewrite the layer's parameters in place
1058 + (e.g. Humming) return ``None``; the caller then skips the param scatter.
1059
1060 Args:
1061 backend: the selected ``WNA16MoEBackend``.
@@ -983,6 +1063,16 @@ def convert_to_wna16_moe_kernel_format(
1063 quant_config: the ``QuantizationConfig`` for this layer.
1064 input_dtype: optional activation dtype, usually should be 16 bit.
1065 """
1066 + if backend == WNA16MoEBackend.HUMMING:
1067 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
1068 + convert_to_humming_moe_kernel_format,
1069 + )
1070 +
1071 + convert_to_humming_moe_kernel_format(
1072 + layer, quant_config=_humming_wna16_weight_schema(quant_config)
1073 + )
1074 + return None
1075 +
1076 if backend in (
1077 WNA16MoEBackend.MARLIN,
1078 WNA16MoEBackend.BATCHED_MARLIN,
vllm/model_executor/layers/fused_moe/oracle/mxfp8.py
+2
@@ -25,6 +25,7 @@ _SUPPORTED_BACKENDS = (
25 # is_supported_config passes (gfx950 + flydsl installed + not EP). On other
26 # devices / no flydsl / EP it is skipped and native is used.
27 Fp8MoeBackend.AITER_MXFP8,
28 + Fp8MoeBackend.HUMMING,
29 )
30
31 _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
@@ -34,6 +35,7 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
35 "xpu": Fp8MoeBackend.XPU,
36 "aiter": Fp8MoeBackend.AITER_MXFP8,
37 "triton": Fp8MoeBackend.TRITON_MXFP8,
38 + "humming": Fp8MoeBackend.HUMMING,
39 }
40
41
vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
+69 -1
@@ -43,6 +43,7 @@ class NvFp4MoeBackend(Enum):
43 FLASHINFER_B12X = "FLASHINFER_B12X"
44 VLLM_CUTLASS = "VLLM_CUTLASS"
45 MARLIN = "MARLIN"
46 + HUMMING = "HUMMING"
47 EMULATION = "EMULATION"
48
49
@@ -119,6 +120,18 @@ def backend_to_kernel_cls(
120 )
121
122 return [MarlinExperts]
123 + elif backend == NvFp4MoeBackend.HUMMING:
124 + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
125 + BatchedHummingGroupedExperts,
126 + HummingGroupedExperts,
127 + HummingIndexedExperts,
128 + )
129 +
130 + return [
131 + BatchedHummingGroupedExperts,
132 + HummingGroupedExperts,
133 + HummingIndexedExperts,
134 + ]
135 elif backend == NvFp4MoeBackend.EMULATION:
136 from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import (
137 Nvfp4QuantizationEmulationTritonExperts,
@@ -138,6 +151,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
151 "flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
152 "flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
153 "marlin": NvFp4MoeBackend.MARLIN,
154 + "humming": NvFp4MoeBackend.HUMMING,
155 "emulation": NvFp4MoeBackend.EMULATION,
156 }
157 if backend := mapping.get(runner_backend):
@@ -169,6 +183,7 @@ def select_nvfp4_moe_backend(
183 NvFp4MoeBackend.FLASHINFER_CUTLASS,
184 NvFp4MoeBackend.VLLM_CUTLASS,
185 NvFp4MoeBackend.MARLIN,
186 + NvFp4MoeBackend.HUMMING,
187 NvFp4MoeBackend.EMULATION,
188 ]
189
@@ -346,6 +361,41 @@ def convert_to_nvfp4_moe_kernel_format(
361 a2_scale=a2_scale,
362 is_act_and_mul=is_act_and_mul,
363 )
364 + elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
365 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
366 + convert_to_humming_moe_kernel_format,
367 + )
368 +
369 + # Discriminate the source checkpoint layout by its global-scale param:
370 + # compressed-tensors uses *_weight_global_scale, modelopt *_weight_scale_2.
371 + # The logical schema is identical (nvfp4 group-16); only the on-layer
372 + # param names differ. TODO: normalize both methods to a single canonical
373 + # layout upstream so the oracle needs neither the probe nor the re-alias.
374 + if hasattr(layer, "w13_weight_global_scale"):
375 + quant_config = {
376 + "quant_method": "compressed-tensors",
377 + "format": "nvfp4-pack-quantized",
378 + "type": "float",
379 + "num_bits": 4,
380 + "strategy": "group",
381 + "group_size": 16,
382 + }
383 + # CT pack-quantized reads `weight_packed`; the method renamed it to
384 + # `weight`. Re-alias (convert replaces all params anyway).
385 + layer.w13_weight_packed = layer.w13_weight
386 + layer.w2_weight_packed = layer.w2_weight
387 + else:
388 + quant_config = {"quant_method": "modelopt", "quant_algo": "nvfp4"}
389 +
390 + convert_to_humming_moe_kernel_format(layer, quant_config=quant_config)
391 + a13_scale = None
392 + a2_scale = None
393 + w13 = layer.w13_weight
394 + w13_scale = layer.w13_weight_scale
395 + w13_scale_2 = getattr(layer, "w13_global_scale", None)
396 + w2 = layer.w2_weight
397 + w2_scale = layer.w2_weight_scale
398 + w2_scale_2 = getattr(layer, "w2_global_scale", None)
399 elif nvfp4_backend == NvFp4MoeBackend.MARLIN:
400 a13_scale = None
401 a2_scale = None
@@ -418,8 +468,17 @@ def make_nvfp4_moe_quant_config(
468 a13_scale: torch.Tensor,
469 a2_scale: torch.Tensor,
470 swiglu_limit: float | None = None,
471 + layer: torch.nn.Module | None = None,
472 ) -> FusedMoEQuantConfig:
422 - if backend == NvFp4MoeBackend.MARLIN:
473 + if backend == NvFp4MoeBackend.HUMMING:
474 + from vllm.model_executor.layers.fused_moe import RoutedExperts
475 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
476 + get_humming_moe_quant_config,
477 + )
478 +
479 + assert isinstance(layer, RoutedExperts)
480 + return get_humming_moe_quant_config(layer)
481 + elif backend == NvFp4MoeBackend.MARLIN:
482 return nvfp4_w4a16_moe_quant_config(
483 g1_alphas=w13_scale_2,
484 g2_alphas=w2_scale_2,
@@ -467,7 +526,9 @@ def make_nvfp4_moe_kernel(
526 moe_quant_config: FusedMoEQuantConfig,
527 moe_config: FusedMoEConfig,
528 experts_cls: type[mk.FusedMoEExperts],
529 + backend: NvFp4MoeBackend,
530 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
531 + layer: torch.nn.Module | None = None,
532 ) -> mk.FusedMoEKernel:
533 # Create Prepare/Finalize.
534 prepare_finalize = maybe_make_prepare_finalize(
@@ -481,6 +542,11 @@ def make_nvfp4_moe_kernel(
542
543 logger.info_once("Using %s", prepare_finalize.__class__.__name__)
544
545 + extra_kwargs = {}
546 + if backend == NvFp4MoeBackend.HUMMING:
547 + assert layer is not None
548 + extra_kwargs = {"layer": layer}
549 +
550 # Create Experts.
551 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
552 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -490,11 +556,13 @@ def make_nvfp4_moe_kernel(
556 quant_config=moe_quant_config,
557 max_num_tokens=max_num_tokens,
558 num_dispatchers=prepare_finalize.num_dispatchers(),
559 + **extra_kwargs,
560 )
561 else:
562 experts = experts_cls(
563 moe_config=moe_config,
564 quant_config=moe_quant_config,
565 + **extra_kwargs,
566 )
567
568 kernel = mk.FusedMoEKernel(
vllm/model_executor/layers/fused_moe/oracle/unquantized.py
+3 -1
@@ -232,7 +232,9 @@ def select_unquantized_moe_backend(
232 raise ValueError(_make_log_unsupported(backend, reason))
233
234 runner_backend = moe_config.moe_backend
235 - if runner_backend != "auto":
235 + # 'humming' is quantization-only; an unquantized layer (e.g. excluded via
236 + # modules_to_not_convert) falls through to auto instead of erroring.
237 + if runner_backend not in ["auto", "humming"]:
238 requested_backend = map_unquantized_backend(runner_backend)
239 if (
240 activation_format == mk.FusedMoEActivationFormat.BatchedExperts
vllm/model_executor/layers/quantization/auto_awq.py
+34 -18
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe import (
26 UnquantizedFusedMoEMethod,
27 )
28 from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
29 + WNA16MoEBackend,
30 convert_to_wna16_moe_kernel_format,
31 make_wna16_moe_kernel,
32 make_wna16_moe_quant_config,
@@ -663,6 +664,26 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
664 layer.workspace = marlin_make_workspace_new(device, 4)
665
666 def process_weights_after_loading(self, layer: RoutedExperts) -> None:
667 + converted = convert_to_wna16_moe_kernel_format(
668 + backend=self.wna16_moe_backend,
669 + layer=layer,
670 + quant_config=self.quant_config,
671 + input_dtype=self.input_dtype,
672 + w13=layer.w13_qweight,
673 + w2=layer.w2_qweight,
674 + w13_scale=layer.w13_scales,
675 + w2_scale=layer.w2_scales,
676 + w13_qzeros=layer.w13_qzeros,
677 + w2_qzeros=layer.w2_qzeros,
678 + w13_bias=getattr(layer, "w13_bias", None),
679 + w2_bias=getattr(layer, "w2_bias", None),
680 + )
681 +
682 + if converted is None:
683 + # Backend rewrote the layer's params in place (e.g. Humming).
684 + self._setup_kernel(layer)
685 + return
686 +
687 (
688 w13,
689 w2,
@@ -678,20 +699,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
699 w2_input_global_scale,
700 w13_bias,
701 w2_bias,
681 - ) = convert_to_wna16_moe_kernel_format(
682 - backend=self.wna16_moe_backend,
683 - layer=layer,
684 - quant_config=self.quant_config,
685 - input_dtype=self.input_dtype,
686 - w13=layer.w13_qweight,
687 - w2=layer.w2_qweight,
688 - w13_scale=layer.w13_scales,
689 - w2_scale=layer.w2_scales,
690 - w13_qzeros=layer.w13_qzeros,
691 - w2_qzeros=layer.w2_qzeros,
692 - w13_bias=getattr(layer, "w13_bias", None),
693 - w2_bias=getattr(layer, "w2_bias", None),
694 - )
702 + ) = converted
703
704 replace_parameter(layer, "w13_qweight", w13)
705 replace_parameter(layer, "w2_qweight", w2)
@@ -734,6 +742,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
742 moe_quant_config=self.moe_quant_config,
743 moe_config=self.moe,
744 experts_cls=self.experts_cls,
745 + backend=self.wna16_moe_backend,
746 + layer=layer,
747 is_k_full=self.is_k_full,
748 w13_g_idx=getattr(layer, "w13_g_idx", None),
749 w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -743,6 +753,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
753 )
754
755 def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
756 + if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
757 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
758 + get_humming_moe_quant_config,
759 + )
760 +
761 + return get_humming_moe_quant_config(layer)
762 return make_wna16_moe_quant_config(
763 w1_scale=layer.w13_scales,
764 w2_scale=layer.w2_scales,
@@ -783,8 +799,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
799 assert self.moe_kernel is not None
800 return self.moe_kernel.apply(
801 hidden_states=x,
786 - w1=layer.w13_qweight,
787 - w2=layer.w2_qweight,
802 + w1=layer.w13_weight,
803 + w2=layer.w2_weight,
804 topk_weights=topk_weights,
805 topk_ids=topk_ids,
806 activation=layer.activation,
@@ -806,8 +822,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
822 assert self.moe_kernel is not None
823 return self.moe_kernel.apply_monolithic(
824 hidden_states=x,
809 - w1=layer.w13_qweight,
810 - w2=layer.w2_qweight,
825 + w1=layer.w13_weight,
826 + w2=layer.w2_weight,
827 router_logits=router_logits,
828 activation=layer.activation,
829 global_num_experts=layer.global_num_experts,
vllm/model_executor/layers/quantization/auto_gptq.py
+40 -20
@@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
658 "W8A8-INT8 is not supported by marlin kernel."
659 )
660
661 + converted = convert_to_wna16_moe_kernel_format(
662 + backend=self.wna16_moe_backend,
663 + layer=layer,
664 + quant_config=self.quant_config,
665 + input_dtype=self.input_dtype,
666 + w13=layer.w13_qweight,
667 + w2=layer.w2_qweight,
668 + w13_scale=layer.w13_scales,
669 + w2_scale=layer.w2_scales,
670 + w13_g_idx=layer.w13_g_idx,
671 + w2_g_idx=layer.w2_g_idx,
672 + w13_bias=getattr(layer, "w13_bias", None),
673 + w2_bias=getattr(layer, "w2_bias", None),
674 + )
675 +
676 + if converted is None:
677 + # Backend rewrote the layer's params in place (e.g. Humming).
678 + self._setup_kernel(layer)
679 + return
680 +
681 (
682 w13,
683 w2,
@@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
693 w2_input_global_scale,
694 w13_bias,
695 w2_bias,
676 - ) = convert_to_wna16_moe_kernel_format(
677 - backend=self.wna16_moe_backend,
678 - layer=layer,
679 - quant_config=self.quant_config,
680 - input_dtype=self.input_dtype,
681 - w13=layer.w13_qweight,
682 - w2=layer.w2_qweight,
683 - w13_scale=layer.w13_scales,
684 - w2_scale=layer.w2_scales,
685 - w13_g_idx=layer.w13_g_idx,
686 - w2_g_idx=layer.w2_g_idx,
687 - w13_bias=getattr(layer, "w13_bias", None),
688 - w2_bias=getattr(layer, "w2_bias", None),
689 - )
696 + ) = converted
697
698 replace_parameter(layer, "w13_qweight", w13)
699 replace_parameter(layer, "w2_qweight", w2)
@@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
740 "w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
741 )
742
743 + # The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
744 + layer.w13_weight = layer.w13_qweight
745 + layer.w2_weight = layer.w2_qweight
746 +
747 self._setup_kernel(layer)
748
749 def _setup_kernel(self, layer: RoutedExperts) -> None:
@@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
754 moe_quant_config=self.moe_quant_config,
755 moe_config=self.moe,
756 experts_cls=self.experts_cls,
757 + backend=self.wna16_moe_backend,
758 + layer=layer,
759 is_k_full=self.is_k_full,
747 - w13_g_idx=layer.w13_g_idx,
748 - w2_g_idx=layer.w2_g_idx,
760 + w13_g_idx=getattr(layer, "w13_g_idx", None),
761 + w2_g_idx=getattr(layer, "w2_g_idx", None),
762 w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
763 w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
764 routing_tables=layer._expert_routing_tables(),
765 )
766
767 def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
768 + if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
769 + from vllm.model_executor.layers.quantization.utils.humming_utils import (
770 + get_humming_moe_quant_config,
771 + )
772 +
773 + return get_humming_moe_quant_config(layer)
774 +
775 from vllm.model_executor.layers.fused_moe.config import (
776 gptq_marlin_moe_quant_config,
777 )
@@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
815 assert self.moe_kernel is not None
816 return self.moe_kernel.apply(
817 hidden_states=x,
798 - w1=layer.w13_qweight,
799 - w2=layer.w2_qweight,
818 + w1=layer.w13_weight,
819 + w2=layer.w2_weight,
820 topk_weights=topk_weights,
821 topk_ids=topk_ids,
822 activation=layer.activation,
@@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
838 assert self.moe_kernel is not None
839 return self.moe_kernel.apply_monolithic(
840 hidden_states=x,
821 - w1=layer.w13_qweight,
822 - w2=layer.w2_qweight,
841 + w1=layer.w13_weight,
842 + w2=layer.w2_weight,
843 router_logits=router_logits,
844 activation=layer.activation,
845 global_num_experts=layer.global_num_experts,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py
+3
@@ -236,7 +236,9 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
236 moe_quant_config=self.moe_quant_config,
237 moe_config=self.moe,
238 experts_cls=self.experts_cls,
239 + backend=self.nvfp4_backend,
240 routing_tables=layer._expert_routing_tables(),
241 + layer=layer,
242 )
243 self.moe_kernel.fused_experts.process_weights_after_loading(layer)
244
@@ -259,6 +261,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
261 a13_scale=layer.w13_input_scale,
262 a2_scale=layer.w2_input_scale,
263 swiglu_limit=getattr(layer, "swiglu_limit", None),
264 + layer=layer,
265 )
266
267 def apply_monolithic(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py
+4 -2
@@ -338,6 +338,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
338 fp8_backend=self.fp8_backend,
339 experts_cls=self.experts_cls,
340 routing_tables=layer._expert_routing_tables(),
341 + layer=layer,
342 )
343
344 def maybe_make_prepare_finalize(
@@ -355,12 +356,13 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
356 fp8_backend=self.fp8_backend,
357 w1_scale=layer.w13_weight_scale,
358 w2_scale=layer.w2_weight_scale,
358 - a1_scale=layer.w13_input_scale,
359 - a2_scale=layer.w2_input_scale,
359 + a1_scale=getattr(layer, "w13_input_scale", None),
360 + a2_scale=getattr(layer, "w2_input_scale", None),
361 per_act_token_quant=is_per_token,
362 per_out_ch_quant=is_per_token,
363 block_shape=self.weight_block_size,
364 swiglu_limit=getattr(layer, "swiglu_limit", None),
365 + layer=layer,
366 )
367
368 def apply_monolithic(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py
+6
@@ -146,6 +146,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
146 int8_backend=self.int8_backend,
147 w13=layer.w13_weight,
148 w2=layer.w2_weight,
149 + layer=layer,
150 + w13_scale=layer.w13_weight_scale,
151 )
152 replace_parameter(layer, "w13_weight", w13)
153 replace_parameter(layer, "w2_weight", w2)
@@ -153,10 +155,12 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
155 self.moe_quant_config = self.get_fused_moe_quant_config(layer)
156 assert self.experts_cls is not None
157 self.moe_kernel = make_int8_moe_kernel(
158 + int8_backend=self.int8_backend,
159 moe_quant_config=self.moe_quant_config,
160 moe_config=self.moe,
161 experts_cls=self.experts_cls,
162 routing_tables=layer._expert_routing_tables(),
163 + layer=layer,
164 )
165
166 def maybe_make_prepare_finalize(
@@ -170,11 +174,13 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
174
175 def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
176 return make_int8_moe_quant_config(
177 + int8_backend=self.int8_backend,
178 w1_scale=layer.w13_weight_scale,
179 w2_scale=layer.w2_weight_scale,
180 a1_scale=layer.w13_input_scale,
181 a2_scale=layer.w2_input_scale,
182 per_act_token_quant=True,
183 + layer=layer,
184 )
185
186 def apply(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py
+2
@@ -140,6 +140,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
140 fp8_backend=self.fp8_backend,
141 experts_cls=self.experts_cls,
142 routing_tables=layer._expert_routing_tables(),
143 + layer=layer,
144 )
145
146 def get_fused_moe_quant_config(
@@ -155,6 +156,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
156 swiglu_limit=getattr(layer, "swiglu_limit", None),
157 gemm1_alpha=getattr(layer, "swiglu_alpha", None),
158 gemm1_beta=getattr(layer, "swiglu_beta", None),
159 + layer=layer,
160 )
161
162 def maybe_make_prepare_finalize(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py
+22 -14
@@ -416,6 +416,27 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
416 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
417 # Process weights using the shared oracle infrastructure
418 is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
419 + converted = convert_to_wna16_moe_kernel_format(
420 + backend=self.wna16_backend,
421 + layer=layer,
422 + quant_config=self.weight_quant,
423 + input_dtype=self.marlin_input_dtype,
424 + w13=layer.w13_weight_packed,
425 + w2=layer.w2_weight_packed,
426 + w13_scale=layer.w13_weight_scale,
427 + w2_scale=layer.w2_weight_scale,
428 + w13_g_idx=layer.w13_weight_g_idx,
429 + w2_g_idx=layer.w2_weight_g_idx,
430 + w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
431 + w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
432 + )
433 + if converted is None:
434 + # In-place backends (e.g. Humming) are not wired through this
435 + # marlin-only method; fail clearly rather than unpacking None.
436 + raise NotImplementedError(
437 + f"{type(self).__name__} does not support the "
438 + f"{self.wna16_backend.value} MoE backend."
439 + )
440 (
441 w13_qweight,
442 w2_qweight,
@@ -431,20 +452,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
452 w2_input_global_scale,
453 _, # w13_bias
454 _, # w2_bias
434 - ) = convert_to_wna16_moe_kernel_format(
435 - backend=self.wna16_backend,
436 - layer=layer,
437 - quant_config=self.weight_quant,
438 - input_dtype=self.marlin_input_dtype,
439 - w13=layer.w13_weight_packed,
440 - w2=layer.w2_weight_packed,
441 - w13_scale=layer.w13_weight_scale,
442 - w2_scale=layer.w2_weight_scale,
443 - w13_g_idx=layer.w13_weight_g_idx,
444 - w2_g_idx=layer.w2_weight_g_idx,
445 - w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
446 - w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
447 - )
455 + ) = converted
456
457 # Replace common parameters
458 replace_parameter(layer, "w13_weight_packed", w13_qweight)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+8
@@ -137,8 +137,16 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
137 "weight_scale",
138 convert_to_channelwise(layer.weight_scale, layer.logical_widths),
139 )
140 + self.strategy = QuantizationStrategy.CHANNEL
141 + self.weight_quant_key = STRATEGY_TO_WEIGHT_QUANT_KEY[self.strategy]
142 + self.linear_kernel.config.weight_quant_key = self.weight_quant_key
143 +
144 # Canonicalize to (K, N) for the kernel.
145 replace_parameter(layer, "weight", layer.weight.t())
146 + # Preserve the dim tags dropped by the transpose so layout-aware
147 + # kernels see (K, N).
148 + layer.weight.input_dim = 0
149 + layer.weight.output_dim = 1
150
151 self.linear_kernel.process_weights_after_loading(layer)
152
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+4
@@ -181,6 +181,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
181
182 # required by torch.compile to be torch.nn.Parameter
183 layer.weight = Parameter(weight.data, requires_grad=False)
184 + # Preserve the dim tags dropped by the transpose so layout-aware
185 + # kernels (humming) see (K, N) instead of assuming (N, K).
186 + layer.weight.input_dim = 0
187 + layer.weight.output_dim = 1
188 layer.weight_scale = Parameter(weight_scale.data, requires_grad=False)
189 if input_scale is not None:
190 layer.input_scale = Parameter(input_scale.data, requires_grad=False)
vllm/model_executor/layers/quantization/fp8.py
+2
@@ -714,6 +714,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
714 fp8_backend=self.fp8_backend,
715 experts_cls=self.experts_cls,
716 routing_tables=layer._expert_routing_tables(),
717 + layer=layer,
718 )
719
720 def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -786,6 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
787 swiglu_limit=getattr(layer, "swiglu_limit", None),
788 gemm1_alpha=getattr(layer, "swiglu_alpha", None),
789 gemm1_beta=getattr(layer, "swiglu_beta", None),
790 + layer=layer,
791 )
792
793 # Inject biases into the quant config if the model has them
vllm/model_executor/layers/quantization/modelopt.py
+7
@@ -905,6 +905,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
905 fp8_backend=self.fp8_backend,
906 experts_cls=self.experts_cls,
907 routing_tables=layer._expert_routing_tables(),
908 + layer=layer,
909 )
910
911 def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -951,6 +952,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
952 a1_scale=a1_scale,
953 a2_scale=a2_scale,
954 swiglu_limit=getattr(layer, "swiglu_limit", None),
955 + layer=layer,
956 )
957
958 def apply_monolithic(
@@ -1602,7 +1604,9 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
1604 moe_quant_config=self.moe_quant_config,
1605 moe_config=self.moe,
1606 experts_cls=self.experts_cls,
1607 + backend=self.nvfp4_backend,
1608 routing_tables=layer._expert_routing_tables(),
1609 + layer=layer,
1610 )
1611 self.moe_kernel.fused_experts.process_weights_after_loading(layer)
1612
@@ -1616,6 +1620,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
1620 a13_scale=layer.w13_input_scale,
1621 a2_scale=layer.w2_input_scale,
1622 swiglu_limit=getattr(layer, "swiglu_limit", None),
1623 + layer=layer,
1624 )
1625
1626 @property
@@ -2162,6 +2167,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
2167 fp8_backend=self.mxfp8_backend,
2168 experts_cls=self.experts_cls,
2169 routing_tables=layer._expert_routing_tables(),
2170 + layer=layer,
2171 )
2172
2173 # No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
@@ -2207,6 +2213,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
2213 swiglu_limit=getattr(layer, "swiglu_limit", None),
2214 gemm1_alpha=getattr(layer, "swiglu_alpha", None),
2215 gemm1_beta=getattr(layer, "swiglu_beta", None),
2216 + layer=layer,
2217 )
2218
2219 def apply_monolithic(
vllm/model_executor/layers/quantization/online/fp8.py
+2
@@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
458 fp8_backend=self.fp8_backend,
459 experts_cls=self.experts_cls,
460 routing_tables=layer._expert_routing_tables(),
461 + layer=layer,
462 )
463
464 def get_fused_moe_quant_config(
@@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
487 swiglu_limit=getattr(layer, "swiglu_limit", None),
488 gemm1_alpha=getattr(layer, "swiglu_alpha", None),
489 gemm1_beta=getattr(layer, "swiglu_beta", None),
490 + layer=layer,
491 )
492
493
vllm/model_executor/layers/quantization/online/int8.py
+17 -2
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
13
14 from vllm.model_executor.layers.fused_moe import RoutedExperts
15 from vllm.model_executor.layers.fused_moe.oracle.int8 import (
16 + convert_to_int8_moe_kernel_format,
17 make_int8_moe_kernel,
18 make_int8_moe_quant_config,
19 select_int8_moe_backend,
@@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
93 replace_parameter(layer, "w2_scale", w2_scale)
94
95 def _setup_kernel(self, layer: RoutedExperts) -> None:
96 + w13, w2 = convert_to_int8_moe_kernel_format(
97 + int8_backend=self.int8_backend,
98 + w13=layer.w13_weight,
99 + w2=layer.w2_weight,
100 + layer=layer,
101 + w13_scale=layer.w13_scale,
102 + )
103 + replace_parameter(layer, "w13_weight", w13)
104 + replace_parameter(layer, "w2_weight", w2)
105 +
106 self.moe_quant_config = self.get_fused_moe_quant_config(layer)
107 assert self.moe_quant_config is not None
108 assert self.experts_cls is not None
109 self.moe_kernel = make_int8_moe_kernel(
110 + int8_backend=self.int8_backend,
111 moe_quant_config=self.moe_quant_config,
112 moe_config=self.moe,
113 experts_cls=self.experts_cls,
114 routing_tables=layer._expert_routing_tables(),
115 + layer=layer,
116 )
117
118 def get_fused_moe_quant_config(
119 self, layer: torch.nn.Module
120 ) -> "FusedMoEQuantConfig | None":
121 return make_int8_moe_quant_config(
109 - w1_scale=layer.w13_scale,
110 - w2_scale=layer.w2_scale,
122 + int8_backend=self.int8_backend,
123 + w1_scale=getattr(layer, "w13_scale", None),
124 + w2_scale=getattr(layer, "w2_scale", None),
125 w1_bias=getattr(layer, "w13_bias", None),
126 w2_bias=getattr(layer, "w2_bias", None),
127 + layer=layer,
128 )
vllm/model_executor/layers/quantization/online/mxfp8.py
+2
@@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
200 fp8_backend=self.fp8_backend,
201 experts_cls=self.experts_cls,
202 routing_tables=layer._expert_routing_tables(),
203 + layer=layer,
204 )
205
206 def get_fused_moe_quant_config(
@@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
227 swiglu_limit=getattr(layer, "swiglu_limit", None),
228 gemm1_alpha=getattr(layer, "swiglu_alpha", None),
229 gemm1_beta=getattr(layer, "swiglu_beta", None),
230 + layer=layer,
231 )
232
233 def process_weights_after_loading(self, layer: Module) -> None:
vllm/model_executor/layers/quantization/quark/quark_moe.py
+3
@@ -1562,7 +1562,9 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
1562 moe_quant_config=self.moe_quant_config,
1563 moe_config=self.moe,
1564 experts_cls=self.experts_cls,
1565 + backend=self.nvfp4_backend,
1566 routing_tables=layer._expert_routing_tables(),
1567 + layer=layer,
1568 )
1569
1570 def get_fused_moe_quant_config(
@@ -1576,6 +1578,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
1578 w2_scale_2=layer.w2_weight_scale_2,
1579 a13_scale=layer.w13_input_scale_2,
1580 a2_scale=layer.w2_input_scale_2,
1581 + layer=layer,
1582 )
1583
1584 def apply(
vllm/model_executor/layers/quantization/utils/humming_utils.py
+4 -6
@@ -764,7 +764,7 @@ def _convert_sublayer_to_humming(
764 Returns:
765 Tuple of (converted_weight_schema, converted_input_schema)
766 """
767 - from humming.schema import HummingWeightSchema
767 + from vllm.utils.humming import HummingWeightSchema
768
769 if isinstance(weight_schema, HummingWeightSchema):
770 # Already in Humming format
@@ -814,7 +814,7 @@ def _prepare_and_transform_sublayer(
814
815 This calls Humming's prepare_layer_meta and transform_humming_layer.
816 """
817 - from humming.layer import HummingMethod
817 + from vllm.utils.humming import HummingMethod
818
819 HummingMethod.prepare_layer_meta(
820 layer=layer,
@@ -866,7 +866,7 @@ def _process_single_sublayer(
866 Returns:
867 Tuple of (final_weight_schema, final_input_schema)
868 """
869 - from humming.schema import HummingWeightSchema
869 + from vllm.utils.humming import HummingWeightSchema
870
871 # Step 1: Convert from checkpoint format to humming format if needed
872 current_weight_schema, current_input_schema = _convert_sublayer_to_humming(
@@ -958,12 +958,10 @@ def convert_to_humming_moe_kernel_format(
958 "Must provide either weight_schema/input_schema or quant_config"
959 )
960
961 - from humming.layer import HummingInputSchema
962 - from humming.schema import BaseWeightSchema
963 -
961 from vllm.model_executor.layers.quantization.utils.humming_utils import (
962 humming_is_layer_skipped,
963 )
964 + from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
965
966 if weight_schema is None:
967 weight_schema = BaseWeightSchema.from_config(quant_config)
vllm/model_executor/warmup/deep_gemm_warmup.py
+9
@@ -11,6 +11,9 @@ from tqdm import tqdm
11
12 import vllm.envs as envs
13 from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
14 +from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
15 + DeepGemmFp8BlockScaledMMKernel,
16 +)
17 from vllm.model_executor.layers.fused_moe import MoERunner
18 from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
19 compute_aligned_M_and_alignment,
@@ -147,6 +150,12 @@ def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool:
150 ):
151 return False
152
153 + if not isinstance(
154 + getattr(module.quant_method, "fp8_linear", None),
155 + DeepGemmFp8BlockScaledMMKernel,
156 + ):
157 + return False
158 +
159 w, _, block_sizes = _extract_data_from_linear_base_module(module)
160 return (
161 block_sizes == get_mk_alignment_for_contiguous_layout()