A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
[Bugfix] Expand quantization method support in perf metrics (#37231)
Signed-off-by: Thillai Chithambaram <thillaichithambaram.a@gmail.com>
Thillai Chithambaram committed
Mar 18, 2026 at 19:54 UTC
828f862acb5f46ffaa1633aa80d85af73c31c97a
2 files changed
+162
-22
tests/v1/metrics/test_perf_metrics.py
+116
@@ -7,6 +7,7 @@ Tests for the analytic estimators in metrics/flops.py.
7
import types
8
from types import SimpleNamespace
9
10
+import pytest
11
from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config
12
from transformers.models.llama4.configuration_llama4 import (
13
Llama4Config,
@@ -21,10 +22,12 @@ from vllm.transformers_utils.model_arch_config_convertor import (
22
ModelArchConfigConvertorBase,
23
)
24
from vllm.v1.metrics.perf import (
25
+ _QUANT_WEIGHT_BYTE_SIZE,
26
AttentionMetrics,
27
BaseConfigParser,
28
ExecutionContext,
29
FfnMetrics,
30
+ InvalidComponent,
31
ModelMetrics,
32
ParsedArgs,
33
UnembedMetrics,
@@ -905,3 +908,116 @@ def test_attention_per_gpu_heads_not_evenly_divisible():
908
assert per_gpu_flops > 0
909
assert global_flops > 0
910
assert global_flops > per_gpu_flops
911
+
912
+
913
+# INT4 / FP4 quantization methods (weight_byte_size == 0.5)
914
+_INT4_FP4_METHODS = [m for m, s in _QUANT_WEIGHT_BYTE_SIZE.items() if s == 0.5]
915
+
916
+
917
+@pytest.mark.parametrize("quant_method", _INT4_FP4_METHODS)
918
+def test_quantization_config_parser_int4_methods(quant_method):
919
+ """Test quantization parsers with INT4/FP4 methods (0.5 bytes)."""
920
+
921
+ class MockQuantConfig:
922
+ def get_name(self):
923
+ return quant_method
924
+
925
+ hf_config = Qwen3Config(
926
+ hidden_size=2048,
927
+ num_attention_heads=16,
928
+ intermediate_size=8192,
929
+ num_hidden_layers=1,
930
+ )
931
+ vllm_config = create_mock_vllm_config(hf_config, quant_config=MockQuantConfig())
932
+
933
+ attn_result = AttentionMetrics.get_parser().parse(vllm_config)
934
+ assert attn_result.weight_byte_size == 0.5, (
935
+ f"Expected 0.5 for {quant_method}, got {attn_result.weight_byte_size}"
936
+ )
937
+
938
+ ffn_result = FfnMetrics.get_parser().parse(vllm_config)
939
+ assert ffn_result.weight_byte_size == 0.5, (
940
+ f"Expected 0.5 for {quant_method}, got {ffn_result.weight_byte_size}"
941
+ )
942
+
943
+
944
+# FP8 / INT8 quantization methods (weight_byte_size == 1)
945
+_FP8_INT8_METHODS = [m for m, s in _QUANT_WEIGHT_BYTE_SIZE.items() if s == 1]
946
+
947
+
948
+@pytest.mark.parametrize("quant_method", _FP8_INT8_METHODS)
949
+def test_quantization_config_parser_fp8_methods(quant_method):
950
+ """Test quantization parsers with FP8/INT8 methods (1 byte)."""
951
+
952
+ class MockQuantConfig:
953
+ def get_name(self):
954
+ return quant_method
955
+
956
+ hf_config = Qwen3Config(
957
+ hidden_size=2048,
958
+ num_attention_heads=16,
959
+ intermediate_size=8192,
960
+ num_hidden_layers=1,
961
+ )
962
+ vllm_config = create_mock_vllm_config(hf_config, quant_config=MockQuantConfig())
963
+
964
+ attn_result = AttentionMetrics.get_parser().parse(vllm_config)
965
+ assert attn_result.weight_byte_size == 1, (
966
+ f"Expected 1 for {quant_method}, got {attn_result.weight_byte_size}"
967
+ )
968
+
969
+ ffn_result = FfnMetrics.get_parser().parse(vllm_config)
970
+ assert ffn_result.weight_byte_size == 1, (
971
+ f"Expected 1 for {quant_method}, got {ffn_result.weight_byte_size}"
972
+ )
973
+
974
+
975
+def test_quantization_config_parser_unknown_method():
976
+ """Test that an unrecognized quant method raises InvalidComponent."""
977
+
978
+ class MockQuantConfig:
979
+ def get_name(self):
980
+ return "unknown_quant_method"
981
+
982
+ hf_config = Qwen3Config(
983
+ hidden_size=2048,
984
+ num_attention_heads=16,
985
+ intermediate_size=8192,
986
+ num_hidden_layers=1,
987
+ )
988
+ vllm_config = create_mock_vllm_config(hf_config, quant_config=MockQuantConfig())
989
+
990
+ with pytest.raises(InvalidComponent):
991
+ AttentionMetrics.get_parser().parse(vllm_config)
992
+
993
+ with pytest.raises(InvalidComponent):
994
+ FfnMetrics.get_parser().parse(vllm_config)
995
+
996
+
997
+def test_quantized_model_metrics_aggregation():
998
+ """Test that ModelMetrics works end-to-end with a quantized model config."""
999
+
1000
+ class MockQuantConfig:
1001
+ def get_name(self):
1002
+ return "gptq"
1003
+
1004
+ hf_config = Qwen3Config(
1005
+ hidden_size=2048,
1006
+ num_attention_heads=16,
1007
+ num_hidden_layers=12,
1008
+ vocab_size=32000,
1009
+ intermediate_size=8192,
1010
+ )
1011
+ vllm_config = create_mock_vllm_config(hf_config, quant_config=MockQuantConfig())
1012
+
1013
+ model_metrics = ModelMetrics(vllm_config)
1014
+ ctx = ExecutionContext.from_single_request(
1015
+ num_tokens=100, context_len=512, is_prefill=True
1016
+ )
1017
+
1018
+ # Should not crash and should produce valid metrics
1019
+ total_flops = model_metrics.get_num_flops(ctx)
1020
+ breakdown = model_metrics.get_num_flops_breakdown(ctx)
1021
+
1022
+ assert total_flops > 0
1023
+ assert total_flops == sum(breakdown.values())
vllm/v1/metrics/perf.py
+46
-22
@@ -40,6 +40,42 @@ class InvalidComponent(Exception):
40
pass
41
42
43
+# Mapping from quantization method name to effective weight byte size.
44
+# Used by both AttentionQuantizationConfigParser and
45
+# FfnQuantizationConfigParser to determine the weight_byte_size for
46
+# flops/memory estimation.
47
+#
48
+# NOTE: Methods like GPTQ and BitsAndBytes support variable bit-widths
49
+# (e.g., 4-bit and 8-bit). We default to 4-bit (0.5 bytes) since this
50
+# is by far the most common configuration.
51
+_QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {
52
+ # FP8 methods (1 byte per weight)
53
+ "fp8": 1,
54
+ "fbgemm_fp8": 1,
55
+ "ptpc_fp8": 1,
56
+ "fp_quant": 1,
57
+ "modelopt": 1,
58
+ "modelopt_mxfp8": 1,
59
+ # FP4 / INT4 methods (0.5 bytes per weight)
60
+ "mxfp4": 0.5,
61
+ "awq": 0.5,
62
+ "awq_marlin": 0.5,
63
+ "gptq": 0.5,
64
+ "gptq_marlin": 0.5,
65
+ "bitsandbytes": 0.5,
66
+ "modelopt_fp4": 0.5,
67
+ "petit_nvfp4": 0.5,
68
+ "gguf": 0.5,
69
+ "compressed-tensors": 0.5,
70
+ "torchao": 0.5,
71
+ "quark": 0.5,
72
+ "moe_wna16": 0.5,
73
+ "inc": 0.5,
74
+ "cpu_awq": 0.5,
75
+ "experts_int8": 1,
76
+}
77
+
78
+
79
#### Basic Data Types ####
80
81
@@ -350,17 +386,12 @@ class AttentionQuantizationConfigParser(Parser):
386
return args
387
388
quant_method = cfg.get_name()
353
- if quant_method in ["fp8", "fbgemm_fp8"]:
354
- # FIXME: This is a hacky coarse-grained fp8 quantization detection.
355
- # FIXME: These configs also have concept of "ignored layers" and we
356
- # need to solve the same problem as above.
357
- args.weight_byte_size = 1
358
- elif quant_method == "mxfp4":
359
- # FIXME: Also has "ignored layers" issue above
360
- args.weight_byte_size = 0.5
389
+ if quant_method in _QUANT_WEIGHT_BYTE_SIZE:
390
+ args.weight_byte_size = _QUANT_WEIGHT_BYTE_SIZE[quant_method]
391
else:
362
- # FIXME: Add more parsing logic for different quant methods.
363
- raise InvalidComponent
392
+ raise InvalidComponent(
393
+ f"Unsupported quantization method for attention metrics: {quant_method}"
394
+ )
395
396
return args
397
@@ -617,19 +648,12 @@ class FfnQuantizationConfigParser(Parser):
648
return args
649
650
quant_method = cfg.get_name()
620
- if quant_method in ["fp8", "fbgemm_fp8"]:
621
- # FIXME: This is a hacky coarse-grained fp8 quantization detection.
622
- # (there might be more quantization methods for fp8).
623
- # FIXME: These configs also have concept of "ignored layers" and we
624
- # need to solve the same problem as above.
625
- args.weight_byte_size = 1
626
- pass
627
- elif quant_method == "mxfp4":
628
- # FIXME: Also has "ignored layers" issue above
629
- args.weight_byte_size = 0.5
651
+ if quant_method in _QUANT_WEIGHT_BYTE_SIZE:
652
+ args.weight_byte_size = _QUANT_WEIGHT_BYTE_SIZE[quant_method]
653
else:
631
- # FIXME: Add more parsing logic for different quant methods.
632
- raise InvalidComponent
654
+ raise InvalidComponent(
655
+ f"Unsupported quantization method for FFN metrics: {quant_method}"
656
+ )
657
658
return args
659