[Bugfix][Metrics][MFU] Size activation traffic from the model dtype (#57070)
Signed-off-by: Thillai Chithambaram <thillaichithambaram.a@gmail.com>
Thillai Chithambaram committed
Sep 16, 2026 at 20:23 UTC
a615f53364c109ade2b6bb69248e16ed40405d1f
2 files changed
+54
-4
tests/v1/metrics/test_perf_metrics.py
+50
-1
@@ -133,7 +133,56 @@ def test_base_config_parser():
133
assert result.num_attention_heads == 16
134
assert result.num_hidden_layers == 24
135
assert result.weight_byte_size == 2 # float16 is 2 bytes
136
- assert result.activation_byte_size == 2 # default activation size
136
+ assert result.activation_byte_size == 2 # float16 activations are 2 bytes
137
+
138
+
139
+@pytest.mark.parametrize(
140
+ "model_dtype,expected_byte_size",
141
+ [("float16", 2), ("bfloat16", 2), ("float32", 4)],
142
+)
143
+def test_base_config_parser_activation_byte_size_follows_dtype(
144
+ model_dtype: str, expected_byte_size: int
145
+):
146
+ """Activation traffic is sized from the model's compute dtype.
147
+
148
+ Activations are produced in the model dtype, so pinning them to 2 bytes
149
+ under-reported memory traffic by half for a float32 model while that same
150
+ model's weight traffic was sized correctly.
151
+ """
152
+ hf_config = Qwen3Config(
153
+ hidden_size=2048,
154
+ num_attention_heads=16,
155
+ num_hidden_layers=24,
156
+ )
157
+ vllm_config = create_mock_vllm_config(hf_config, model_dtype=model_dtype)
158
+
159
+ result = BaseConfigParser().parse(ParsedArgs(), vllm_config)
160
+
161
+ assert result.activation_byte_size == expected_byte_size
162
+ assert result.weight_byte_size == expected_byte_size
163
+
164
+
165
+def test_attention_read_bytes_scale_with_activation_dtype():
166
+ """A float32 model reads twice the activation bytes of a bfloat16 one."""
167
+ hf_config = Qwen3Config(
168
+ hidden_size=2048,
169
+ num_attention_heads=16,
170
+ num_key_value_heads=16,
171
+ num_hidden_layers=4,
172
+ )
173
+ ctx = ExecutionContext.from_single_request(
174
+ num_tokens=128, context_len=128, is_prefill=True
175
+ )
176
+
177
+ bf16 = AttentionMetrics.from_vllm_config(
178
+ create_mock_vllm_config(hf_config, model_dtype="bfloat16")
179
+ ).get_read_bytes_breakdown(ctx)
180
+ fp32 = AttentionMetrics.from_vllm_config(
181
+ create_mock_vllm_config(hf_config, model_dtype="float32")
182
+ ).get_read_bytes_breakdown(ctx)
183
+
184
+ # qkv_input is pure activation traffic, so it doubles exactly.
185
+ assert fp32["qkv_input"] == 2 * bf16["qkv_input"]
186
187
188
def test_base_attention_config_parser_with_gqa():
vllm/v1/metrics/perf.py
+4
-3
@@ -369,9 +369,10 @@ class BaseConfigParser(Parser):
369
370
args.weight_byte_size = get_dtype_size(torch_dtype)
371
372
- # FIXME: handle this better by parsing whether activations use
373
- # bf16, fp32, etc...
374
- args.activation_byte_size = 2
372
+ # Activations are produced in the model's compute dtype. Quantization
373
+ # overrides weight_byte_size below but leaves activations alone, so
374
+ # this stays keyed on the model dtype.
375
+ args.activation_byte_size = get_dtype_size(torch_dtype)
376
377
args.dp_size = vllm_config.parallel_config.data_parallel_size
378
args.tp_size = vllm_config.parallel_config.tensor_parallel_size