A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 03:45 UTC
[Quantization] support logical_widths for fp8 marlin (#30962)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Jinzhen Lin committed
Dec 21, 2025 at 04:02 UTC
ee52d9901d368da35a80048c299ccb53e78a9a01
1 file changed
+10
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+10
@@ -142,10 +142,20 @@ def prepare_fp8_layer_for_marlin(
142
# marlin kernel only support channel-wise and group-wise quantization
143
# we need to convert the scales
144
if weight_block_size is None:
145
+ logical_widths = getattr(layer, "logical_widths", [])
146
if scales.nelement() == 1:
147
# tensor-wise quantization -> channel-wise quantization
148
# (1, 1) =>(repeat)=> (1, size_n)
149
scales = scales.view(1, 1).repeat_interleave(part_size_n, 1)
150
+ elif scales.nelement() == len(logical_widths):
151
+ # tensor-wise quantization with logical_widths ->
152
+ # channel-wise quantization
153
+ assert sum(logical_widths) == part_size_n, (
154
+ f"Sum of logical_widths ({sum(logical_widths)}) must be equal "
155
+ f"to part_size_n ({part_size_n})"
156
+ )
157
+ lw_tensor = scales.new_tensor(logical_widths, dtype=torch.int64)
158
+ scales = scales.view(1, -1).repeat_interleave(lw_tensor, dim=1)
159
elif scales.nelement() > 1 and scales.nelement() != part_size_n:
160
assert part_size_n % scales.nelement() == 0
161
s_size = scales.nelement()