A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 02:45 UTC
[CI] Fix LM Eval Large Models (H100) (#32423)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Matthew Bonanni committed
Jan 15, 2026 at 19:52 UTC
bcf2333cd6514543f579d9bb6d309c5b8a0bfd0d
1 file changed
+3
-1
vllm/model_executor/layers/quantization/input_quant_fp8.py
+3
-1
@@ -98,7 +98,9 @@ class QuantFP8(CustomOp):
98
num_token_padding=self.num_token_padding,
99
scale_ub=scale_ub,
100
use_per_token_if_dynamic=self.use_per_token_if_dynamic,
101
- group_shape=self.group_shape if self.static else None,
101
+ group_shape=(self.group_shape.row, self.group_shape.col)
102
+ if self.static
103
+ else None,
104
)
105
106
def forward_hip(