A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
[Bugfix] Fix Qwen3.5-FP8 Weight Loading Error on TPU (#37348)
Signed-off-by: Jacob Platin <jacobplatin@google.com>
Jacob Platin committed
Mar 25, 2026 at 17:46 UTC
d7d51a7ee5ce3b0fe420e773e2dcd38336c338ff
1 file changed
+14
vllm/model_executor/layers/linear.py
+14
@@ -768,6 +768,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
768
# Special case for Quantization.
769
# If quantized, we need to adjust the offset and size to account
770
# for the packing.
771
+ # Add check to adjust the size/offset for FP8 block scales
772
+ if isinstance(param, BlockQuantScaleParameter):
773
+ weight_block_size = getattr(self, "weight_block_size", None)
774
+ shard_size, shard_offset = adjust_block_scale_shard(
775
+ weight_block_size, shard_size, shard_offset
776
+ )
777
+
778
if packed_dim == output_dim:
779
shard_size = shard_size // param.packed_factor
780
shard_offset = shard_offset // param.packed_factor
@@ -1218,6 +1225,13 @@ class QKVParallelLinear(ColumnParallelLinear):
1225
# Special case for Quantized Weights.
1226
# If quantized, we need to adjust the offset and size to account
1227
# for the packing.
1228
+ # Add check to adjust the size/offset for FP8 block scales
1229
+ if isinstance(param, BlockQuantScaleParameter):
1230
+ weight_block_size = getattr(self, "weight_block_size", None)
1231
+ shard_size, shard_offset = adjust_block_scale_shard(
1232
+ weight_block_size, shard_size, shard_offset
1233
+ )
1234
+
1235
if packed_dim == output_dim:
1236
shard_size = shard_size // param.packed_factor
1237
shard_offset = shard_offset // param.packed_factor