A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Model] Qwen3 Dense FP8 Compat Fixes (#17318)
Signed-off-by: simon-mo <xmo@berkeley.edu>
Simon Mo committed
Apr 28, 2025 at 14:12 UTC
dcbac4cb4bd565f964104911b5fac7a5cb837b3b
1 file changed
+9
vllm/model_executor/layers/linear.py
+9
@@ -929,6 +929,15 @@ class QKVParallelLinear(ColumnParallelLinear):
929
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
930
shard_size = self._get_shard_size_mapping(loaded_shard_id)
931
932
+ # Note(simon): This is needed for Qwen3's fp8 quantization.
933
+ if isinstance(param, BlockQuantScaleParameter):
934
+ assert self.quant_method is not None
935
+ assert hasattr(self.quant_method, "quant_config")
936
+ weight_block_size = self.quant_method.quant_config.weight_block_size
937
+ block_n, _ = weight_block_size[0], weight_block_size[1]
938
+ shard_offset = (shard_offset + block_n - 1) // block_n
939
+ shard_size = (shard_size + block_n - 1) // block_n
940
+
941
param.load_qkv_weight(loaded_weight=loaded_weight,
942
num_heads=self.num_kv_head_replicas,
943
shard_id=loaded_shard_id,