A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Granite-4 support loading quantized checkpoint (#22925)
Signed-off-by: Chih-Chieh-Yang <7364402+cyang49@users.noreply.github.com>
Chih-Chieh Yang committed
Aug 15, 2025 at 14:47 UTC
6cd69f51bf9312b2e7f85d4831c1a101c7e9a6e5
1 file changed
+6
-2
vllm/model_executor/models/granitemoehybrid.py
+6
-2
@@ -471,7 +471,10 @@ class GraniteMoeHybridModel(nn.Module):
471
# Mapping different experts' layout:
472
# from HF (input_linear, output_linear, router)
473
# to vLLM (experts_w13({e}.w1, {e}.w2), experts_w3({e}.w3), gate)
474
- if n.endswith('.block_sparse_moe.input_linear.weight'):
474
+ # The renaming and parameter loading logic is the same for weight
475
+ # and weight_scale tensors so we can reuse them without issues.
476
+ if (n.endswith('.block_sparse_moe.input_linear.weight') or
477
+ n.endswith('.block_sparse_moe.input_linear.weight_scale')):
478
for e in range(p.size(0)):
479
w1_name = n.replace(
480
'.block_sparse_moe.input_linear.weight',
@@ -490,7 +493,8 @@ class GraniteMoeHybridModel(nn.Module):
493
w3_name,
494
shard_id='w3',
495
expert_id=e)
493
- elif n.endswith('.block_sparse_moe.output_linear.weight'):
496
+ elif (n.endswith('.block_sparse_moe.output_linear.weight') or
497
+ n.endswith('.block_sparse_moe.output_linear.weight_scale')):
498
for e in range(p.size(0)):
499
w2_name = n.replace(
500
'.block_sparse_moe.output_linear.weight',