A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] Fix Tensor Parallelism Padding Consistency in Granite Models (#20843)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Jul 12, 2025 at 07:11 UTC
c2a2f19abad77e8a8b97c178c4ea1684c2747348
1 file changed
+4
vllm/model_executor/models/granite.py
+4
@@ -273,6 +273,10 @@ class GraniteModel(nn.Module):
273
self.vocab_size,
274
config.hidden_size,
275
org_num_embeddings=config.vocab_size,
276
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
277
+ # We need bigger padding if using lora for kernel
278
+ # compatibility
279
+ if not lora_config else lora_config.lora_vocab_padding_size,
280
quant_config=quant_config,
281
)
282
else: