A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Fix TP > 1 for new granite (#8544)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Sep 17, 2024 at 17:17 UTC
98f9713399bd602ff954a83e6e6abcb4cf8b8864
1 file changed
+2
-1
vllm/model_executor/models/granite.py
+2
-1
@@ -428,7 +428,8 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA):
428
sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
429
logits = self.logits_processor(self.lm_head, hidden_states,
430
sampling_metadata)
431
- logits /= self.config.logits_scaling
431
+ if logits is not None:
432
+ logits /= self.config.logits_scaling
433
return logits
434
435
def sample(