A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[FIX] Minor bug fixes (#1035)
* [FIX] Minor bug fixes * Address review comments
Zhuohan Li committed
Sep 13, 2023 at 16:38 UTC
f04908cae782e1a2404eb3e4f331718d311d1e0d
2 files changed
+4
-3
vllm/model_executor/layers/sampler.py
+3
-2
@@ -82,8 +82,9 @@ class Sampler(nn.Module):
82
# We use float32 for probabilities and log probabilities.
83
# Compute the probabilities.
84
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
85
- # Compute the log probabilities (before applying top-p and top-k).
86
- logprobs = torch.log(probs)
85
+ # Compute the log probabilities.
86
+ # Use log_softmax to ensure numerical stability.
87
+ logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
88
89
# Sample the next tokens.
90
return _sample(probs, logprobs, input_metadata)
vllm/sequence.py
+1
-1
@@ -350,7 +350,7 @@ class SequenceOutputs:
350
351
def __eq__(self, other: object) -> bool:
352
if not isinstance(other, SequenceOutputs):
353
- return NotImplementedError()
353
+ raise NotImplementedError()
354
return (self.parent_seq_id == other.parent_seq_id
355
and self.output_token == other.output_token
356
and self.logprobs == other.logprobs)