@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f04908cae7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[FIX] Minor bug fixes (#1035)

* [FIX] Minor bug fixes * Address review comments

Zhuohan Li committed Sep 13, 2023 at 16:38 UTC f04908cae782e1a2404eb3e4f331718d311d1e0d
2 files changed +4 -3
vllm/model_executor/layers/sampler.py
+3 -2
@@ -82,8 +82,9 @@ class Sampler(nn.Module):
82 # We use float32 for probabilities and log probabilities.
83 # Compute the probabilities.
84 probs = torch.softmax(logits, dim=-1, dtype=torch.float)
85 - # Compute the log probabilities (before applying top-p and top-k).
86 - logprobs = torch.log(probs)
85 + # Compute the log probabilities.
86 + # Use log_softmax to ensure numerical stability.
87 + logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
88
89 # Sample the next tokens.
90 return _sample(probs, logprobs, input_metadata)
vllm/sequence.py
+1 -1
@@ -350,7 +350,7 @@ class SequenceOutputs:
350
351 def __eq__(self, other: object) -> bool:
352 if not isinstance(other, SequenceOutputs):
353 - return NotImplementedError()
353 + raise NotImplementedError()
354 return (self.parent_seq_id == other.parent_seq_id
355 and self.output_token == other.output_token
356 and self.logprobs == other.logprobs)