@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 425040d4c1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

remove floats == 0 comparison (#285)

Lily Liu committed Jun 28, 2023 at 14:11 UTC 425040d4c15464a3185c5a6010ce58d1919591ae
2 files changed +11 -9
vllm/model_executor/layers/sampler.py
+6 -5
@@ -11,6 +11,7 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
11 from vllm.sampling_params import SamplingParams
12 from vllm.sequence import SequenceOutputs
13
14 +_SAMPLING_EPS = 1e-5
15
16 class Sampler(nn.Module):
17 """Samples the next tokens from the model's outputs.
@@ -74,7 +75,7 @@ class Sampler(nn.Module):
75 # Apply top-p and top-k truncation.
76 top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
77 assert len(top_ps) == len(top_ks) == probs.shape[0]
77 - if any(p < 1.0 for p in top_ps) or any(k != self.vocab_size for k in top_ks):
78 + if any(p < 1.0 - _SAMPLING_EPS for p in top_ps) or any(k != self.vocab_size for k in top_ks):
79 probs = _apply_top_p_top_k(probs, top_ps, top_ks)
80
81 # Sample the next tokens.
@@ -152,7 +153,7 @@ def _apply_penalties(
153 continue
154 p = presence_penalties[i]
155 f = frequency_penalties[i]
155 - if p == 0.0 and f == 0.0:
156 + if p < _SAMPLING_EPS and f < _SAMPLING_EPS:
157 continue
158 indices.append(i)
159
@@ -190,7 +191,7 @@ def _get_temperatures(
191 for i, seq_group in enumerate(input_metadata.seq_groups):
192 seq_ids, sampling_params = seq_group
193 temperature = sampling_params.temperature
193 - if temperature == 0.0:
194 + if temperature < _SAMPLING_EPS:
195 # NOTE: Zero temperature means deterministic sampling
196 # (i.e., greedy sampling or beam search).
197 # Set the temperature to 1 to avoid division by zero.
@@ -286,7 +287,7 @@ def _sample_from_prompt(
287 beam_width = sampling_params.best_of
288 _, next_token_ids = torch.topk(prob, beam_width)
289 next_token_ids = next_token_ids.tolist()
289 - elif sampling_params.temperature == 0.0:
290 + elif sampling_params.temperature < _SAMPLING_EPS:
291 # Greedy sampling.
292 assert sampling_params.best_of == 1
293 next_token_id = torch.argmax(prob)
@@ -343,7 +344,7 @@ def _sample_from_generation_tokens(
344
345 parent_seq_ids = [beam_outputs[seq_id][0] for seq_id in seq_ids]
346 next_token_ids = [beam_outputs[seq_id][1] for seq_id in seq_ids]
346 - elif sampling_params.temperature == 0.0:
347 + elif sampling_params.temperature < _SAMPLING_EPS:
348 # Greedy sampling.
349 assert len(seq_ids) == 1
350 next_token_id = torch.argmax(probs, dim=-1)
vllm/sampling_params.py
+5 -4
@@ -1,6 +1,7 @@
1 """Sampling parameters for text generation."""
2 from typing import List, Optional, Union
3
4 +_SAMPLING_EPS = 1e-5
5
6 class SamplingParams:
7 """Sampling parameters for text generation.
@@ -71,7 +72,7 @@ class SamplingParams:
72 self._verify_args()
73 if self.use_beam_search:
74 self._verity_beam_search()
74 - elif self.temperature == 0.0:
75 + elif self.temperature < _SAMPLING_EPS:
76 # Zero temperature means greedy sampling.
77 self._verify_greedy_sampling()
78
@@ -106,9 +107,9 @@ class SamplingParams:
107 if self.best_of == 1:
108 raise ValueError("best_of must be greater than 1 when using beam "
109 f"search. Got {self.best_of}.")
109 - if self.temperature > 0.0:
110 + if self.temperature > _SAMPLING_EPS:
111 raise ValueError("temperature must be 0 when using beam search.")
111 - if self.top_p < 1.0:
112 + if self.top_p < 1.0 - _SAMPLING_EPS:
113 raise ValueError("top_p must be 1 when using beam search.")
114 if self.top_k != -1:
115 raise ValueError("top_k must be -1 when using beam search.")
@@ -117,7 +118,7 @@ class SamplingParams:
118 if self.best_of > 1:
119 raise ValueError("best_of must be 1 when using greedy sampling."
120 f"Got {self.best_of}.")
120 - if self.top_p < 1.0:
121 + if self.top_p < 1.0 - _SAMPLING_EPS:
122 raise ValueError("top_p must be 1 when using greedy sampling.")
123 if self.top_k != -1:
124 raise ValueError("top_k must be -1 when using greedy sampling.")