A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
remove floats == 0 comparison (#285)
Lily Liu committed
Jun 28, 2023 at 14:11 UTC
425040d4c15464a3185c5a6010ce58d1919591ae
2 files changed
+11
-9
vllm/model_executor/layers/sampler.py
+6
-5
@@ -11,6 +11,7 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
11
from vllm.sampling_params import SamplingParams
12
from vllm.sequence import SequenceOutputs
13
14
+_SAMPLING_EPS = 1e-5
15
16
class Sampler(nn.Module):
17
"""Samples the next tokens from the model's outputs.
@@ -74,7 +75,7 @@ class Sampler(nn.Module):
75
# Apply top-p and top-k truncation.
76
top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
77
assert len(top_ps) == len(top_ks) == probs.shape[0]
77
- if any(p < 1.0 for p in top_ps) or any(k != self.vocab_size for k in top_ks):
78
+ if any(p < 1.0 - _SAMPLING_EPS for p in top_ps) or any(k != self.vocab_size for k in top_ks):
79
probs = _apply_top_p_top_k(probs, top_ps, top_ks)
80
81
# Sample the next tokens.
@@ -152,7 +153,7 @@ def _apply_penalties(
153
continue
154
p = presence_penalties[i]
155
f = frequency_penalties[i]
155
- if p == 0.0 and f == 0.0:
156
+ if p < _SAMPLING_EPS and f < _SAMPLING_EPS:
157
continue
158
indices.append(i)
159
@@ -190,7 +191,7 @@ def _get_temperatures(
191
for i, seq_group in enumerate(input_metadata.seq_groups):
192
seq_ids, sampling_params = seq_group
193
temperature = sampling_params.temperature
193
- if temperature == 0.0:
194
+ if temperature < _SAMPLING_EPS:
195
# NOTE: Zero temperature means deterministic sampling
196
# (i.e., greedy sampling or beam search).
197
# Set the temperature to 1 to avoid division by zero.
@@ -286,7 +287,7 @@ def _sample_from_prompt(
287
beam_width = sampling_params.best_of
288
_, next_token_ids = torch.topk(prob, beam_width)
289
next_token_ids = next_token_ids.tolist()
289
- elif sampling_params.temperature == 0.0:
290
+ elif sampling_params.temperature < _SAMPLING_EPS:
291
# Greedy sampling.
292
assert sampling_params.best_of == 1
293
next_token_id = torch.argmax(prob)
@@ -343,7 +344,7 @@ def _sample_from_generation_tokens(
344
345
parent_seq_ids = [beam_outputs[seq_id][0] for seq_id in seq_ids]
346
next_token_ids = [beam_outputs[seq_id][1] for seq_id in seq_ids]
346
- elif sampling_params.temperature == 0.0:
347
+ elif sampling_params.temperature < _SAMPLING_EPS:
348
# Greedy sampling.
349
assert len(seq_ids) == 1
350
next_token_id = torch.argmax(probs, dim=-1)
vllm/sampling_params.py
+5
-4
@@ -1,6 +1,7 @@
1
"""Sampling parameters for text generation."""
2
from typing import List, Optional, Union
3
4
+_SAMPLING_EPS = 1e-5
5
6
class SamplingParams:
7
"""Sampling parameters for text generation.
@@ -71,7 +72,7 @@ class SamplingParams:
72
self._verify_args()
73
if self.use_beam_search:
74
self._verity_beam_search()
74
- elif self.temperature == 0.0:
75
+ elif self.temperature < _SAMPLING_EPS:
76
# Zero temperature means greedy sampling.
77
self._verify_greedy_sampling()
78
@@ -106,9 +107,9 @@ class SamplingParams:
107
if self.best_of == 1:
108
raise ValueError("best_of must be greater than 1 when using beam "
109
f"search. Got {self.best_of}.")
109
- if self.temperature > 0.0:
110
+ if self.temperature > _SAMPLING_EPS:
111
raise ValueError("temperature must be 0 when using beam search.")
111
- if self.top_p < 1.0:
112
+ if self.top_p < 1.0 - _SAMPLING_EPS:
113
raise ValueError("top_p must be 1 when using beam search.")
114
if self.top_k != -1:
115
raise ValueError("top_k must be -1 when using beam search.")
@@ -117,7 +118,7 @@ class SamplingParams:
118
if self.best_of > 1:
119
raise ValueError("best_of must be 1 when using greedy sampling."
120
f"Got {self.best_of}.")
120
- if self.top_p < 1.0:
121
+ if self.top_p < 1.0 - _SAMPLING_EPS:
122
raise ValueError("top_p must be 1 when using greedy sampling.")
123
if self.top_k != -1:
124
raise ValueError("top_k must be -1 when using greedy sampling.")