A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support Min P Sampler (#1642)
Roy committed
Nov 18, 2023 at 08:20 UTC
e87557b069db2b945563c5d7a27ddc27b8361105
2 files changed
+40
-4
vllm/model_executor/layers/sampler.py
+31
-4
@@ -71,13 +71,18 @@ class Sampler(nn.Module):
71
logits.div_(t.unsqueeze(dim=1))
72
73
# Apply top-p and top-k truncation.
74
- top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
74
+ top_ps, top_ks, min_ps = _get_top_p_top_k_min_p(
75
+ input_metadata, self.vocab_size)
76
assert len(top_ps) == len(top_ks) == logits.shape[0]
77
do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
78
do_top_k = any(k != self.vocab_size for k in top_ks)
79
if do_top_p or do_top_k:
80
logits = _apply_top_p_top_k(logits, top_ps, top_ks)
81
82
+ do_min_p = any(mp > _SAMPLING_EPS for mp in min_ps)
83
+ if do_min_p:
84
+ logits = _apply_min_p(logits, min_ps)
85
+
86
# We use float32 for probabilities and log probabilities.
87
# Compute the probabilities.
88
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
@@ -261,15 +266,17 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
266
return temperatures
267
268
264
-def _get_top_p_top_k(
269
+def _get_top_p_top_k_min_p(
270
input_metadata: InputMetadata,
271
vocab_size: int,
267
-) -> Tuple[List[float], List[int]]:
272
+) -> Tuple[List[float], List[int], List[float]]:
273
top_ps: List[float] = []
274
top_ks: List[int] = []
275
+ min_ps: List[float] = []
276
for i, seq_group in enumerate(input_metadata.seq_groups):
277
seq_ids, sampling_params = seq_group
278
top_p = sampling_params.top_p
279
+ min_p = sampling_params.min_p
280
# k should not be greater than the vocab size.
281
top_k = min(sampling_params.top_k, vocab_size)
282
# k=-1 means no truncation.
@@ -279,9 +286,11 @@ def _get_top_p_top_k(
286
prompt_len = input_metadata.prompt_lens[i]
287
top_ps += [top_p] * (prompt_len - 1)
288
top_ks += [top_k] * (prompt_len - 1)
289
+ min_ps += [min_p] * (prompt_len - 1)
290
top_ps += [top_p] * len(seq_ids)
291
top_ks += [top_k] * len(seq_ids)
284
- return top_ps, top_ks
292
+ min_ps += [min_p] * len(seq_ids)
293
+ return top_ps, top_ks, min_ps
294
295
296
def _apply_top_p_top_k(
@@ -313,6 +322,24 @@ def _apply_top_p_top_k(
322
return logits
323
324
325
+def _apply_min_p(
326
+ logits: torch.Tensor,
327
+ min_ps: List[float],
328
+) -> torch.Tensor:
329
+ """
330
+ Adapted from
331
+ https://github.com/oobabooga/text-generation-webui/blob/3146124ec01f02c8fb1650a6517cf1b60b537aaf/modules/sampler_hijack.py#L16C17-L16C17
332
+ """
333
+ min_p = torch.tensor(min_ps, dtype=logits.dtype, device=logits.device)
334
+ probs = torch.softmax(logits, dim=-1)
335
+ top_probs, _ = probs.max(dim=-1, keepdim=True)
336
+ scaled_min_p = min_p.unsqueeze(dim=1) * top_probs
337
+ tokens_to_remove = probs < scaled_min_p
338
+ logits = logits.masked_fill(tokens_to_remove, -float("inf"))
339
+
340
+ return logits
341
+
342
+
343
def _greedy_sample(
344
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
345
logprobs: torch.Tensor,
vllm/sampling_params.py
+9
@@ -52,6 +52,9 @@ class SamplingParams:
52
to consider. Must be in (0, 1]. Set to 1 to consider all tokens.
53
top_k: Integer that controls the number of top tokens to consider. Set
54
to -1 to consider all tokens.
55
+ min_p: Float that represents the minimum probability for a token to be
56
+ considered, relative to the probability of the most likely token.
57
+ Must be in [0, 1]. Set to 0 to disable this.
58
use_beam_search: Whether to use beam search instead of sampling.
59
length_penalty: Float that penalizes sequences based on their length.
60
Used in beam search.
@@ -94,6 +97,7 @@ class SamplingParams:
97
temperature: float = 1.0,
98
top_p: float = 1.0,
99
top_k: int = -1,
100
+ min_p: int = 0.0,
101
use_beam_search: bool = False,
102
length_penalty: float = 1.0,
103
early_stopping: Union[bool, str] = False,
@@ -115,6 +119,7 @@ class SamplingParams:
119
self.temperature = temperature
120
self.top_p = top_p
121
self.top_k = top_k
122
+ self.min_p = min_p
123
self.use_beam_search = use_beam_search
124
self.length_penalty = length_penalty
125
self.early_stopping = early_stopping
@@ -167,6 +172,9 @@ class SamplingParams:
172
if self.top_k < -1 or self.top_k == 0:
173
raise ValueError(f"top_k must be -1 (disable), or at least 1, "
174
f"got {self.top_k}.")
175
+ if not 0.0 <= self.min_p <= 1.0:
176
+ raise ValueError("min_p must be in [0, 1], got "
177
+ f"{self.min_p}.")
178
if self.max_tokens < 1:
179
raise ValueError(
180
f"max_tokens must be at least 1, got {self.max_tokens}.")
@@ -228,6 +236,7 @@ class SamplingParams:
236
f"temperature={self.temperature}, "
237
f"top_p={self.top_p}, "
238
f"top_k={self.top_k}, "
239
+ f"min_p={self.min_p}, "
240
f"use_beam_search={self.use_beam_search}, "
241
f"length_penalty={self.length_penalty}, "
242
f"early_stopping={self.early_stopping}, "