A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
Reinstate `best_of` for V0 (#14356)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 6, 2025 at 17:34 UTC
bf0560bda98bc664d5fe3a66440417e5cd483c53
6 files changed
+50
-3
tests/v1/sample/test_sampling_params_e2e.py
+8
@@ -25,6 +25,14 @@ def test_n_gt_1(model):
25
assert len(outputs[0].outputs) == 3
26
27
28
+def test_best_of(model):
29
+ """Raise a ValueError since best_of is deprecated."""
30
+
31
+ params = SamplingParams(n=2, best_of=3)
32
+ with pytest.raises(ValueError):
33
+ _ = model.generate(PROMPT, params)
34
+
35
+
36
def test_penalties(model):
37
"""Check that we do not get errors if applied."""
38
vllm/entrypoints/llm.py
+5
-1
@@ -97,7 +97,11 @@ class LLM:
97
throughput. However, if the value is too high, it may cause out-of-
98
memory (OOM) errors.
99
swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
100
- Too small values may cause out-of-memory (OOM) errors.
100
+ This can be used for temporarily storing the states of the requests
101
+ when their `best_of` sampling parameters are larger than 1. If all
102
+ requests will have `best_of=1`, you can safely set this to 0.
103
+ Noting that `best_of` is only supported in V0. Otherwise, too small
104
+ values may cause out-of-memory (OOM) errors.
105
cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
106
the model weights. This virtually increases the GPU memory space
107
you can use to hold the model weights, at the cost of CPU-GPU data
vllm/entrypoints/openai/protocol.py
+4
@@ -242,6 +242,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
242
user: Optional[str] = None
243
244
# doc: begin-chat-completion-sampling-params
245
+ best_of: Optional[int] = None
246
use_beam_search: bool = False
247
top_k: Optional[int] = None
248
min_p: Optional[float] = None
@@ -478,6 +479,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
479
480
return SamplingParams.from_optional(
481
n=self.n,
482
+ best_of=self.best_of,
483
presence_penalty=self.presence_penalty,
484
frequency_penalty=self.frequency_penalty,
485
repetition_penalty=repetition_penalty,
@@ -648,6 +650,7 @@ class CompletionRequest(OpenAIBaseModel):
650
# https://platform.openai.com/docs/api-reference/completions/create
651
model: Optional[str] = None
652
prompt: Union[list[int], list[list[int]], str, list[str]]
653
+ best_of: Optional[int] = None
654
echo: Optional[bool] = False
655
frequency_penalty: Optional[float] = 0.0
656
logit_bias: Optional[dict[str, float]] = None
@@ -845,6 +848,7 @@ class CompletionRequest(OpenAIBaseModel):
848
849
return SamplingParams.from_optional(
850
n=self.n,
851
+ best_of=self.best_of,
852
presence_penalty=self.presence_penalty,
853
frequency_penalty=self.frequency_penalty,
854
repetition_penalty=repetition_penalty,
vllm/entrypoints/openai/serving_completion.py
+6
-2
@@ -168,8 +168,12 @@ class OpenAIServingCompletion(OpenAIServing):
168
model_name = self._get_model_name(request.model, lora_request)
169
num_prompts = len(engine_prompts)
170
171
- # We do not stream the results when use beam search.
172
- stream = (request.stream and not request.use_beam_search)
171
+ # Similar to the OpenAI API, when n != best_of, we do not stream the
172
+ # results. Noting that best_of is only supported in V0. In addition,
173
+ # we do not stream the results when use beam search.
174
+ stream = (request.stream
175
+ and (request.best_of is None or request.n == request.best_of)
176
+ and not request.use_beam_search)
177
178
# Streaming response
179
if stream:
vllm/sampling_params.py
+24
@@ -116,6 +116,10 @@ class SamplingParams(
116
117
Args:
118
n: Number of output sequences to return for the given prompt.
119
+ best_of: Number of output sequences that are generated from the prompt.
120
+ From these `best_of` sequences, the top `n` sequences are returned.
121
+ `best_of` must be greater than or equal to `n`. By default,
122
+ `best_of` is set to `n`. Warning, this is only supported in V0.
123
presence_penalty: Float that penalizes new tokens based on whether they
124
appear in the generated text so far. Values > 0 encourage the model
125
to use new tokens, while values < 0 encourage the model to repeat
@@ -183,6 +187,7 @@ class SamplingParams(
187
"""
188
189
n: int = 1
190
+ best_of: Optional[int] = None
191
_real_n: Optional[int] = None
192
presence_penalty: float = 0.0
193
frequency_penalty: float = 0.0
@@ -226,6 +231,7 @@ class SamplingParams(
231
@staticmethod
232
def from_optional(
233
n: Optional[int] = 1,
234
+ best_of: Optional[int] = None,
235
presence_penalty: Optional[float] = 0.0,
236
frequency_penalty: Optional[float] = 0.0,
237
repetition_penalty: Optional[float] = 1.0,
@@ -264,6 +270,7 @@ class SamplingParams(
270
271
return SamplingParams(
272
n=1 if n is None else n,
273
+ best_of=best_of,
274
presence_penalty=0.0
275
if presence_penalty is None else presence_penalty,
276
frequency_penalty=0.0
@@ -296,6 +303,20 @@ class SamplingParams(
303
)
304
305
def __post_init__(self) -> None:
306
+ # how we deal with `best_of``:
307
+ # if `best_of`` is not set, we default to `n`;
308
+ # if `best_of`` is set, we set `n`` to `best_of`,
309
+ # and set `_real_n`` to the original `n`.
310
+ # when we return the result, we will check
311
+ # if we need to return `n` or `_real_n` results
312
+ if self.best_of:
313
+ if self.best_of < self.n:
314
+ raise ValueError(
315
+ f"best_of must be greater than or equal to n, "
316
+ f"got n={self.n} and best_of={self.best_of}.")
317
+ if not self._real_n:
318
+ self._real_n = self.n
319
+ self.n = self.best_of
320
321
if 0 < self.temperature < _MAX_TEMP:
322
logger.warning(
@@ -402,6 +423,9 @@ class SamplingParams(
423
raise ValueError(
424
"stop strings are only supported when detokenize is True. "
425
"Set detokenize=True to use stop.")
426
+ if self.best_of != self._real_n and self.output_kind == (
427
+ RequestOutputKind.DELTA):
428
+ raise ValueError("best_of must equal n to use output_kind=DELTA")
429
430
def _verify_greedy_sampling(self) -> None:
431
if self.n > 1:
vllm/v1/engine/processor.py
+3
@@ -93,6 +93,9 @@ class Processor:
93
self,
94
params: SamplingParams,
95
) -> None:
96
+ # Best of not yet supported.
97
+ if params.best_of is not None and params.best_of > 1:
98
+ raise ValueError("VLLM V1 does not yet support best_of.")
99
# Bad words not yet supported.
100
if params.bad_words:
101
raise ValueError("VLLM V1 does not yet support bad_words.")