A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
Deprecate `best_of` Sampling Parameter in anticipation for vLLM V1 (#13997)
Signed-off-by: vincent-4 <vincentzhongy+githubvincent4@gmail.com> Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Vincent committed
Mar 5, 2025 at 15:22 UTC
a4f1ee35d6864d305cd0521607743f80e7ad5d6b
12 files changed
+16
-88
benchmarks/backend_request_func.py
-5
@@ -27,7 +27,6 @@ class RequestFuncInput:
27
output_len: int
28
model: str
29
model_name: Optional[str] = None
30
- best_of: int = 1
30
logprobs: Optional[int] = None
31
extra_body: Optional[dict] = None
32
multi_modal_content: Optional[dict] = None
@@ -58,7 +57,6 @@ async def async_request_tgi(
57
async with aiohttp.ClientSession(trust_env=True,
58
timeout=AIOHTTP_TIMEOUT) as session:
59
params = {
61
- "best_of": request_func_input.best_of,
60
"max_new_tokens": request_func_input.output_len,
61
"do_sample": True,
62
"temperature": 0.01, # TGI does not accept 0.0 temperature.
@@ -130,7 +128,6 @@ async def async_request_trt_llm(
128
129
async with aiohttp.ClientSession(trust_env=True,
130
timeout=AIOHTTP_TIMEOUT) as session:
133
- assert request_func_input.best_of == 1
131
payload = {
132
"accumulate_tokens": True,
133
"text_input": request_func_input.prompt,
@@ -195,7 +192,6 @@ async def async_request_deepspeed_mii(
192
) -> RequestFuncOutput:
193
async with aiohttp.ClientSession(trust_env=True,
194
timeout=AIOHTTP_TIMEOUT) as session:
198
- assert request_func_input.best_of == 1
195
196
payload = {
197
"prompt": request_func_input.prompt,
@@ -249,7 +245,6 @@ async def async_request_openai_completions(
245
if request_func_input.model_name else request_func_input.model,
246
"prompt": request_func_input.prompt,
247
"temperature": 0.0,
252
- "best_of": request_func_input.best_of,
248
"max_tokens": request_func_input.output_len,
249
"logprobs": request_func_input.logprobs,
250
"stream": True,
benchmarks/benchmark_serving.py
-14
@@ -560,7 +560,6 @@ async def benchmark(
560
tokenizer: PreTrainedTokenizerBase,
561
input_requests: list[tuple[str, int, int]],
562
logprobs: Optional[int],
563
- best_of: int,
563
request_rate: float,
564
burstiness: float,
565
disable_tqdm: bool,
@@ -592,7 +591,6 @@ async def benchmark(
591
prompt_len=test_prompt_len,
592
output_len=test_output_len,
593
logprobs=logprobs,
595
- best_of=best_of,
594
multi_modal_content=test_mm_content,
595
ignore_eos=ignore_eos,
596
)
@@ -619,7 +617,6 @@ async def benchmark(
617
prompt_len=test_prompt_len,
618
output_len=test_output_len,
619
logprobs=logprobs,
622
- best_of=best_of,
620
multi_modal_content=test_mm_content,
621
ignore_eos=ignore_eos)
622
profile_output = await request_func(request_func_input=profile_input)
@@ -668,7 +665,6 @@ async def benchmark(
665
prompt_len=prompt_len,
666
output_len=output_len,
667
logprobs=logprobs,
671
- best_of=best_of,
668
multi_modal_content=mm_content,
669
ignore_eos=ignore_eos)
670
tasks.append(
@@ -686,7 +682,6 @@ async def benchmark(
682
prompt_len=test_prompt_len,
683
output_len=test_output_len,
684
logprobs=logprobs,
689
- best_of=best_of,
685
)
686
profile_output = await request_func(request_func_input=profile_input)
687
if profile_output.success:
@@ -958,7 +953,6 @@ def main(args: argparse.Namespace):
953
tokenizer=tokenizer,
954
input_requests=input_requests,
955
logprobs=args.logprobs,
961
- best_of=args.best_of,
956
request_rate=args.request_rate,
957
burstiness=args.burstiness,
958
disable_tqdm=args.disable_tqdm,
@@ -983,7 +977,6 @@ def main(args: argparse.Namespace):
977
result_json["backend"] = backend
978
result_json["model_id"] = model_id
979
result_json["tokenizer_id"] = tokenizer_id
986
- result_json["best_of"] = args.best_of
980
result_json["num_prompts"] = args.num_prompts
981
982
# Metadata
@@ -1081,13 +1074,6 @@ if __name__ == "__main__":
1074
help=
1075
"Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
1076
)
1084
- parser.add_argument(
1085
- "--best-of",
1086
- type=int,
1087
- default=1,
1088
- help="Generates `best_of` sequences per prompt and "
1089
- "returns the best one.",
1090
- )
1077
parser.add_argument("--use-beam-search", action="store_true")
1078
parser.add_argument(
1079
"--num-prompts",
examples/offline_inference/llm_engine_example.py
-1
@@ -15,7 +15,6 @@ def create_test_prompts() -> list[tuple[str, SamplingParams]]:
15
SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
16
("What is the meaning of life?",
17
SamplingParams(n=2,
18
- best_of=5,
18
temperature=0.8,
19
top_p=0.95,
20
frequency_penalty=0.1)),
examples/online_serving/opentelemetry/dummy_client.py
-1
@@ -28,7 +28,6 @@ with tracer.start_as_current_span("client-span", kind=SpanKind.CLIENT) as span:
28
"model": "facebook/opt-125m",
29
"prompt": prompt,
30
"max_tokens": 10,
31
- "best_of": 20,
31
"n": 3,
32
"use_beam_search": "true",
33
"temperature": 0.0,
tests/core/test_scheduler.py
-4
@@ -617,7 +617,6 @@ def test_schedule_decode_blocks_to_copy_update():
617
num_gpu_blocks=16)
618
_, seq_group = create_dummy_prompt("1",
619
prompt_length=60,
620
- best_of=2,
620
block_size=block_size)
621
curr_loras = None
622
scheduler._allocate_and_set_running(seq_group)
@@ -686,7 +685,6 @@ def test_schedule_swapped_cannot_swap_in():
685
for i in range(2):
686
_, seq_group = create_dummy_prompt(str(i),
687
prompt_length=60,
689
- best_of=2,
688
block_size=block_size)
689
scheduler._allocate_and_set_running(seq_group)
690
append_new_token_seq_group(60, seq_group, 1)
@@ -717,7 +715,6 @@ def test_infeasible_swap():
715
for i in range(2):
716
_, seq_group = create_dummy_prompt(str(i),
717
prompt_length=60,
720
- best_of=2,
718
block_size=block_size)
719
scheduler._allocate_and_set_running(seq_group)
720
append_new_token_seq_group(60, seq_group, 1)
@@ -747,7 +744,6 @@ def test_schedule_swapped_blocks_to_copy():
744
curr_loras = None
745
_, seq_group = create_dummy_prompt("1",
746
prompt_length=60,
750
- best_of=2,
747
block_size=block_size)
748
scheduler._allocate_and_set_running(seq_group)
749
append_new_token_seq_group(60, seq_group, 1)
tests/core/utils.py
+13
-14
@@ -18,7 +18,6 @@ def create_dummy_prompt(
18
prompt_length: int = -1,
19
block_size: Optional[int] = None,
20
lora_request: Optional[LoRARequest] = None,
21
- best_of: int = 1,
21
prompt_tokens: Optional[list[int]] = None,
22
min_tokens: int = 0,
23
max_tokens: int = 16,
@@ -32,17 +31,19 @@ def create_dummy_prompt(
31
prompt_tokens = list(range(prompt_length))
32
33
prompt_str = " ".join([str(t) for t in prompt_tokens])
35
- prompt = Sequence(int(request_id),
36
- inputs=token_inputs(prompt_tokens, prompt=prompt_str),
37
- block_size=block_size)
38
- seq_group = SequenceGroup(request_id=request_id,
39
- seqs=[prompt],
40
- arrival_time=time.time(),
41
- sampling_params=SamplingParams(
42
- best_of=best_of,
43
- max_tokens=max_tokens,
44
- min_tokens=min_tokens),
45
- lora_request=lora_request)
34
+ prompt = Sequence(
35
+ int(request_id),
36
+ inputs=token_inputs(prompt_tokens, prompt=prompt_str),
37
+ block_size=block_size,
38
+ )
39
+ seq_group = SequenceGroup(
40
+ request_id=request_id,
41
+ seqs=[prompt],
42
+ arrival_time=time.time(),
43
+ sampling_params=SamplingParams(max_tokens=max_tokens,
44
+ min_tokens=min_tokens),
45
+ lora_request=lora_request,
46
+ )
47
48
return prompt, seq_group
49
@@ -72,7 +73,6 @@ def create_dummy_prompt_encoder_decoder(
73
encoder_prompt_length: int,
74
block_size: Optional[int] = None,
75
lora_request: Optional[LoRARequest] = None,
75
- best_of: int = 1,
76
) -> tuple[Sequence, Sequence, SequenceGroup]:
77
if not block_size:
78
block_size = decoder_prompt_length
@@ -102,7 +102,6 @@ def create_dummy_prompt_encoder_decoder(
102
103
seq_group = SequenceGroup(request_id=request_id,
104
seqs=[decoder_prompt],
105
- sampling_params=SamplingParams(best_of=best_of),
105
arrival_time=time.time(),
106
lora_request=lora_request,
107
encoder_seq=encoder_prompt)
tests/v1/sample/test_sampling_params_e2e.py
-8
@@ -25,14 +25,6 @@ def test_n_gt_1(model):
25
assert len(outputs[0].outputs) == 3
26
27
28
-def test_best_of(model):
29
- """Raise a ValueError since best_of is deprecated."""
30
-
31
- params = SamplingParams(n=2, best_of=3)
32
- with pytest.raises(ValueError):
33
- _ = model.generate(PROMPT, params)
34
-
35
-
28
def test_penalties(model):
29
"""Check that we do not get errors if applied."""
30
vllm/entrypoints/llm.py
+1
-4
@@ -97,10 +97,7 @@ class LLM:
97
throughput. However, if the value is too high, it may cause out-of-
98
memory (OOM) errors.
99
swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
100
- This can be used for temporarily storing the states of the requests
101
- when their `best_of` sampling parameters are larger than 1. If all
102
- requests will have `best_of=1`, you can safely set this to 0.
103
- Otherwise, too small values may cause out-of-memory (OOM) errors.
100
+ Too small values may cause out-of-memory (OOM) errors.
101
cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
102
the model weights. This virtually increases the GPU memory space
103
you can use to hold the model weights, at the cost of CPU-GPU data
vllm/entrypoints/openai/protocol.py
-4
@@ -242,7 +242,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
242
user: Optional[str] = None
243
244
# doc: begin-chat-completion-sampling-params
245
- best_of: Optional[int] = None
245
use_beam_search: bool = False
246
top_k: Optional[int] = None
247
min_p: Optional[float] = None
@@ -479,7 +478,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
478
479
return SamplingParams.from_optional(
480
n=self.n,
482
- best_of=self.best_of,
481
presence_penalty=self.presence_penalty,
482
frequency_penalty=self.frequency_penalty,
483
repetition_penalty=repetition_penalty,
@@ -650,7 +648,6 @@ class CompletionRequest(OpenAIBaseModel):
648
# https://platform.openai.com/docs/api-reference/completions/create
649
model: Optional[str] = None
650
prompt: Union[list[int], list[list[int]], str, list[str]]
653
- best_of: Optional[int] = None
651
echo: Optional[bool] = False
652
frequency_penalty: Optional[float] = 0.0
653
logit_bias: Optional[dict[str, float]] = None
@@ -848,7 +845,6 @@ class CompletionRequest(OpenAIBaseModel):
845
846
return SamplingParams.from_optional(
847
n=self.n,
851
- best_of=self.best_of,
848
presence_penalty=self.presence_penalty,
849
frequency_penalty=self.frequency_penalty,
850
repetition_penalty=repetition_penalty,
vllm/entrypoints/openai/serving_completion.py
+2
-6
@@ -168,12 +168,8 @@ class OpenAIServingCompletion(OpenAIServing):
168
model_name = self._get_model_name(request.model, lora_request)
169
num_prompts = len(engine_prompts)
170
171
- # Similar to the OpenAI API, when n != best_of, we do not stream the
172
- # results. In addition, we do not stream the results when use
173
- # beam search.
174
- stream = (request.stream
175
- and (request.best_of is None or request.n == request.best_of)
176
- and not request.use_beam_search)
171
+ # We do not stream the results when use beam search.
172
+ stream = (request.stream and not request.use_beam_search)
173
174
# Streaming response
175
if stream:
vllm/sampling_params.py
-24
@@ -116,10 +116,6 @@ class SamplingParams(
116
117
Args:
118
n: Number of output sequences to return for the given prompt.
119
- best_of: Number of output sequences that are generated from the prompt.
120
- From these `best_of` sequences, the top `n` sequences are returned.
121
- `best_of` must be greater than or equal to `n`. By default,
122
- `best_of` is set to `n`.
119
presence_penalty: Float that penalizes new tokens based on whether they
120
appear in the generated text so far. Values > 0 encourage the model
121
to use new tokens, while values < 0 encourage the model to repeat
@@ -187,7 +183,6 @@ class SamplingParams(
183
"""
184
185
n: int = 1
190
- best_of: Optional[int] = None
186
_real_n: Optional[int] = None
187
presence_penalty: float = 0.0
188
frequency_penalty: float = 0.0
@@ -231,7 +226,6 @@ class SamplingParams(
226
@staticmethod
227
def from_optional(
228
n: Optional[int] = 1,
234
- best_of: Optional[int] = None,
229
presence_penalty: Optional[float] = 0.0,
230
frequency_penalty: Optional[float] = 0.0,
231
repetition_penalty: Optional[float] = 1.0,
@@ -270,7 +264,6 @@ class SamplingParams(
264
265
return SamplingParams(
266
n=1 if n is None else n,
273
- best_of=best_of,
267
presence_penalty=0.0
268
if presence_penalty is None else presence_penalty,
269
frequency_penalty=0.0
@@ -303,20 +296,6 @@ class SamplingParams(
296
)
297
298
def __post_init__(self) -> None:
306
- # how we deal with `best_of``:
307
- # if `best_of`` is not set, we default to `n`;
308
- # if `best_of`` is set, we set `n`` to `best_of`,
309
- # and set `_real_n`` to the original `n`.
310
- # when we return the result, we will check
311
- # if we need to return `n` or `_real_n` results
312
- if self.best_of:
313
- if self.best_of < self.n:
314
- raise ValueError(
315
- f"best_of must be greater than or equal to n, "
316
- f"got n={self.n} and best_of={self.best_of}.")
317
- if not self._real_n:
318
- self._real_n = self.n
319
- self.n = self.best_of
299
300
if 0 < self.temperature < _MAX_TEMP:
301
logger.warning(
@@ -423,9 +402,6 @@ class SamplingParams(
402
raise ValueError(
403
"stop strings are only supported when detokenize is True. "
404
"Set detokenize=True to use stop.")
426
- if self.best_of != self._real_n and self.output_kind == (
427
- RequestOutputKind.DELTA):
428
- raise ValueError("best_of must equal n to use output_kind=DELTA")
405
406
def _verify_greedy_sampling(self) -> None:
407
if self.n > 1:
vllm/v1/engine/processor.py
-3
@@ -93,9 +93,6 @@ class Processor:
93
self,
94
params: SamplingParams,
95
) -> None:
96
- # Best of not yet supported.
97
- if params.best_of:
98
- raise ValueError("VLLM V1 does not yet support best_of.")
96
# Bad words not yet supported.
97
if params.bad_words:
98
raise ValueError("VLLM V1 does not yet support bad_words.")