A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[core] remove beam search from the core (#9105)
youkaichao committed
Oct 6, 2024 at 22:47 UTC
18b296fdb2248e8a65bf005e7193ebd523b875b6
25 files changed
+98
-596
benchmarks/backend_request_func.py
-6
@@ -23,7 +23,6 @@ class RequestFuncInput:
23
output_len: int
24
model: str
25
best_of: int = 1
26
- use_beam_search: bool = False
26
logprobs: Optional[int] = None
27
multi_modal_content: Optional[dict] = None
28
ignore_eos: bool = False
@@ -49,7 +48,6 @@ async def async_request_tgi(
48
assert api_url.endswith("generate_stream")
49
50
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
52
- assert not request_func_input.use_beam_search
51
params = {
52
"best_of": request_func_input.best_of,
53
"max_new_tokens": request_func_input.output_len,
@@ -121,7 +119,6 @@ async def async_request_trt_llm(
119
assert api_url.endswith("generate_stream")
120
121
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
124
- assert not request_func_input.use_beam_search
122
assert request_func_input.best_of == 1
123
payload = {
124
"accumulate_tokens": True,
@@ -187,7 +184,6 @@ async def async_request_deepspeed_mii(
184
) -> RequestFuncOutput:
185
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
186
assert request_func_input.best_of == 1
190
- assert not request_func_input.use_beam_search
187
188
payload = {
189
"prompt": request_func_input.prompt,
@@ -235,7 +231,6 @@ async def async_request_openai_completions(
231
), "OpenAI Completions API URL must end with 'completions' or 'profile'."
232
233
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
238
- assert not request_func_input.use_beam_search
234
payload = {
235
"model": request_func_input.model,
236
"prompt": request_func_input.prompt,
@@ -317,7 +312,6 @@ async def async_request_openai_chat_completions(
312
), "OpenAI Chat Completions API URL must end with 'chat/completions'."
313
314
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
320
- assert not request_func_input.use_beam_search
315
content = [{"type": "text", "text": request_func_input.prompt}]
316
if request_func_input.multi_modal_content:
317
content.append(request_func_input.multi_modal_content)
benchmarks/benchmark_latency.py
+1
-2
@@ -51,9 +51,8 @@ def main(args: argparse.Namespace):
51
52
sampling_params = SamplingParams(
53
n=args.n,
54
- temperature=0.0 if args.use_beam_search else 1.0,
54
+ temperature=1.0,
55
top_p=1.0,
56
- use_beam_search=args.use_beam_search,
56
ignore_eos=True,
57
max_tokens=args.output_len,
58
)
benchmarks/benchmark_prioritization.py
+11
-13
@@ -68,7 +68,6 @@ def run_vllm(
68
tensor_parallel_size: int,
69
seed: int,
70
n: int,
71
- use_beam_search: bool,
71
trust_remote_code: bool,
72
dtype: str,
73
max_model_len: Optional[int],
@@ -114,9 +113,8 @@ def run_vllm(
113
sampling_params.append(
114
SamplingParams(
115
n=n,
117
- temperature=0.0 if use_beam_search else 1.0,
116
+ temperature=1.0,
117
top_p=1.0,
119
- use_beam_search=use_beam_search,
118
ignore_eos=True,
119
max_tokens=output_len,
120
))
@@ -144,15 +142,16 @@ def main(args: argparse.Namespace):
142
args.output_len)
143
144
if args.backend == "vllm":
147
- elapsed_time = run_vllm(
148
- requests, args.model, args.tokenizer, args.quantization,
149
- args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
150
- args.trust_remote_code, args.dtype, args.max_model_len,
151
- args.enforce_eager, args.kv_cache_dtype,
152
- args.quantization_param_path, args.device,
153
- args.enable_prefix_caching, args.enable_chunked_prefill,
154
- args.max_num_batched_tokens, args.gpu_memory_utilization,
155
- args.download_dir)
145
+ elapsed_time = run_vllm(requests, args.model, args.tokenizer,
146
+ args.quantization, args.tensor_parallel_size,
147
+ args.seed, args.n, args.trust_remote_code,
148
+ args.dtype, args.max_model_len,
149
+ args.enforce_eager, args.kv_cache_dtype,
150
+ args.quantization_param_path, args.device,
151
+ args.enable_prefix_caching,
152
+ args.enable_chunked_prefill,
153
+ args.max_num_batched_tokens,
154
+ args.gpu_memory_utilization, args.download_dir)
155
else:
156
raise ValueError(f"Unknown backend: {args.backend}")
157
total_num_tokens = sum(prompt_len + output_len
@@ -203,7 +202,6 @@ if __name__ == "__main__":
202
type=int,
203
default=1,
204
help="Number of generated sequences per prompt.")
206
- parser.add_argument("--use-beam-search", action="store_true")
205
parser.add_argument("--num-prompts",
206
type=int,
207
default=200,
benchmarks/benchmark_serving.py
-7
@@ -391,7 +391,6 @@ async def benchmark(
391
input_requests: List[Tuple[str, int, int]],
392
logprobs: Optional[int],
393
best_of: int,
394
- use_beam_search: bool,
394
request_rate: float,
395
disable_tqdm: bool,
396
profile: bool,
@@ -419,7 +418,6 @@ async def benchmark(
418
output_len=test_output_len,
419
logprobs=logprobs,
420
best_of=best_of,
422
- use_beam_search=use_beam_search,
421
multi_modal_content=test_mm_content,
422
ignore_eos=ignore_eos,
423
)
@@ -441,7 +439,6 @@ async def benchmark(
439
output_len=test_output_len,
440
logprobs=logprobs,
441
best_of=best_of,
444
- use_beam_search=use_beam_search,
442
multi_modal_content=test_mm_content,
443
)
444
profile_output = await request_func(request_func_input=profile_input)
@@ -464,7 +461,6 @@ async def benchmark(
461
output_len=output_len,
462
logprobs=logprobs,
463
best_of=best_of,
467
- use_beam_search=use_beam_search,
464
multi_modal_content=mm_content,
465
)
466
tasks.append(
@@ -483,7 +479,6 @@ async def benchmark(
479
output_len=test_output_len,
480
logprobs=logprobs,
481
best_of=best_of,
486
- use_beam_search=use_beam_search,
482
)
483
profile_output = await request_func(request_func_input=profile_input)
484
if profile_output.success:
@@ -679,7 +674,6 @@ def main(args: argparse.Namespace):
674
input_requests=input_requests,
675
logprobs=args.logprobs,
676
best_of=args.best_of,
682
- use_beam_search=args.use_beam_search,
677
request_rate=args.request_rate,
678
disable_tqdm=args.disable_tqdm,
679
profile=args.profile,
@@ -701,7 +695,6 @@ def main(args: argparse.Namespace):
695
result_json["model_id"] = model_id
696
result_json["tokenizer_id"] = tokenizer_id
697
result_json["best_of"] = args.best_of
704
- result_json["use_beam_search"] = args.use_beam_search
698
result_json["num_prompts"] = args.num_prompts
699
700
# Metadata
benchmarks/benchmark_throughput.py
+9
-20
@@ -73,7 +73,6 @@ def run_vllm(
73
tensor_parallel_size: int,
74
seed: int,
75
n: int,
76
- use_beam_search: bool,
76
trust_remote_code: bool,
77
dtype: str,
78
max_model_len: Optional[int],
@@ -91,7 +90,6 @@ def run_vllm(
90
download_dir: Optional[str] = None,
91
load_format: str = EngineArgs.load_format,
92
disable_async_output_proc: bool = False,
94
- use_new_beam_search_impl: bool = False,
93
) -> float:
94
from vllm import LLM, SamplingParams
95
llm = LLM(
@@ -127,19 +125,19 @@ def run_vllm(
125
sampling_params.append(
126
SamplingParams(
127
n=n,
130
- temperature=0.0 if use_beam_search else 1.0,
128
+ temperature=1.0,
129
top_p=1.0,
132
- use_beam_search=use_beam_search,
130
ignore_eos=True,
131
max_tokens=output_len,
132
))
133
137
- if not use_new_beam_search_impl:
134
+ use_beam_search = False
135
+
136
+ if not use_beam_search:
137
start = time.perf_counter()
138
llm.generate(prompts, sampling_params, use_tqdm=True)
139
end = time.perf_counter()
140
else:
142
- assert use_beam_search
141
prompts = [prompt for prompt, _, _ in requests]
142
# output_len should be the same for all requests.
143
output_len = requests[0][2]
@@ -165,7 +163,6 @@ async def run_vllm_async(
163
tensor_parallel_size: int,
164
seed: int,
165
n: int,
168
- use_beam_search: bool,
166
trust_remote_code: bool,
167
dtype: str,
168
max_model_len: Optional[int],
@@ -224,9 +221,8 @@ async def run_vllm_async(
221
sampling_params.append(
222
SamplingParams(
223
n=n,
227
- temperature=0.0 if use_beam_search else 1.0,
224
+ temperature=1.0,
225
top_p=1.0,
229
- use_beam_search=use_beam_search,
226
ignore_eos=True,
227
max_tokens=output_len,
228
))
@@ -248,11 +244,9 @@ def run_hf(
244
model: str,
245
tokenizer: PreTrainedTokenizerBase,
246
n: int,
251
- use_beam_search: bool,
247
max_batch_size: int,
248
trust_remote_code: bool,
249
) -> float:
255
- assert not use_beam_search
250
llm = AutoModelForCausalLM.from_pretrained(
251
model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
252
if llm.config.model_type == "llama":
@@ -284,7 +278,7 @@ def run_hf(
278
padding=True).input_ids
279
llm_outputs = llm.generate(
280
input_ids=input_ids.cuda(),
287
- do_sample=not use_beam_search,
281
+ do_sample=True,
282
num_return_sequences=n,
283
temperature=1.0,
284
top_p=1.0,
@@ -340,7 +334,7 @@ def main(args: argparse.Namespace):
334
if args.backend == "vllm":
335
run_args = [
336
requests, args.model, args.tokenizer, args.quantization,
343
- args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
337
+ args.tensor_parallel_size, args.seed, args.n,
338
args.trust_remote_code, args.dtype, args.max_model_len,
339
args.enforce_eager, args.kv_cache_dtype,
340
args.quantization_param_path, args.device,
@@ -355,12 +349,11 @@ def main(args: argparse.Namespace):
349
run_args.append(args.disable_frontend_multiprocessing)
350
elapsed_time = uvloop.run(run_vllm_async(*run_args))
351
else:
358
- elapsed_time = run_vllm(*run_args, args.use_new_beam_search_impl)
352
+ elapsed_time = run_vllm(*run_args)
353
elif args.backend == "hf":
354
assert args.tensor_parallel_size == 1
355
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
362
- args.use_beam_search, args.hf_max_batch_size,
363
- args.trust_remote_code)
356
+ args.hf_max_batch_size, args.trust_remote_code)
357
elif args.backend == "mii":
358
elapsed_time = run_mii(requests, args.model, args.tensor_parallel_size,
359
args.output_len)
@@ -414,8 +407,6 @@ if __name__ == "__main__":
407
type=int,
408
default=1,
409
help="Number of generated sequences per prompt.")
417
- parser.add_argument("--use-beam-search", action="store_true")
418
- parser.add_argument("--use-new-beam-search-impl", action="store_true")
410
parser.add_argument("--num-prompts",
411
type=int,
412
default=1000,
@@ -570,8 +561,6 @@ if __name__ == "__main__":
561
raise ValueError("dtype must be auto for MII backend.")
562
if args.n != 1:
563
raise ValueError("n must be 1 for MII backend.")
573
- if args.use_beam_search:
574
- raise ValueError("Beam search is not supported for MII backend.")
564
if args.quantization is not None:
565
raise ValueError("Quantization is only for vLLM backend.")
566
if args.hf_max_batch_size is not None:
examples/llm_engine_example.py
-3
@@ -18,9 +18,6 @@ def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
18
temperature=0.8,
19
top_p=0.95,
20
frequency_penalty=0.1)),
21
- ("It is only with the heart that one can see rightly",
22
- SamplingParams(n=3, best_of=3, use_beam_search=True,
23
- temperature=0.0)),
21
]
22
23
examples/multilora_inference.py
-18
@@ -43,15 +43,6 @@ def create_test_prompts(
43
max_tokens=128,
44
stop_token_ids=[32003]),
45
LoRARequest("sql-lora", 1, lora_path)),
46
- (
47
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
48
- SamplingParams(n=3,
49
- best_of=3,
50
- use_beam_search=True,
51
- temperature=0,
52
- max_tokens=128,
53
- stop_token_ids=[32003]),
54
- LoRARequest("sql-lora", 1, lora_path)),
46
(
47
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
48
SamplingParams(temperature=0.0,
@@ -60,15 +51,6 @@ def create_test_prompts(
51
max_tokens=128,
52
stop_token_ids=[32003]),
53
LoRARequest("sql-lora2", 2, lora_path)),
63
- (
64
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
65
- SamplingParams(n=3,
66
- best_of=3,
67
- use_beam_search=True,
68
- temperature=0,
69
- max_tokens=128,
70
- stop_token_ids=[32003]),
71
- LoRARequest("sql-lora", 1, lora_path)),
54
]
55
56
tests/basic_correctness/test_preemption.py
+2
-112
@@ -23,11 +23,9 @@ MODELS = [
23
@pytest.fixture(scope="module", autouse=True)
24
def check_settings():
25
assert ENABLE_ARTIFICIAL_PREEMPT is True, (
26
- "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1, "
27
- "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1. "
26
+ "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1."
27
"`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 "
29
- "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 pytest "
30
- "tests/basic_correctness/test_preemption.py`")
28
+ "pytest tests/basic_correctness/test_preemption.py`")
29
30
31
@pytest.fixture
@@ -137,114 +135,6 @@ def test_preemption(
135
assert total_preemption == total_recorded_preemption
136
137
140
-@pytest.mark.parametrize("model", MODELS)
141
-@pytest.mark.parametrize("dtype", ["float"])
142
-@pytest.mark.parametrize("max_tokens", [96])
143
-@pytest.mark.parametrize("beam_width", [4])
144
-def test_swap(
145
- caplog_vllm,
146
- hf_runner,
147
- vllm_runner,
148
- example_prompts,
149
- model: str,
150
- dtype: str,
151
- max_tokens: int,
152
- beam_width: int,
153
- worker_use_ray: bool,
154
-) -> None:
155
- """Use beam search enables swapping."""
156
- example_prompts = example_prompts[:1]
157
- with hf_runner(model, dtype=dtype) as hf_model:
158
- hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
159
- max_tokens)
160
-
161
- with vllm_runner(
162
- model,
163
- dtype=dtype,
164
- swap_space=10,
165
- disable_log_stats=False,
166
- worker_use_ray=worker_use_ray,
167
- ) as vllm_model:
168
- vllm_outputs = vllm_model.generate_beam_search(example_prompts,
169
- beam_width, max_tokens)
170
- assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
171
- < ARTIFICIAL_PREEMPTION_MAX_CNT)
172
- total_preemption = (
173
- vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)
174
-
175
- for i in range(len(example_prompts)):
176
- hf_output_ids, _ = hf_outputs[i]
177
- vllm_output_ids, _ = vllm_outputs[i]
178
- assert len(hf_output_ids) == len(vllm_output_ids)
179
- for j in range(len(hf_output_ids)):
180
- assert hf_output_ids[j] == vllm_output_ids[j], (
181
- f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
182
- f"vLLM: {vllm_output_ids}")
183
-
184
- assert ("is preempted by PreemptionMode.SWAP mode because there "
185
- "is not enough KV cache space." in caplog_vllm.text)
186
- # Ensure the count bucket of request-level histogram metrics matches
187
- # the number of requests as a simple sanity check to ensure metrics are
188
- # generated
189
- preemption_metrics = None
190
- for m in REGISTRY.collect():
191
- if m.name == "vllm:num_preemptions":
192
- preemption_metrics = m
193
- assert preemption_metrics is not None
194
- total_recorded_preemption = 0
195
- for sample in preemption_metrics.samples:
196
- total_recorded_preemption += sample.value
197
- assert total_preemption == total_recorded_preemption
198
-
199
-
200
-@pytest.mark.parametrize("model", MODELS)
201
-@pytest.mark.parametrize("dtype", ["float"])
202
-@pytest.mark.parametrize("max_tokens", [96])
203
-@pytest.mark.parametrize("beam_width", [4])
204
-@pytest.mark.parametrize("use_v2_block_manager", [True, False])
205
-def test_swap_infeasible(
206
- vllm_runner,
207
- example_prompts,
208
- model: str,
209
- dtype: str,
210
- max_tokens: int,
211
- beam_width: int,
212
- worker_use_ray: bool,
213
- use_v2_block_manager: bool,
214
-) -> None:
215
- """Verify infeasible swap request will be ignored."""
216
- BLOCK_SIZE = 16
217
- prefill_blocks = 2
218
- decode_blocks = max_tokens // BLOCK_SIZE
219
- example_prompts = example_prompts[:1]
220
- with vllm_runner(
221
- model,
222
- dtype=dtype,
223
- swap_space=10,
224
- block_size=BLOCK_SIZE,
225
- # Since beam search have more than 1 sequence, prefill +
226
- # decode blocks are not enough to finish.
227
- num_gpu_blocks_override=prefill_blocks + decode_blocks,
228
- max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
229
- worker_use_ray=worker_use_ray,
230
- use_v2_block_manager=use_v2_block_manager,
231
- ) as vllm_model:
232
- sampling_params = SamplingParams(n=beam_width,
233
- use_beam_search=True,
234
- temperature=0.0,
235
- max_tokens=max_tokens,
236
- ignore_eos=True)
237
- req_outputs = vllm_model.model.generate(
238
- example_prompts,
239
- sampling_params=sampling_params,
240
- )
241
- assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
242
- < ARTIFICIAL_PREEMPTION_MAX_CNT)
243
-
244
- # Verify the request is ignored and not hang.
245
- assert req_outputs[0].outputs[0].finish_reason == "length"
246
-
247
-
138
@pytest.mark.parametrize("model", MODELS)
139
@pytest.mark.parametrize("dtype", ["float"])
140
@pytest.mark.parametrize("max_tokens", [96])
tests/conftest.py
-14
@@ -782,7 +782,6 @@ class VllmRunner:
782
List[TokensTextLogprobsPromptLogprobs]]:
783
greedy_logprobs_params = SamplingParams(
784
temperature=0.0,
785
- use_beam_search=False,
785
max_tokens=max_tokens,
786
logprobs=num_logprobs,
787
prompt_logprobs=(num_prompt_logprobs),
@@ -795,19 +794,6 @@ class VllmRunner:
794
encoder_decoder_prompts, greedy_logprobs_params)
795
796
def generate_beam_search(
798
- self,
799
- prompts: List[str],
800
- beam_width: int,
801
- max_tokens: int,
802
- ) -> List[Tuple[List[List[int]], List[str]]]:
803
- beam_search_params = SamplingParams(n=beam_width,
804
- use_beam_search=True,
805
- temperature=0.0,
806
- max_tokens=max_tokens)
807
- outputs = self.generate(prompts, beam_search_params)
808
- return outputs
809
-
810
- def generate_beam_search_new(
797
self,
798
prompts: Union[List[str], List[List[int]]],
799
beam_width: int,
tests/core/block/e2e/test_correctness.py
-67
@@ -85,73 +85,6 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
85
assert baseline_token_ids == test_token_ids
86
87
88
-@pytest.mark.parametrize(
89
- "common_llm_kwargs",
90
- [{
91
- # Use a small model for a fast test.
92
- "model": "facebook/opt-125m",
93
-
94
- # skip cuda graph creation for fast test.
95
- "enforce_eager": True,
96
-
97
- # Use a large block size to trigger more copy-on-writes.
98
- "block_size": 32,
99
- }])
100
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
101
-@pytest.mark.parametrize("baseline_llm_kwargs", [{
102
- "use_v2_block_manager": False
103
-}])
104
-@pytest.mark.parametrize("test_llm_kwargs", [{
105
- "use_v2_block_manager": True,
106
- "preemption_mode": "swap"
107
-}, {
108
- "use_v2_block_manager": True,
109
- "preemption_mode": "recompute"
110
-}])
111
-@pytest.mark.parametrize("batch_size", [10])
112
-@pytest.mark.parametrize("seed", [1])
113
-def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
114
- test_llm_generator, batch_size):
115
- """Verify beam search equality with block manager v1 and v2.
116
-
117
- This requires copy-on-writes; if the v1 and v2 output is the same, then
118
- we have some confidence cow is working.
119
- """
120
- output_len = 128
121
- temperature = 0.0
122
-
123
- prompts = [
124
- "Hello, my name is",
125
- "The president of the United States is",
126
- "The capital of France is",
127
- "The future of AI is",
128
- ]
129
-
130
- prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
131
-
132
- sampling_params = SamplingParams(
133
- max_tokens=output_len,
134
- ignore_eos=True,
135
- temperature=temperature,
136
- use_beam_search=True,
137
- best_of=2,
138
- )
139
-
140
- print('Getting token ids from block manager v1')
141
- baseline_token_ids = get_token_ids_from_llm_generator(
142
- baseline_llm_generator, prompts, sampling_params)
143
-
144
- print('Getting token ids from block manager v2')
145
- test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
146
- prompts, sampling_params)
147
-
148
- for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
149
- test_token_ids):
150
- assert expected_token_ids == actual_token_ids
151
-
152
- assert baseline_token_ids == test_token_ids
153
-
154
-
88
@pytest.mark.parametrize(
89
"common_llm_kwargs",
90
[{
tests/core/utils.py
+1
-6
@@ -13,7 +13,6 @@ def create_dummy_prompt(
13
prompt_length: int,
14
block_size: Optional[int] = None,
15
lora_request: Optional[LoRARequest] = None,
16
- use_beam_search: bool = False,
16
best_of: int = 1,
17
prompt_tokens: Optional[List[int]] = None,
18
min_tokens: int = 0,
@@ -37,7 +36,6 @@ def create_dummy_prompt(
36
seqs=[prompt],
37
arrival_time=time.time(),
38
sampling_params=SamplingParams(
40
- use_beam_search=use_beam_search,
39
best_of=best_of,
40
max_tokens=max_tokens,
41
min_tokens=min_tokens),
@@ -52,7 +50,6 @@ def create_dummy_prompt_encoder_decoder(
50
encoder_prompt_length: int,
51
block_size: Optional[int] = None,
52
lora_request: Optional[LoRARequest] = None,
55
- use_beam_search: bool = False,
53
best_of: int = 1,
54
) -> Tuple[Sequence, Sequence, SequenceGroup]:
55
if not block_size:
@@ -85,9 +82,7 @@ def create_dummy_prompt_encoder_decoder(
82
from_decoder_prompt=False)
83
seq_group = SequenceGroup(request_id=request_id,
84
seqs=[decoder_prompt],
88
- sampling_params=SamplingParams(
89
- use_beam_search=use_beam_search,
90
- best_of=best_of),
85
+ sampling_params=SamplingParams(best_of=best_of),
86
arrival_time=time.time(),
87
lora_request=lora_request,
88
encoder_seq=encoder_prompt)
tests/samplers/test_beam_search.py
+2
-2
@@ -33,8 +33,8 @@ def test_beam_search_single_input(
33
max_tokens)
34
35
with vllm_runner(model, dtype=dtype) as vllm_model:
36
- vllm_outputs = vllm_model.generate_beam_search_new(
37
- example_prompts, beam_width, max_tokens)
36
+ vllm_outputs = vllm_model.generate_beam_search(example_prompts,
37
+ beam_width, max_tokens)
38
39
for i in range(len(example_prompts)):
40
hf_output_ids, hf_output_texts = hf_outputs[i]
tests/samplers/test_sampler.py
+2
-28
@@ -159,26 +159,6 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
159
assert first_sampler_output == second_sampler_output
160
161
162
-@pytest.mark.parametrize("seed", RANDOM_SEEDS)
163
-@pytest.mark.parametrize("device", CUDA_DEVICES)
164
-def test_sampler_all_beam(seed: int, device: str):
165
- set_random_seed(seed)
166
- torch.set_default_device(device)
167
- batch_size = random.randint(1, 256)
168
- _, fake_logits, sampler = _prepare_test(batch_size)
169
-
170
- sampling_params = SamplingParams(
171
- temperature=0,
172
- best_of=2,
173
- use_beam_search=True,
174
- )
175
- _do_sample(batch_size, fake_logits, sampler, sampling_params, device)
176
- # no assertion here as I am not sure how to determine whether
177
- # the outputs are expected - in other words, this just tests
178
- # whether there are no exceptions in the sampler
179
- # when handling an all-beam search case.
180
-
181
-
162
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
163
@pytest.mark.parametrize("device", CUDA_DEVICES)
164
def test_sampler_min_tokens_penalty(seed: int, device: str):
@@ -479,7 +459,7 @@ def test_sampler_mixed(seed: int, device: str):
459
seq_lens: List[int] = []
460
for i in range(batch_size):
461
expected: Optional[List[int]] = None
482
- sampling_type = random.randint(0, 3)
462
+ sampling_type = random.randint(0, 2)
463
if sampling_type == 0:
464
sampling_params = SamplingParams(temperature=0)
465
expected = [int(torch.argmax(fake_logits[i], dim=-1).item())]
@@ -498,10 +478,7 @@ def test_sampler_mixed(seed: int, device: str):
478
for idx in range(n):
479
fake_logits[i, i + idx] = 1e2
480
expected = list(range(i, i + n))
501
- else:
502
- sampling_params = SamplingParams(temperature=0,
503
- use_beam_search=True,
504
- best_of=2)
481
+
482
expected_tokens.append(expected)
483
seq_group_metadata_list.append(
484
SequenceGroupMetadata(
@@ -530,9 +507,6 @@ def test_sampler_mixed(seed: int, device: str):
507
zip(sampler_output, seq_group_metadata_list)):
508
assert metadata.sampling_params is not None
509
533
- if metadata.sampling_params.use_beam_search:
534
- continue
535
-
510
if (metadata.sampling_params.seed is not None
511
and expected_tokens[i] is None):
512
# Record seeded random result to compare with results of
vllm/core/scheduler.py
+2
-2
@@ -1202,9 +1202,9 @@ class Scheduler:
1202
seq_group=seq_group, num_lookahead_slots=num_lookahead_slots)
1203
1204
def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1205
+ # TODO: does it work with parallel sampling?
1206
no_beam_search = seq_group.sampling_params is None or (
1206
- seq_group.sampling_params.best_of == 1
1207
- and not seq_group.sampling_params.use_beam_search)
1207
+ seq_group.sampling_params.best_of == 1)
1208
return no_beam_search
1209
1210
def schedule(
vllm/engine/async_llm_engine.py
+9
-7
@@ -33,7 +33,7 @@ from vllm.sequence import ExecuteModelRequest
33
from vllm.transformers_utils.tokenizer import AnyTokenizer
34
from vllm.usage.usage_lib import UsageContext
35
from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36
- random_uuid, weak_bind)
36
+ get_beam_search_score, random_uuid, weak_bind)
37
38
logger = init_logger(__name__)
39
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -1050,6 +1050,12 @@ class AsyncLLMEngine:
1050
max_tokens = params.max_tokens
1051
ignore_eos = params.ignore_eos
1052
temperature = params.temperature
1053
+ length_penalty = params.length_penalty
1054
+
1055
+ def sort_beams_key(x: BeamSearchSequence) -> float:
1056
+ return get_beam_search_score(x.tokens, x.cum_logprob,
1057
+ tokenizer.eos_token_id,
1058
+ length_penalty)
1059
1060
tokenizer = await self.get_tokenizer()
1061
tokenizedPrompt = prompt if isinstance(
@@ -1103,15 +1109,11 @@ class AsyncLLMEngine:
1109
else:
1110
new_beams.append(new_beam)
1111
1106
- sorted_beams = sorted(new_beams,
1107
- key=lambda x: x.cum_logprob,
1108
- reverse=True)
1112
+ sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
1113
all_beams = sorted_beams[:beam_width]
1114
1115
completed.extend(all_beams)
1112
- sorted_completed = sorted(completed,
1113
- key=lambda x: x.cum_logprob,
1114
- reverse=True)
1116
+ sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
1117
best_beams = sorted_completed[:beam_width]
1118
1119
for beam in best_beams:
vllm/engine/output_processor/single_step.py
+7
-157
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Optional, Tuple, Union
1
+from typing import Dict, List, Tuple
2
3
from vllm.config import SchedulerConfig
4
from vllm.core.scheduler import Scheduler
@@ -6,7 +6,6 @@ from vllm.engine.output_processor.interfaces import (
6
SequenceGroupOutputProcessor)
7
from vllm.engine.output_processor.stop_checker import StopChecker
8
from vllm.logger import init_logger
9
-from vllm.sampling_params import SamplingParams
9
from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
10
SequenceOutput, SequenceStatus)
11
from vllm.transformers_utils.detokenizer import Detokenizer
@@ -113,7 +112,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
112
outputs: SequenceGroupOutput,
113
is_async: bool) -> None:
114
sampling_params = seq_group.sampling_params
116
- if sampling_params.best_of == 1 and not sampling_params.use_beam_search:
115
+ if sampling_params.best_of == 1:
116
# only have one output sample
117
sample = outputs.samples[0]
118
# only have one sequence
@@ -142,7 +141,6 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
141
# Process samples
142
samples = outputs.samples
143
parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
145
- existing_finished_seqs = seq_group.get_finished_seqs()
144
parent_child_dict: Dict[int, List[SequenceOutput]] = {
145
parent_seq.seq_id: []
146
for parent_seq in parent_seqs
@@ -197,106 +195,9 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
195
lora_req=seq_group.lora_request,
196
)
197
200
- # Non-beam search case
201
- if not sampling_params.use_beam_search:
202
- # For newly created child sequences, add them to the sequence group
203
- # and fork them in block manager if they are not finished.
204
- for seq, parent in child_seqs:
205
- if seq is not parent:
206
- seq_group.add(seq)
207
- if not seq.is_finished():
208
- for scheduler in self.scheduler:
209
- scheduler.fork_seq(parent, seq)
210
-
211
- # Free the finished and selected parent sequences' memory in block
212
- # manager. Keep them in the sequence group as candidate output.
213
- # NOTE: we need to fork the new sequences before freeing the
214
- # old sequences.
215
- for seq, parent in child_seqs:
216
- if seq is parent and seq.is_finished():
217
- for scheduler in self.scheduler:
218
- scheduler.free_seq(seq)
219
- return
220
-
221
- # Beam search case
222
- # Select the child sequences to keep in the sequence group.
223
- selected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
224
- unselected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
225
- beam_width = sampling_params.best_of
226
- length_penalty = sampling_params.length_penalty
227
-
228
- # Select the newly finished sequences with the highest scores
229
- # to replace existing finished sequences.
230
- # Tuple of (seq, parent, is_new)
231
- existing_finished_seqs = [(seq, None, False)
232
- for seq in existing_finished_seqs]
233
- new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
234
- if seq.is_finished()]
235
- all_finished_seqs = existing_finished_seqs + new_finished_seqs
236
- # Sort the finished sequences by their scores.
237
- all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
238
- length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
239
- reverse=True)
240
- for seq, parent, is_new in all_finished_seqs[:beam_width]:
241
- if is_new:
242
- # A newly generated child sequence finishes and has a high
243
- # score, so we will add it into the sequence group.
244
- selected_child_seqs.append((seq, parent))
245
- for seq, parent, is_new in all_finished_seqs[beam_width:]:
246
- if is_new:
247
- # A newly generated child sequence finishes but has a low
248
- # score, so we will not add it into the sequence group.
249
- # Additionally, if this sequence is a continuation of a
250
- # parent sequence, we will need remove the parent sequence
251
- # from the sequence group.
252
- unselected_child_seqs.append((seq, parent))
253
- else:
254
- # An existing finished sequence has a low score, so we will
255
- # remove it from the sequence group.
256
- seq_group.remove(seq.seq_id)
257
-
258
- # select the top beam_width sequences from the running
259
- # sequences for the next iteration to continue the beam
260
- # search.
261
- running_child_seqs = [(seq, parent) for seq, parent in child_seqs
262
- if not seq.is_finished()]
263
- # Sort the running sequences by their scores.
264
- running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
265
- length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
266
- reverse=True)
267
-
268
- # Check if we can stop the beam search.
269
- if len(running_child_seqs) == 0:
270
- # No running sequences, stop the beam search.
271
- stop_beam_search = True
272
- elif len(all_finished_seqs) < beam_width:
273
- # Not enough finished sequences, continue the beam search.
274
- stop_beam_search = False
275
- else:
276
- # Check the early stopping criteria
277
- best_running_seq = running_child_seqs[0][0]
278
- current_worst_seq = all_finished_seqs[beam_width - 1][0]
279
- stop_beam_search = self._check_beam_search_early_stopping(
280
- sampling_params.early_stopping, sampling_params,
281
- best_running_seq, current_worst_seq)
282
-
283
- if stop_beam_search:
284
- # Stop the beam search and remove all the running sequences from
285
- # the sequence group.
286
- unselected_child_seqs.extend(running_child_seqs)
287
- else:
288
- # Continue the beam search and select the top beam_width sequences
289
- # to continue the beam search.
290
- selected_child_seqs.extend(running_child_seqs[:beam_width])
291
- # The remaining running sequences will not be used in the next
292
- # iteration. Again, if these sequences are continuations of
293
- # parent sequences, we will need to remove the parent sequences
294
- # from the sequence group.
295
- unselected_child_seqs.extend(running_child_seqs[beam_width:])
296
-
198
# For newly created child sequences, add them to the sequence group
199
# and fork them in block manager if they are not finished.
299
- for seq, parent in selected_child_seqs:
200
+ for seq, parent in child_seqs:
201
if seq is not parent:
202
seq_group.add(seq)
203
if not seq.is_finished():
@@ -305,61 +206,10 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
206
207
# Free the finished and selected parent sequences' memory in block
208
# manager. Keep them in the sequence group as candidate output.
308
- for seq, parent in selected_child_seqs:
209
+ # NOTE: we need to fork the new sequences before freeing the
210
+ # old sequences.
211
+ for seq, parent in child_seqs:
212
if seq is parent and seq.is_finished():
213
for scheduler in self.scheduler:
214
scheduler.free_seq(seq)
312
-
313
- # Remove the unselected parent sequences from the sequence group and
314
- # free their memory in block manager.
315
- for seq, parent in unselected_child_seqs:
316
- if seq is parent:
317
- # Remove the parent sequence if it is not selected for next
318
- # iteration
319
- seq_group.remove(seq.seq_id)
320
- for scheduler in self.scheduler:
321
- scheduler.free_seq(seq)
322
-
323
- def _check_beam_search_early_stopping(
324
- self,
325
- early_stopping: Union[bool, str],
326
- sampling_params: SamplingParams,
327
- best_running_seq: Sequence,
328
- current_worst_seq: Sequence,
329
- ) -> bool:
330
- assert sampling_params.use_beam_search
331
- length_penalty = sampling_params.length_penalty
332
- if early_stopping is True:
333
- return True
334
-
335
- current_worst_score = current_worst_seq.get_beam_search_score(
336
- length_penalty=length_penalty,
337
- eos_token_id=current_worst_seq.eos_token_id)
338
- if early_stopping is False:
339
- highest_attainable_score = best_running_seq.get_beam_search_score(
340
- length_penalty=length_penalty,
341
- eos_token_id=best_running_seq.eos_token_id)
342
- else:
343
- assert early_stopping == "never"
344
- if length_penalty > 0.0:
345
- # If length_penalty > 0.0, beam search will prefer longer
346
- # sequences. The highest attainable score calculation is
347
- # based on the longest possible sequence length in this case.
348
- max_possible_length = max(
349
- best_running_seq.get_prompt_len() +
350
- sampling_params.max_tokens,
351
- self.scheduler_config.max_model_len)
352
- highest_attainable_score = (
353
- best_running_seq.get_beam_search_score(
354
- length_penalty=length_penalty,
355
- eos_token_id=best_running_seq.eos_token_id,
356
- seq_len=max_possible_length))
357
- else:
358
- # Otherwise, beam search will prefer shorter sequences. The
359
- # highest attainable score calculation is based on the current
360
- # sequence length.
361
- highest_attainable_score = (
362
- best_running_seq.get_beam_search_score(
363
- length_penalty=length_penalty,
364
- eos_token_id=best_running_seq.eos_token_id))
365
- return current_worst_score >= highest_attainable_score
215
+ return
vllm/entrypoints/llm.py
+10
-3
@@ -28,7 +28,8 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
28
get_cached_tokenizer)
29
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
30
from vllm.usage.usage_lib import UsageContext
31
-from vllm.utils import Counter, deprecate_kwargs, is_list_of
31
+from vllm.utils import (Counter, deprecate_kwargs, get_beam_search_score,
32
+ is_list_of)
33
34
logger = init_logger(__name__)
35
@@ -404,6 +405,12 @@ class LLM:
405
max_tokens = params.max_tokens
406
temperature = params.temperature
407
ignore_eos = params.ignore_eos
408
+ length_penalty = params.length_penalty
409
+
410
+ def sort_beams_key(x: BeamSearchSequence) -> float:
411
+ return get_beam_search_score(x.tokens, x.cum_logprob,
412
+ tokenizer.eos_token_id,
413
+ length_penalty)
414
415
tokenizer = self.get_tokenizer()
416
# generate 2 * beam_width candidates at each step
@@ -466,7 +473,7 @@ class LLM:
473
else:
474
instance_new_beams.append(new_beam)
475
sorted_beams = sorted(instance_new_beams,
469
- key=lambda x: x.cum_logprob,
476
+ key=sort_beams_key,
477
reverse=True)
478
instance.beams = sorted_beams[:beam_width]
479
@@ -474,7 +481,7 @@ class LLM:
481
for instance in instances:
482
instance.completed.extend(instance.beams)
483
sorted_completed = sorted(instance.completed,
477
- key=lambda x: x.cum_logprob,
484
+ key=sort_beams_key,
485
reverse=True)
486
best_beams = sorted_completed[:beam_width]
487
vllm/entrypoints/openai/protocol.py
+2
-8
@@ -184,7 +184,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
184
min_p: float = 0.0
185
repetition_penalty: float = 1.0
186
length_penalty: float = 1.0
187
- early_stopping: bool = False
187
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
188
include_stop_str_in_output: bool = False
189
ignore_eos: bool = False
@@ -302,6 +301,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
301
max_tokens=max_tokens,
302
ignore_eos=self.ignore_eos,
303
temperature=temperature,
304
+ length_penalty=self.length_penalty,
305
)
306
307
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
@@ -345,12 +345,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
345
ignore_eos=self.ignore_eos,
346
max_tokens=max_tokens,
347
min_tokens=self.min_tokens,
348
- use_beam_search=self.use_beam_search,
349
- early_stopping=self.early_stopping,
348
skip_special_tokens=self.skip_special_tokens,
349
spaces_between_special_tokens=self.spaces_between_special_tokens,
350
include_stop_str_in_output=self.include_stop_str_in_output,
353
- length_penalty=self.length_penalty,
351
truncate_prompt_tokens=self.truncate_prompt_tokens,
352
output_kind=RequestOutputKind.DELTA if self.stream \
353
else RequestOutputKind.FINAL_ONLY,
@@ -518,7 +515,6 @@ class CompletionRequest(OpenAIBaseModel):
515
min_p: float = 0.0
516
repetition_penalty: float = 1.0
517
length_penalty: float = 1.0
521
- early_stopping: bool = False
518
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
519
include_stop_str_in_output: bool = False
520
ignore_eos: bool = False
@@ -597,6 +593,7 @@ class CompletionRequest(OpenAIBaseModel):
593
max_tokens=max_tokens,
594
ignore_eos=self.ignore_eos,
595
temperature=temperature,
596
+ length_penalty=self.length_penalty,
597
)
598
599
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
@@ -641,13 +638,10 @@ class CompletionRequest(OpenAIBaseModel):
638
ignore_eos=self.ignore_eos,
639
max_tokens=max_tokens if not echo_without_generation else 1,
640
min_tokens=self.min_tokens,
644
- use_beam_search=self.use_beam_search,
645
- early_stopping=self.early_stopping,
641
prompt_logprobs=prompt_logprobs,
642
skip_special_tokens=self.skip_special_tokens,
643
spaces_between_special_tokens=self.spaces_between_special_tokens,
644
include_stop_str_in_output=self.include_stop_str_in_output,
650
- length_penalty=self.length_penalty,
645
truncate_prompt_tokens=self.truncate_prompt_tokens,
646
output_kind=RequestOutputKind.DELTA if self.stream \
647
else RequestOutputKind.FINAL_ONLY,
vllm/envs.py
-5
@@ -63,7 +63,6 @@ if TYPE_CHECKING:
63
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
64
VLLM_USE_TRITON_AWQ: bool = False
65
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66
- VLLM_ALLOW_DEPRECATED_BEAM_SEARCH: bool = False
66
VLLM_SKIP_P2P_CHECK: bool = False
67
68
@@ -198,10 +197,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
197
lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
198
("true", "1")),
199
201
- # If set, allowing the use of deprecated beam search implementation
202
- "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH":
203
- lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BEAM_SEARCH", "0") == "1",
204
-
200
# Internal flag to enable Dynamo graph capture
201
"VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
202
lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
vllm/model_executor/layers/sampler.py
+2
-7
@@ -947,8 +947,6 @@ def get_logprobs(
947
# largest num logprobs in this API. If every logprobs is None, it will be
948
# set to -1.
949
largest_num_logprobs = -1
950
- # If beam search is enabled.
951
- use_beam_search = False
950
951
# Select indices to compute logprob from, ranks of token ids, and the top
952
# k token ids from logprobs.
@@ -981,8 +979,6 @@ def get_logprobs(
979
largest_num_logprobs = max(largest_num_logprobs,
980
sampling_params.logprobs)
981
984
- use_beam_search = use_beam_search or sampling_params.use_beam_search
985
-
982
assert len(next_token_ids) == len(query_indices)
983
984
if len(query_indices) == 0:
@@ -995,7 +991,7 @@ def get_logprobs(
991
992
# If largest_num_logprobs == -1, i.e. no logprobs are requested, we can
993
# skip the whole logprob calculation.
998
- if largest_num_logprobs >= 0 or use_beam_search:
994
+ if largest_num_logprobs >= 0:
995
query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
996
next_token_ids_gpu = torch.tensor(next_token_ids,
997
device=logprobs.device)
@@ -1121,13 +1117,12 @@ def _get_sampled_logprob_if_needed(
1117
"""Compute the sample logprob if needed."""
1118
seq_ids = seq_group.seq_ids
1119
num_logprobs = seq_group.sampling_params.logprobs
1124
- use_beam_search = seq_group.sampling_params.use_beam_search
1120
sampled_logprobs: SampleLogprobs = []
1121
next_token_ids, parent_seq_ids = sample_result
1122
1123
if seq_group.do_sample:
1124
assert len(next_token_ids) > 0
1130
- if num_logprobs is None and not use_beam_search:
1125
+ if num_logprobs is None:
1126
for next_token_id in next_token_ids:
1127
# Use a dummy logprob
1128
sampled_logprobs.append({next_token_id: Logprob(inf)})
vllm/outputs.py
+1
-5
@@ -142,11 +142,7 @@ class RequestOutput:
142
else:
143
# Get the top-n sequences.
144
n = sampling_params.n
145
- if sampling_params.use_beam_search:
146
- sorting_key = lambda seq: seq.get_beam_search_score(
147
- sampling_params.length_penalty)
148
- else:
149
- sorting_key = lambda seq: seq.get_cumulative_logprob()
145
+ sorting_key = lambda seq: seq.get_cumulative_logprob()
146
sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
147
top_n_seqs = sorted_seqs[:n]
148
vllm/sampling_params.py
+8
-65
@@ -10,7 +10,6 @@ import torch
10
from pydantic import BaseModel
11
from typing_extensions import Annotated
12
13
-import vllm.envs as envs
13
from vllm.logger import init_logger
14
15
logger = init_logger(__name__)
@@ -23,7 +22,6 @@ class SamplingType(IntEnum):
22
GREEDY = 0
23
RANDOM = 1
24
RANDOM_SEED = 2
26
- BEAM = 3
25
26
27
LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
@@ -134,16 +132,6 @@ class SamplingParams(
132
considered, relative to the probability of the most likely token.
133
Must be in [0, 1]. Set to 0 to disable this.
134
seed: Random seed to use for the generation.
137
- use_beam_search: Whether to use beam search instead of sampling.
138
- length_penalty: Float that penalizes sequences based on their length.
139
- Used in beam search.
140
- early_stopping: Controls the stopping condition for beam search. It
141
- accepts the following values: `True`, where the generation stops as
142
- soon as there are `best_of` complete candidates; `False`, where an
143
- heuristic is applied and the generation stops when is it very
144
- unlikely to find better candidates; `"never"`, where the beam search
145
- procedure only stops when there cannot be better candidates
146
- (canonical beam search algorithm).
135
stop: List of strings that stop the generation when they are generated.
136
The returned output will not contain the stop strings.
137
stop_token_ids: List of tokens that stop the generation when they are
@@ -193,9 +181,6 @@ class SamplingParams(
181
top_k: int = -1
182
min_p: float = 0.0
183
seed: Optional[int] = None
196
- use_beam_search: bool = False
197
- length_penalty: float = 1.0
198
- early_stopping: Union[bool, str] = False
184
stop: Optional[Union[str, List[str]]] = None
185
stop_token_ids: Optional[List[int]] = None
186
ignore_eos: bool = False
@@ -238,9 +223,6 @@ class SamplingParams(
223
top_k: int = -1,
224
min_p: float = 0.0,
225
seed: Optional[int] = None,
241
- use_beam_search: bool = False,
242
- length_penalty: float = 1.0,
243
- early_stopping: Union[bool, str] = False,
226
stop: Optional[Union[str, List[str]]] = None,
227
stop_token_ids: Optional[List[int]] = None,
228
include_stop_str_in_output: bool = False,
@@ -280,9 +262,6 @@ class SamplingParams(
262
top_k=top_k,
263
min_p=min_p,
264
seed=seed,
283
- use_beam_search=use_beam_search,
284
- length_penalty=length_penalty,
285
- early_stopping=early_stopping,
265
stop=stop,
266
stop_token_ids=stop_token_ids,
267
include_stop_str_in_output=include_stop_str_in_output,
@@ -334,20 +313,13 @@ class SamplingParams(
313
self.output_text_buffer_length = max(len(s) for s in self.stop) - 1
314
315
self._verify_args()
337
- if self.use_beam_search:
338
- if not envs.VLLM_ALLOW_DEPRECATED_BEAM_SEARCH:
339
- raise ValueError(
340
- "Using beam search as a sampling parameter is deprecated, and will be removed in the future release. Please use the `vllm.LLM.use_beam_search` method for dedicated beam search instead, or set the environment variable `VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1` to suppress this error. For more details, see https://github.com/vllm-project/vllm/issues/8306 ." # noqa
341
- )
342
- self._verify_beam_search()
343
- else:
344
- self._verify_non_beam_search()
345
- if self.temperature < _SAMPLING_EPS:
346
- # Zero temperature means greedy sampling.
347
- self.top_p = 1.0
348
- self.top_k = -1
349
- self.min_p = 0.0
350
- self._verify_greedy_sampling()
316
+
317
+ if self.temperature < _SAMPLING_EPS:
318
+ # Zero temperature means greedy sampling.
319
+ self.top_p = 1.0
320
+ self.top_k = -1
321
+ self.min_p = 0.0
322
+ self._verify_greedy_sampling()
323
# eos_token_id is added to this by the engine
324
self._all_stop_token_ids = set(self.stop_token_ids)
325
@@ -417,31 +389,6 @@ class SamplingParams(
389
RequestOutputKind.DELTA):
390
raise ValueError("best_of must equal n to use output_kind=DELTA")
391
420
- def _verify_beam_search(self) -> None:
421
- if self.best_of == 1:
422
- raise ValueError("best_of must be greater than 1 when using beam "
423
- f"search. Got {self.best_of}.")
424
- if self.temperature > _SAMPLING_EPS:
425
- raise ValueError("temperature must be 0 when using beam search.")
426
- if self.top_p < 1.0 - _SAMPLING_EPS:
427
- raise ValueError("top_p must be 1 when using beam search.")
428
- if self.top_k != -1:
429
- raise ValueError("top_k must be -1 when using beam search.")
430
- if self.early_stopping not in [True, False, "never"]:
431
- raise ValueError(
432
- f"early_stopping must be True, False, or 'never', "
433
- f"got {self.early_stopping}.")
434
-
435
- def _verify_non_beam_search(self) -> None:
436
- if self.early_stopping is not False:
437
- raise ValueError("early_stopping is not effective and must be "
438
- "False when not using beam search.")
439
- if (self.length_penalty < 1.0 - _SAMPLING_EPS
440
- or self.length_penalty > 1.0 + _SAMPLING_EPS):
441
- raise ValueError(
442
- "length_penalty is not effective and must be the "
443
- "default value of 1.0 when not using beam search.")
444
-
392
def _verify_greedy_sampling(self) -> None:
393
assert isinstance(self.best_of, int)
394
if self.best_of > 1:
@@ -476,8 +423,6 @@ class SamplingParams(
423
424
@cached_property
425
def sampling_type(self) -> SamplingType:
479
- if self.use_beam_search:
480
- return SamplingType.BEAM
426
if self.temperature < _SAMPLING_EPS:
427
return SamplingType.GREEDY
428
if self.seed is not None:
@@ -514,9 +459,6 @@ class SamplingParams(
459
f"top_k={self.top_k}, "
460
f"min_p={self.min_p}, "
461
f"seed={self.seed}, "
517
- f"use_beam_search={self.use_beam_search}, "
518
- f"length_penalty={self.length_penalty}, "
519
- f"early_stopping={self.early_stopping}, "
462
f"stop={self.stop}, "
463
f"stop_token_ids={self.stop_token_ids}, "
464
f"include_stop_str_in_output={self.include_stop_str_in_output}, "
@@ -542,3 +484,4 @@ class BeamSearchParams(
484
max_tokens: int
485
ignore_eos: bool = False
486
temperature: float = 0.0
487
+ length_penalty: float = 1.0
vllm/sequence.py
+10
-36
@@ -577,25 +577,6 @@ class Sequence:
577
def get_cumulative_logprob(self) -> float:
578
return self.data.cumulative_logprob
579
580
- def get_beam_search_score(self,
581
- length_penalty: float = 1.0,
582
- seq_len: Optional[int] = None,
583
- eos_token_id: Optional[int] = None) -> float:
584
- """Calculate the beam search score with length penalty.
585
-
586
- Adapted from
587
-
588
- https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
589
- """
590
- if seq_len is None:
591
- seq_len = self.get_len()
592
- # NOTE: HF implementation does not count the EOS token
593
- # towards the length, we align with that here for testing.
594
- if (eos_token_id is not None
595
- and self.get_last_token_id() == eos_token_id):
596
- seq_len -= 1
597
- return self.get_cumulative_logprob() / (seq_len**length_penalty)
598
-
580
def is_finished(self) -> bool:
581
return SequenceStatus.is_finished(self.status)
582
@@ -809,25 +790,18 @@ class SequenceGroup:
790
def get_max_num_running_seqs(self) -> int:
791
"""The maximum number of sequences running in parallel in the remaining
792
lifetime of the request."""
812
- if self.sampling_params and self.sampling_params.use_beam_search:
813
- # For beam search, maximally there will always be `best_of` beam
814
- # candidates running in the future.
793
+ if self.sampling_params:
794
best_of = self.sampling_params.best_of
795
assert isinstance(best_of, int)
817
- return best_of
818
- else:
819
- if self.sampling_params:
820
- best_of = self.sampling_params.best_of
821
- assert isinstance(best_of, int)
822
- if best_of > self.num_seqs():
823
- # At prompt stage, the sequence group is not yet filled up
824
- # and only have one sequence running. However, in the
825
- # generation stage, we will have `best_of` sequences
826
- # running.
827
- return best_of
828
- # At sampling stages, return the number of actual sequences
829
- # that are not finished yet.
830
- return self.num_unfinished_seqs()
796
+ if best_of > self.num_seqs():
797
+ # At prompt stage, the sequence group is not yet filled up
798
+ # and only have one sequence running. However, in the
799
+ # generation stage, we will have `best_of` sequences
800
+ # running.
801
+ return best_of
802
+ # At sampling stages, return the number of actual sequences
803
+ # that are not finished yet.
804
+ return self.num_unfinished_seqs()
805
806
def get_seqs(
807
self,
vllm/utils.py
+19
@@ -1361,3 +1361,22 @@ class AtomicCounter:
1361
@property
1362
def value(self):
1363
return self._value
1364
+
1365
+
1366
+def get_beam_search_score(
1367
+ tokens: List[int],
1368
+ cumulative_logprob: float,
1369
+ eos_token_id: int,
1370
+ length_penalty: float = 1.0,
1371
+) -> float:
1372
+ """Calculate the beam search score with length penalty.
1373
+
1374
+ Adapted from
1375
+
1376
+ https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
1377
+ """
1378
+ seq_len = len(tokens)
1379
+ if tokens[-1] == eos_token_id:
1380
+ seq_len -= 1
1381
+
1382
+ return cumulative_logprob / (seq_len**length_penalty)
vllm/worker/tpu_model_runner.py
-3
@@ -453,9 +453,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
453
f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
454
"backend.")
455
best_of.append(sampling_params.best_of)
456
- if sampling_params.use_beam_search:
457
- raise NotImplementedError(
458
- "Beam search is not supported by the TPU backend.")
456
if sampling_params.logprobs is not None:
457
raise NotImplementedError(
458
"logprobs is not currently supported by the TPU backend.")