@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18b296fdb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[core] remove beam search from the core (#9105)

youkaichao committed Oct 6, 2024 at 22:47 UTC 18b296fdb2248e8a65bf005e7193ebd523b875b6
25 files changed +98 -596
benchmarks/backend_request_func.py
-6
@@ -23,7 +23,6 @@ class RequestFuncInput:
23 output_len: int
24 model: str
25 best_of: int = 1
26 - use_beam_search: bool = False
26 logprobs: Optional[int] = None
27 multi_modal_content: Optional[dict] = None
28 ignore_eos: bool = False
@@ -49,7 +48,6 @@ async def async_request_tgi(
48 assert api_url.endswith("generate_stream")
49
50 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
52 - assert not request_func_input.use_beam_search
51 params = {
52 "best_of": request_func_input.best_of,
53 "max_new_tokens": request_func_input.output_len,
@@ -121,7 +119,6 @@ async def async_request_trt_llm(
119 assert api_url.endswith("generate_stream")
120
121 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
124 - assert not request_func_input.use_beam_search
122 assert request_func_input.best_of == 1
123 payload = {
124 "accumulate_tokens": True,
@@ -187,7 +184,6 @@ async def async_request_deepspeed_mii(
184 ) -> RequestFuncOutput:
185 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
186 assert request_func_input.best_of == 1
190 - assert not request_func_input.use_beam_search
187
188 payload = {
189 "prompt": request_func_input.prompt,
@@ -235,7 +231,6 @@ async def async_request_openai_completions(
231 ), "OpenAI Completions API URL must end with 'completions' or 'profile'."
232
233 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
238 - assert not request_func_input.use_beam_search
234 payload = {
235 "model": request_func_input.model,
236 "prompt": request_func_input.prompt,
@@ -317,7 +312,6 @@ async def async_request_openai_chat_completions(
312 ), "OpenAI Chat Completions API URL must end with 'chat/completions'."
313
314 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
320 - assert not request_func_input.use_beam_search
315 content = [{"type": "text", "text": request_func_input.prompt}]
316 if request_func_input.multi_modal_content:
317 content.append(request_func_input.multi_modal_content)
benchmarks/benchmark_latency.py
+1 -2
@@ -51,9 +51,8 @@ def main(args: argparse.Namespace):
51
52 sampling_params = SamplingParams(
53 n=args.n,
54 - temperature=0.0 if args.use_beam_search else 1.0,
54 + temperature=1.0,
55 top_p=1.0,
56 - use_beam_search=args.use_beam_search,
56 ignore_eos=True,
57 max_tokens=args.output_len,
58 )
benchmarks/benchmark_prioritization.py
+11 -13
@@ -68,7 +68,6 @@ def run_vllm(
68 tensor_parallel_size: int,
69 seed: int,
70 n: int,
71 - use_beam_search: bool,
71 trust_remote_code: bool,
72 dtype: str,
73 max_model_len: Optional[int],
@@ -114,9 +113,8 @@ def run_vllm(
113 sampling_params.append(
114 SamplingParams(
115 n=n,
117 - temperature=0.0 if use_beam_search else 1.0,
116 + temperature=1.0,
117 top_p=1.0,
119 - use_beam_search=use_beam_search,
118 ignore_eos=True,
119 max_tokens=output_len,
120 ))
@@ -144,15 +142,16 @@ def main(args: argparse.Namespace):
142 args.output_len)
143
144 if args.backend == "vllm":
147 - elapsed_time = run_vllm(
148 - requests, args.model, args.tokenizer, args.quantization,
149 - args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
150 - args.trust_remote_code, args.dtype, args.max_model_len,
151 - args.enforce_eager, args.kv_cache_dtype,
152 - args.quantization_param_path, args.device,
153 - args.enable_prefix_caching, args.enable_chunked_prefill,
154 - args.max_num_batched_tokens, args.gpu_memory_utilization,
155 - args.download_dir)
145 + elapsed_time = run_vllm(requests, args.model, args.tokenizer,
146 + args.quantization, args.tensor_parallel_size,
147 + args.seed, args.n, args.trust_remote_code,
148 + args.dtype, args.max_model_len,
149 + args.enforce_eager, args.kv_cache_dtype,
150 + args.quantization_param_path, args.device,
151 + args.enable_prefix_caching,
152 + args.enable_chunked_prefill,
153 + args.max_num_batched_tokens,
154 + args.gpu_memory_utilization, args.download_dir)
155 else:
156 raise ValueError(f"Unknown backend: {args.backend}")
157 total_num_tokens = sum(prompt_len + output_len
@@ -203,7 +202,6 @@ if __name__ == "__main__":
202 type=int,
203 default=1,
204 help="Number of generated sequences per prompt.")
206 - parser.add_argument("--use-beam-search", action="store_true")
205 parser.add_argument("--num-prompts",
206 type=int,
207 default=200,
benchmarks/benchmark_serving.py
-7
@@ -391,7 +391,6 @@ async def benchmark(
391 input_requests: List[Tuple[str, int, int]],
392 logprobs: Optional[int],
393 best_of: int,
394 - use_beam_search: bool,
394 request_rate: float,
395 disable_tqdm: bool,
396 profile: bool,
@@ -419,7 +418,6 @@ async def benchmark(
418 output_len=test_output_len,
419 logprobs=logprobs,
420 best_of=best_of,
422 - use_beam_search=use_beam_search,
421 multi_modal_content=test_mm_content,
422 ignore_eos=ignore_eos,
423 )
@@ -441,7 +439,6 @@ async def benchmark(
439 output_len=test_output_len,
440 logprobs=logprobs,
441 best_of=best_of,
444 - use_beam_search=use_beam_search,
442 multi_modal_content=test_mm_content,
443 )
444 profile_output = await request_func(request_func_input=profile_input)
@@ -464,7 +461,6 @@ async def benchmark(
461 output_len=output_len,
462 logprobs=logprobs,
463 best_of=best_of,
467 - use_beam_search=use_beam_search,
464 multi_modal_content=mm_content,
465 )
466 tasks.append(
@@ -483,7 +479,6 @@ async def benchmark(
479 output_len=test_output_len,
480 logprobs=logprobs,
481 best_of=best_of,
486 - use_beam_search=use_beam_search,
482 )
483 profile_output = await request_func(request_func_input=profile_input)
484 if profile_output.success:
@@ -679,7 +674,6 @@ def main(args: argparse.Namespace):
674 input_requests=input_requests,
675 logprobs=args.logprobs,
676 best_of=args.best_of,
682 - use_beam_search=args.use_beam_search,
677 request_rate=args.request_rate,
678 disable_tqdm=args.disable_tqdm,
679 profile=args.profile,
@@ -701,7 +695,6 @@ def main(args: argparse.Namespace):
695 result_json["model_id"] = model_id
696 result_json["tokenizer_id"] = tokenizer_id
697 result_json["best_of"] = args.best_of
704 - result_json["use_beam_search"] = args.use_beam_search
698 result_json["num_prompts"] = args.num_prompts
699
700 # Metadata
benchmarks/benchmark_throughput.py
+9 -20
@@ -73,7 +73,6 @@ def run_vllm(
73 tensor_parallel_size: int,
74 seed: int,
75 n: int,
76 - use_beam_search: bool,
76 trust_remote_code: bool,
77 dtype: str,
78 max_model_len: Optional[int],
@@ -91,7 +90,6 @@ def run_vllm(
90 download_dir: Optional[str] = None,
91 load_format: str = EngineArgs.load_format,
92 disable_async_output_proc: bool = False,
94 - use_new_beam_search_impl: bool = False,
93 ) -> float:
94 from vllm import LLM, SamplingParams
95 llm = LLM(
@@ -127,19 +125,19 @@ def run_vllm(
125 sampling_params.append(
126 SamplingParams(
127 n=n,
130 - temperature=0.0 if use_beam_search else 1.0,
128 + temperature=1.0,
129 top_p=1.0,
132 - use_beam_search=use_beam_search,
130 ignore_eos=True,
131 max_tokens=output_len,
132 ))
133
137 - if not use_new_beam_search_impl:
134 + use_beam_search = False
135 +
136 + if not use_beam_search:
137 start = time.perf_counter()
138 llm.generate(prompts, sampling_params, use_tqdm=True)
139 end = time.perf_counter()
140 else:
142 - assert use_beam_search
141 prompts = [prompt for prompt, _, _ in requests]
142 # output_len should be the same for all requests.
143 output_len = requests[0][2]
@@ -165,7 +163,6 @@ async def run_vllm_async(
163 tensor_parallel_size: int,
164 seed: int,
165 n: int,
168 - use_beam_search: bool,
166 trust_remote_code: bool,
167 dtype: str,
168 max_model_len: Optional[int],
@@ -224,9 +221,8 @@ async def run_vllm_async(
221 sampling_params.append(
222 SamplingParams(
223 n=n,
227 - temperature=0.0 if use_beam_search else 1.0,
224 + temperature=1.0,
225 top_p=1.0,
229 - use_beam_search=use_beam_search,
226 ignore_eos=True,
227 max_tokens=output_len,
228 ))
@@ -248,11 +244,9 @@ def run_hf(
244 model: str,
245 tokenizer: PreTrainedTokenizerBase,
246 n: int,
251 - use_beam_search: bool,
247 max_batch_size: int,
248 trust_remote_code: bool,
249 ) -> float:
255 - assert not use_beam_search
250 llm = AutoModelForCausalLM.from_pretrained(
251 model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
252 if llm.config.model_type == "llama":
@@ -284,7 +278,7 @@ def run_hf(
278 padding=True).input_ids
279 llm_outputs = llm.generate(
280 input_ids=input_ids.cuda(),
287 - do_sample=not use_beam_search,
281 + do_sample=True,
282 num_return_sequences=n,
283 temperature=1.0,
284 top_p=1.0,
@@ -340,7 +334,7 @@ def main(args: argparse.Namespace):
334 if args.backend == "vllm":
335 run_args = [
336 requests, args.model, args.tokenizer, args.quantization,
343 - args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
337 + args.tensor_parallel_size, args.seed, args.n,
338 args.trust_remote_code, args.dtype, args.max_model_len,
339 args.enforce_eager, args.kv_cache_dtype,
340 args.quantization_param_path, args.device,
@@ -355,12 +349,11 @@ def main(args: argparse.Namespace):
349 run_args.append(args.disable_frontend_multiprocessing)
350 elapsed_time = uvloop.run(run_vllm_async(*run_args))
351 else:
358 - elapsed_time = run_vllm(*run_args, args.use_new_beam_search_impl)
352 + elapsed_time = run_vllm(*run_args)
353 elif args.backend == "hf":
354 assert args.tensor_parallel_size == 1
355 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
362 - args.use_beam_search, args.hf_max_batch_size,
363 - args.trust_remote_code)
356 + args.hf_max_batch_size, args.trust_remote_code)
357 elif args.backend == "mii":
358 elapsed_time = run_mii(requests, args.model, args.tensor_parallel_size,
359 args.output_len)
@@ -414,8 +407,6 @@ if __name__ == "__main__":
407 type=int,
408 default=1,
409 help="Number of generated sequences per prompt.")
417 - parser.add_argument("--use-beam-search", action="store_true")
418 - parser.add_argument("--use-new-beam-search-impl", action="store_true")
410 parser.add_argument("--num-prompts",
411 type=int,
412 default=1000,
@@ -570,8 +561,6 @@ if __name__ == "__main__":
561 raise ValueError("dtype must be auto for MII backend.")
562 if args.n != 1:
563 raise ValueError("n must be 1 for MII backend.")
573 - if args.use_beam_search:
574 - raise ValueError("Beam search is not supported for MII backend.")
564 if args.quantization is not None:
565 raise ValueError("Quantization is only for vLLM backend.")
566 if args.hf_max_batch_size is not None:
examples/llm_engine_example.py
-3
@@ -18,9 +18,6 @@ def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
18 temperature=0.8,
19 top_p=0.95,
20 frequency_penalty=0.1)),
21 - ("It is only with the heart that one can see rightly",
22 - SamplingParams(n=3, best_of=3, use_beam_search=True,
23 - temperature=0.0)),
21 ]
22
23
examples/multilora_inference.py
-18
@@ -43,15 +43,6 @@ def create_test_prompts(
43 max_tokens=128,
44 stop_token_ids=[32003]),
45 LoRARequest("sql-lora", 1, lora_path)),
46 - (
47 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
48 - SamplingParams(n=3,
49 - best_of=3,
50 - use_beam_search=True,
51 - temperature=0,
52 - max_tokens=128,
53 - stop_token_ids=[32003]),
54 - LoRARequest("sql-lora", 1, lora_path)),
46 (
47 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
48 SamplingParams(temperature=0.0,
@@ -60,15 +51,6 @@ def create_test_prompts(
51 max_tokens=128,
52 stop_token_ids=[32003]),
53 LoRARequest("sql-lora2", 2, lora_path)),
63 - (
64 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
65 - SamplingParams(n=3,
66 - best_of=3,
67 - use_beam_search=True,
68 - temperature=0,
69 - max_tokens=128,
70 - stop_token_ids=[32003]),
71 - LoRARequest("sql-lora", 1, lora_path)),
54 ]
55
56
tests/basic_correctness/test_preemption.py
+2 -112
@@ -23,11 +23,9 @@ MODELS = [
23 @pytest.fixture(scope="module", autouse=True)
24 def check_settings():
25 assert ENABLE_ARTIFICIAL_PREEMPT is True, (
26 - "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1, "
27 - "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1. "
26 + "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1."
27 "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 "
29 - "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 pytest "
30 - "tests/basic_correctness/test_preemption.py`")
28 + "pytest tests/basic_correctness/test_preemption.py`")
29
30
31 @pytest.fixture
@@ -137,114 +135,6 @@ def test_preemption(
135 assert total_preemption == total_recorded_preemption
136
137
140 -@pytest.mark.parametrize("model", MODELS)
141 -@pytest.mark.parametrize("dtype", ["float"])
142 -@pytest.mark.parametrize("max_tokens", [96])
143 -@pytest.mark.parametrize("beam_width", [4])
144 -def test_swap(
145 - caplog_vllm,
146 - hf_runner,
147 - vllm_runner,
148 - example_prompts,
149 - model: str,
150 - dtype: str,
151 - max_tokens: int,
152 - beam_width: int,
153 - worker_use_ray: bool,
154 -) -> None:
155 - """Use beam search enables swapping."""
156 - example_prompts = example_prompts[:1]
157 - with hf_runner(model, dtype=dtype) as hf_model:
158 - hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
159 - max_tokens)
160 -
161 - with vllm_runner(
162 - model,
163 - dtype=dtype,
164 - swap_space=10,
165 - disable_log_stats=False,
166 - worker_use_ray=worker_use_ray,
167 - ) as vllm_model:
168 - vllm_outputs = vllm_model.generate_beam_search(example_prompts,
169 - beam_width, max_tokens)
170 - assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
171 - < ARTIFICIAL_PREEMPTION_MAX_CNT)
172 - total_preemption = (
173 - vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)
174 -
175 - for i in range(len(example_prompts)):
176 - hf_output_ids, _ = hf_outputs[i]
177 - vllm_output_ids, _ = vllm_outputs[i]
178 - assert len(hf_output_ids) == len(vllm_output_ids)
179 - for j in range(len(hf_output_ids)):
180 - assert hf_output_ids[j] == vllm_output_ids[j], (
181 - f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
182 - f"vLLM: {vllm_output_ids}")
183 -
184 - assert ("is preempted by PreemptionMode.SWAP mode because there "
185 - "is not enough KV cache space." in caplog_vllm.text)
186 - # Ensure the count bucket of request-level histogram metrics matches
187 - # the number of requests as a simple sanity check to ensure metrics are
188 - # generated
189 - preemption_metrics = None
190 - for m in REGISTRY.collect():
191 - if m.name == "vllm:num_preemptions":
192 - preemption_metrics = m
193 - assert preemption_metrics is not None
194 - total_recorded_preemption = 0
195 - for sample in preemption_metrics.samples:
196 - total_recorded_preemption += sample.value
197 - assert total_preemption == total_recorded_preemption
198 -
199 -
200 -@pytest.mark.parametrize("model", MODELS)
201 -@pytest.mark.parametrize("dtype", ["float"])
202 -@pytest.mark.parametrize("max_tokens", [96])
203 -@pytest.mark.parametrize("beam_width", [4])
204 -@pytest.mark.parametrize("use_v2_block_manager", [True, False])
205 -def test_swap_infeasible(
206 - vllm_runner,
207 - example_prompts,
208 - model: str,
209 - dtype: str,
210 - max_tokens: int,
211 - beam_width: int,
212 - worker_use_ray: bool,
213 - use_v2_block_manager: bool,
214 -) -> None:
215 - """Verify infeasible swap request will be ignored."""
216 - BLOCK_SIZE = 16
217 - prefill_blocks = 2
218 - decode_blocks = max_tokens // BLOCK_SIZE
219 - example_prompts = example_prompts[:1]
220 - with vllm_runner(
221 - model,
222 - dtype=dtype,
223 - swap_space=10,
224 - block_size=BLOCK_SIZE,
225 - # Since beam search have more than 1 sequence, prefill +
226 - # decode blocks are not enough to finish.
227 - num_gpu_blocks_override=prefill_blocks + decode_blocks,
228 - max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
229 - worker_use_ray=worker_use_ray,
230 - use_v2_block_manager=use_v2_block_manager,
231 - ) as vllm_model:
232 - sampling_params = SamplingParams(n=beam_width,
233 - use_beam_search=True,
234 - temperature=0.0,
235 - max_tokens=max_tokens,
236 - ignore_eos=True)
237 - req_outputs = vllm_model.model.generate(
238 - example_prompts,
239 - sampling_params=sampling_params,
240 - )
241 - assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
242 - < ARTIFICIAL_PREEMPTION_MAX_CNT)
243 -
244 - # Verify the request is ignored and not hang.
245 - assert req_outputs[0].outputs[0].finish_reason == "length"
246 -
247 -
138 @pytest.mark.parametrize("model", MODELS)
139 @pytest.mark.parametrize("dtype", ["float"])
140 @pytest.mark.parametrize("max_tokens", [96])
tests/conftest.py
-14
@@ -782,7 +782,6 @@ class VllmRunner:
782 List[TokensTextLogprobsPromptLogprobs]]:
783 greedy_logprobs_params = SamplingParams(
784 temperature=0.0,
785 - use_beam_search=False,
785 max_tokens=max_tokens,
786 logprobs=num_logprobs,
787 prompt_logprobs=(num_prompt_logprobs),
@@ -795,19 +794,6 @@ class VllmRunner:
794 encoder_decoder_prompts, greedy_logprobs_params)
795
796 def generate_beam_search(
798 - self,
799 - prompts: List[str],
800 - beam_width: int,
801 - max_tokens: int,
802 - ) -> List[Tuple[List[List[int]], List[str]]]:
803 - beam_search_params = SamplingParams(n=beam_width,
804 - use_beam_search=True,
805 - temperature=0.0,
806 - max_tokens=max_tokens)
807 - outputs = self.generate(prompts, beam_search_params)
808 - return outputs
809 -
810 - def generate_beam_search_new(
797 self,
798 prompts: Union[List[str], List[List[int]]],
799 beam_width: int,
tests/core/block/e2e/test_correctness.py
-67
@@ -85,73 +85,6 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
85 assert baseline_token_ids == test_token_ids
86
87
88 -@pytest.mark.parametrize(
89 - "common_llm_kwargs",
90 - [{
91 - # Use a small model for a fast test.
92 - "model": "facebook/opt-125m",
93 -
94 - # skip cuda graph creation for fast test.
95 - "enforce_eager": True,
96 -
97 - # Use a large block size to trigger more copy-on-writes.
98 - "block_size": 32,
99 - }])
100 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
101 -@pytest.mark.parametrize("baseline_llm_kwargs", [{
102 - "use_v2_block_manager": False
103 -}])
104 -@pytest.mark.parametrize("test_llm_kwargs", [{
105 - "use_v2_block_manager": True,
106 - "preemption_mode": "swap"
107 -}, {
108 - "use_v2_block_manager": True,
109 - "preemption_mode": "recompute"
110 -}])
111 -@pytest.mark.parametrize("batch_size", [10])
112 -@pytest.mark.parametrize("seed", [1])
113 -def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
114 - test_llm_generator, batch_size):
115 - """Verify beam search equality with block manager v1 and v2.
116 -
117 - This requires copy-on-writes; if the v1 and v2 output is the same, then
118 - we have some confidence cow is working.
119 - """
120 - output_len = 128
121 - temperature = 0.0
122 -
123 - prompts = [
124 - "Hello, my name is",
125 - "The president of the United States is",
126 - "The capital of France is",
127 - "The future of AI is",
128 - ]
129 -
130 - prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
131 -
132 - sampling_params = SamplingParams(
133 - max_tokens=output_len,
134 - ignore_eos=True,
135 - temperature=temperature,
136 - use_beam_search=True,
137 - best_of=2,
138 - )
139 -
140 - print('Getting token ids from block manager v1')
141 - baseline_token_ids = get_token_ids_from_llm_generator(
142 - baseline_llm_generator, prompts, sampling_params)
143 -
144 - print('Getting token ids from block manager v2')
145 - test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
146 - prompts, sampling_params)
147 -
148 - for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
149 - test_token_ids):
150 - assert expected_token_ids == actual_token_ids
151 -
152 - assert baseline_token_ids == test_token_ids
153 -
154 -
88 @pytest.mark.parametrize(
89 "common_llm_kwargs",
90 [{
tests/core/utils.py
+1 -6
@@ -13,7 +13,6 @@ def create_dummy_prompt(
13 prompt_length: int,
14 block_size: Optional[int] = None,
15 lora_request: Optional[LoRARequest] = None,
16 - use_beam_search: bool = False,
16 best_of: int = 1,
17 prompt_tokens: Optional[List[int]] = None,
18 min_tokens: int = 0,
@@ -37,7 +36,6 @@ def create_dummy_prompt(
36 seqs=[prompt],
37 arrival_time=time.time(),
38 sampling_params=SamplingParams(
40 - use_beam_search=use_beam_search,
39 best_of=best_of,
40 max_tokens=max_tokens,
41 min_tokens=min_tokens),
@@ -52,7 +50,6 @@ def create_dummy_prompt_encoder_decoder(
50 encoder_prompt_length: int,
51 block_size: Optional[int] = None,
52 lora_request: Optional[LoRARequest] = None,
55 - use_beam_search: bool = False,
53 best_of: int = 1,
54 ) -> Tuple[Sequence, Sequence, SequenceGroup]:
55 if not block_size:
@@ -85,9 +82,7 @@ def create_dummy_prompt_encoder_decoder(
82 from_decoder_prompt=False)
83 seq_group = SequenceGroup(request_id=request_id,
84 seqs=[decoder_prompt],
88 - sampling_params=SamplingParams(
89 - use_beam_search=use_beam_search,
90 - best_of=best_of),
85 + sampling_params=SamplingParams(best_of=best_of),
86 arrival_time=time.time(),
87 lora_request=lora_request,
88 encoder_seq=encoder_prompt)
tests/samplers/test_beam_search.py
+2 -2
@@ -33,8 +33,8 @@ def test_beam_search_single_input(
33 max_tokens)
34
35 with vllm_runner(model, dtype=dtype) as vllm_model:
36 - vllm_outputs = vllm_model.generate_beam_search_new(
37 - example_prompts, beam_width, max_tokens)
36 + vllm_outputs = vllm_model.generate_beam_search(example_prompts,
37 + beam_width, max_tokens)
38
39 for i in range(len(example_prompts)):
40 hf_output_ids, hf_output_texts = hf_outputs[i]
tests/samplers/test_sampler.py
+2 -28
@@ -159,26 +159,6 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
159 assert first_sampler_output == second_sampler_output
160
161
162 -@pytest.mark.parametrize("seed", RANDOM_SEEDS)
163 -@pytest.mark.parametrize("device", CUDA_DEVICES)
164 -def test_sampler_all_beam(seed: int, device: str):
165 - set_random_seed(seed)
166 - torch.set_default_device(device)
167 - batch_size = random.randint(1, 256)
168 - _, fake_logits, sampler = _prepare_test(batch_size)
169 -
170 - sampling_params = SamplingParams(
171 - temperature=0,
172 - best_of=2,
173 - use_beam_search=True,
174 - )
175 - _do_sample(batch_size, fake_logits, sampler, sampling_params, device)
176 - # no assertion here as I am not sure how to determine whether
177 - # the outputs are expected - in other words, this just tests
178 - # whether there are no exceptions in the sampler
179 - # when handling an all-beam search case.
180 -
181 -
162 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
163 @pytest.mark.parametrize("device", CUDA_DEVICES)
164 def test_sampler_min_tokens_penalty(seed: int, device: str):
@@ -479,7 +459,7 @@ def test_sampler_mixed(seed: int, device: str):
459 seq_lens: List[int] = []
460 for i in range(batch_size):
461 expected: Optional[List[int]] = None
482 - sampling_type = random.randint(0, 3)
462 + sampling_type = random.randint(0, 2)
463 if sampling_type == 0:
464 sampling_params = SamplingParams(temperature=0)
465 expected = [int(torch.argmax(fake_logits[i], dim=-1).item())]
@@ -498,10 +478,7 @@ def test_sampler_mixed(seed: int, device: str):
478 for idx in range(n):
479 fake_logits[i, i + idx] = 1e2
480 expected = list(range(i, i + n))
501 - else:
502 - sampling_params = SamplingParams(temperature=0,
503 - use_beam_search=True,
504 - best_of=2)
481 +
482 expected_tokens.append(expected)
483 seq_group_metadata_list.append(
484 SequenceGroupMetadata(
@@ -530,9 +507,6 @@ def test_sampler_mixed(seed: int, device: str):
507 zip(sampler_output, seq_group_metadata_list)):
508 assert metadata.sampling_params is not None
509
533 - if metadata.sampling_params.use_beam_search:
534 - continue
535 -
510 if (metadata.sampling_params.seed is not None
511 and expected_tokens[i] is None):
512 # Record seeded random result to compare with results of
vllm/core/scheduler.py
+2 -2
@@ -1202,9 +1202,9 @@ class Scheduler:
1202 seq_group=seq_group, num_lookahead_slots=num_lookahead_slots)
1203
1204 def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1205 + # TODO: does it work with parallel sampling?
1206 no_beam_search = seq_group.sampling_params is None or (
1206 - seq_group.sampling_params.best_of == 1
1207 - and not seq_group.sampling_params.use_beam_search)
1207 + seq_group.sampling_params.best_of == 1)
1208 return no_beam_search
1209
1210 def schedule(
vllm/engine/async_llm_engine.py
+9 -7
@@ -33,7 +33,7 @@ from vllm.sequence import ExecuteModelRequest
33 from vllm.transformers_utils.tokenizer import AnyTokenizer
34 from vllm.usage.usage_lib import UsageContext
35 from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36 - random_uuid, weak_bind)
36 + get_beam_search_score, random_uuid, weak_bind)
37
38 logger = init_logger(__name__)
39 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -1050,6 +1050,12 @@ class AsyncLLMEngine:
1050 max_tokens = params.max_tokens
1051 ignore_eos = params.ignore_eos
1052 temperature = params.temperature
1053 + length_penalty = params.length_penalty
1054 +
1055 + def sort_beams_key(x: BeamSearchSequence) -> float:
1056 + return get_beam_search_score(x.tokens, x.cum_logprob,
1057 + tokenizer.eos_token_id,
1058 + length_penalty)
1059
1060 tokenizer = await self.get_tokenizer()
1061 tokenizedPrompt = prompt if isinstance(
@@ -1103,15 +1109,11 @@ class AsyncLLMEngine:
1109 else:
1110 new_beams.append(new_beam)
1111
1106 - sorted_beams = sorted(new_beams,
1107 - key=lambda x: x.cum_logprob,
1108 - reverse=True)
1112 + sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
1113 all_beams = sorted_beams[:beam_width]
1114
1115 completed.extend(all_beams)
1112 - sorted_completed = sorted(completed,
1113 - key=lambda x: x.cum_logprob,
1114 - reverse=True)
1116 + sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
1117 best_beams = sorted_completed[:beam_width]
1118
1119 for beam in best_beams:
vllm/engine/output_processor/single_step.py
+7 -157
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Optional, Tuple, Union
1 +from typing import Dict, List, Tuple
2
3 from vllm.config import SchedulerConfig
4 from vllm.core.scheduler import Scheduler
@@ -6,7 +6,6 @@ from vllm.engine.output_processor.interfaces import (
6 SequenceGroupOutputProcessor)
7 from vllm.engine.output_processor.stop_checker import StopChecker
8 from vllm.logger import init_logger
9 -from vllm.sampling_params import SamplingParams
9 from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
10 SequenceOutput, SequenceStatus)
11 from vllm.transformers_utils.detokenizer import Detokenizer
@@ -113,7 +112,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
112 outputs: SequenceGroupOutput,
113 is_async: bool) -> None:
114 sampling_params = seq_group.sampling_params
116 - if sampling_params.best_of == 1 and not sampling_params.use_beam_search:
115 + if sampling_params.best_of == 1:
116 # only have one output sample
117 sample = outputs.samples[0]
118 # only have one sequence
@@ -142,7 +141,6 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
141 # Process samples
142 samples = outputs.samples
143 parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
145 - existing_finished_seqs = seq_group.get_finished_seqs()
144 parent_child_dict: Dict[int, List[SequenceOutput]] = {
145 parent_seq.seq_id: []
146 for parent_seq in parent_seqs
@@ -197,106 +195,9 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
195 lora_req=seq_group.lora_request,
196 )
197
200 - # Non-beam search case
201 - if not sampling_params.use_beam_search:
202 - # For newly created child sequences, add them to the sequence group
203 - # and fork them in block manager if they are not finished.
204 - for seq, parent in child_seqs:
205 - if seq is not parent:
206 - seq_group.add(seq)
207 - if not seq.is_finished():
208 - for scheduler in self.scheduler:
209 - scheduler.fork_seq(parent, seq)
210 -
211 - # Free the finished and selected parent sequences' memory in block
212 - # manager. Keep them in the sequence group as candidate output.
213 - # NOTE: we need to fork the new sequences before freeing the
214 - # old sequences.
215 - for seq, parent in child_seqs:
216 - if seq is parent and seq.is_finished():
217 - for scheduler in self.scheduler:
218 - scheduler.free_seq(seq)
219 - return
220 -
221 - # Beam search case
222 - # Select the child sequences to keep in the sequence group.
223 - selected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
224 - unselected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
225 - beam_width = sampling_params.best_of
226 - length_penalty = sampling_params.length_penalty
227 -
228 - # Select the newly finished sequences with the highest scores
229 - # to replace existing finished sequences.
230 - # Tuple of (seq, parent, is_new)
231 - existing_finished_seqs = [(seq, None, False)
232 - for seq in existing_finished_seqs]
233 - new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
234 - if seq.is_finished()]
235 - all_finished_seqs = existing_finished_seqs + new_finished_seqs
236 - # Sort the finished sequences by their scores.
237 - all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
238 - length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
239 - reverse=True)
240 - for seq, parent, is_new in all_finished_seqs[:beam_width]:
241 - if is_new:
242 - # A newly generated child sequence finishes and has a high
243 - # score, so we will add it into the sequence group.
244 - selected_child_seqs.append((seq, parent))
245 - for seq, parent, is_new in all_finished_seqs[beam_width:]:
246 - if is_new:
247 - # A newly generated child sequence finishes but has a low
248 - # score, so we will not add it into the sequence group.
249 - # Additionally, if this sequence is a continuation of a
250 - # parent sequence, we will need remove the parent sequence
251 - # from the sequence group.
252 - unselected_child_seqs.append((seq, parent))
253 - else:
254 - # An existing finished sequence has a low score, so we will
255 - # remove it from the sequence group.
256 - seq_group.remove(seq.seq_id)
257 -
258 - # select the top beam_width sequences from the running
259 - # sequences for the next iteration to continue the beam
260 - # search.
261 - running_child_seqs = [(seq, parent) for seq, parent in child_seqs
262 - if not seq.is_finished()]
263 - # Sort the running sequences by their scores.
264 - running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
265 - length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
266 - reverse=True)
267 -
268 - # Check if we can stop the beam search.
269 - if len(running_child_seqs) == 0:
270 - # No running sequences, stop the beam search.
271 - stop_beam_search = True
272 - elif len(all_finished_seqs) < beam_width:
273 - # Not enough finished sequences, continue the beam search.
274 - stop_beam_search = False
275 - else:
276 - # Check the early stopping criteria
277 - best_running_seq = running_child_seqs[0][0]
278 - current_worst_seq = all_finished_seqs[beam_width - 1][0]
279 - stop_beam_search = self._check_beam_search_early_stopping(
280 - sampling_params.early_stopping, sampling_params,
281 - best_running_seq, current_worst_seq)
282 -
283 - if stop_beam_search:
284 - # Stop the beam search and remove all the running sequences from
285 - # the sequence group.
286 - unselected_child_seqs.extend(running_child_seqs)
287 - else:
288 - # Continue the beam search and select the top beam_width sequences
289 - # to continue the beam search.
290 - selected_child_seqs.extend(running_child_seqs[:beam_width])
291 - # The remaining running sequences will not be used in the next
292 - # iteration. Again, if these sequences are continuations of
293 - # parent sequences, we will need to remove the parent sequences
294 - # from the sequence group.
295 - unselected_child_seqs.extend(running_child_seqs[beam_width:])
296 -
198 # For newly created child sequences, add them to the sequence group
199 # and fork them in block manager if they are not finished.
299 - for seq, parent in selected_child_seqs:
200 + for seq, parent in child_seqs:
201 if seq is not parent:
202 seq_group.add(seq)
203 if not seq.is_finished():
@@ -305,61 +206,10 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
206
207 # Free the finished and selected parent sequences' memory in block
208 # manager. Keep them in the sequence group as candidate output.
308 - for seq, parent in selected_child_seqs:
209 + # NOTE: we need to fork the new sequences before freeing the
210 + # old sequences.
211 + for seq, parent in child_seqs:
212 if seq is parent and seq.is_finished():
213 for scheduler in self.scheduler:
214 scheduler.free_seq(seq)
312 -
313 - # Remove the unselected parent sequences from the sequence group and
314 - # free their memory in block manager.
315 - for seq, parent in unselected_child_seqs:
316 - if seq is parent:
317 - # Remove the parent sequence if it is not selected for next
318 - # iteration
319 - seq_group.remove(seq.seq_id)
320 - for scheduler in self.scheduler:
321 - scheduler.free_seq(seq)
322 -
323 - def _check_beam_search_early_stopping(
324 - self,
325 - early_stopping: Union[bool, str],
326 - sampling_params: SamplingParams,
327 - best_running_seq: Sequence,
328 - current_worst_seq: Sequence,
329 - ) -> bool:
330 - assert sampling_params.use_beam_search
331 - length_penalty = sampling_params.length_penalty
332 - if early_stopping is True:
333 - return True
334 -
335 - current_worst_score = current_worst_seq.get_beam_search_score(
336 - length_penalty=length_penalty,
337 - eos_token_id=current_worst_seq.eos_token_id)
338 - if early_stopping is False:
339 - highest_attainable_score = best_running_seq.get_beam_search_score(
340 - length_penalty=length_penalty,
341 - eos_token_id=best_running_seq.eos_token_id)
342 - else:
343 - assert early_stopping == "never"
344 - if length_penalty > 0.0:
345 - # If length_penalty > 0.0, beam search will prefer longer
346 - # sequences. The highest attainable score calculation is
347 - # based on the longest possible sequence length in this case.
348 - max_possible_length = max(
349 - best_running_seq.get_prompt_len() +
350 - sampling_params.max_tokens,
351 - self.scheduler_config.max_model_len)
352 - highest_attainable_score = (
353 - best_running_seq.get_beam_search_score(
354 - length_penalty=length_penalty,
355 - eos_token_id=best_running_seq.eos_token_id,
356 - seq_len=max_possible_length))
357 - else:
358 - # Otherwise, beam search will prefer shorter sequences. The
359 - # highest attainable score calculation is based on the current
360 - # sequence length.
361 - highest_attainable_score = (
362 - best_running_seq.get_beam_search_score(
363 - length_penalty=length_penalty,
364 - eos_token_id=best_running_seq.eos_token_id))
365 - return current_worst_score >= highest_attainable_score
215 + return
vllm/entrypoints/llm.py
+10 -3
@@ -28,7 +28,8 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
28 get_cached_tokenizer)
29 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
30 from vllm.usage.usage_lib import UsageContext
31 -from vllm.utils import Counter, deprecate_kwargs, is_list_of
31 +from vllm.utils import (Counter, deprecate_kwargs, get_beam_search_score,
32 + is_list_of)
33
34 logger = init_logger(__name__)
35
@@ -404,6 +405,12 @@ class LLM:
405 max_tokens = params.max_tokens
406 temperature = params.temperature
407 ignore_eos = params.ignore_eos
408 + length_penalty = params.length_penalty
409 +
410 + def sort_beams_key(x: BeamSearchSequence) -> float:
411 + return get_beam_search_score(x.tokens, x.cum_logprob,
412 + tokenizer.eos_token_id,
413 + length_penalty)
414
415 tokenizer = self.get_tokenizer()
416 # generate 2 * beam_width candidates at each step
@@ -466,7 +473,7 @@ class LLM:
473 else:
474 instance_new_beams.append(new_beam)
475 sorted_beams = sorted(instance_new_beams,
469 - key=lambda x: x.cum_logprob,
476 + key=sort_beams_key,
477 reverse=True)
478 instance.beams = sorted_beams[:beam_width]
479
@@ -474,7 +481,7 @@ class LLM:
481 for instance in instances:
482 instance.completed.extend(instance.beams)
483 sorted_completed = sorted(instance.completed,
477 - key=lambda x: x.cum_logprob,
484 + key=sort_beams_key,
485 reverse=True)
486 best_beams = sorted_completed[:beam_width]
487
vllm/entrypoints/openai/protocol.py
+2 -8
@@ -184,7 +184,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
184 min_p: float = 0.0
185 repetition_penalty: float = 1.0
186 length_penalty: float = 1.0
187 - early_stopping: bool = False
187 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
188 include_stop_str_in_output: bool = False
189 ignore_eos: bool = False
@@ -302,6 +301,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
301 max_tokens=max_tokens,
302 ignore_eos=self.ignore_eos,
303 temperature=temperature,
304 + length_penalty=self.length_penalty,
305 )
306
307 def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
@@ -345,12 +345,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
345 ignore_eos=self.ignore_eos,
346 max_tokens=max_tokens,
347 min_tokens=self.min_tokens,
348 - use_beam_search=self.use_beam_search,
349 - early_stopping=self.early_stopping,
348 skip_special_tokens=self.skip_special_tokens,
349 spaces_between_special_tokens=self.spaces_between_special_tokens,
350 include_stop_str_in_output=self.include_stop_str_in_output,
353 - length_penalty=self.length_penalty,
351 truncate_prompt_tokens=self.truncate_prompt_tokens,
352 output_kind=RequestOutputKind.DELTA if self.stream \
353 else RequestOutputKind.FINAL_ONLY,
@@ -518,7 +515,6 @@ class CompletionRequest(OpenAIBaseModel):
515 min_p: float = 0.0
516 repetition_penalty: float = 1.0
517 length_penalty: float = 1.0
521 - early_stopping: bool = False
518 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
519 include_stop_str_in_output: bool = False
520 ignore_eos: bool = False
@@ -597,6 +593,7 @@ class CompletionRequest(OpenAIBaseModel):
593 max_tokens=max_tokens,
594 ignore_eos=self.ignore_eos,
595 temperature=temperature,
596 + length_penalty=self.length_penalty,
597 )
598
599 def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
@@ -641,13 +638,10 @@ class CompletionRequest(OpenAIBaseModel):
638 ignore_eos=self.ignore_eos,
639 max_tokens=max_tokens if not echo_without_generation else 1,
640 min_tokens=self.min_tokens,
644 - use_beam_search=self.use_beam_search,
645 - early_stopping=self.early_stopping,
641 prompt_logprobs=prompt_logprobs,
642 skip_special_tokens=self.skip_special_tokens,
643 spaces_between_special_tokens=self.spaces_between_special_tokens,
644 include_stop_str_in_output=self.include_stop_str_in_output,
650 - length_penalty=self.length_penalty,
645 truncate_prompt_tokens=self.truncate_prompt_tokens,
646 output_kind=RequestOutputKind.DELTA if self.stream \
647 else RequestOutputKind.FINAL_ONLY,
vllm/envs.py
-5
@@ -63,7 +63,6 @@ if TYPE_CHECKING:
63 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
64 VLLM_USE_TRITON_AWQ: bool = False
65 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66 - VLLM_ALLOW_DEPRECATED_BEAM_SEARCH: bool = False
66 VLLM_SKIP_P2P_CHECK: bool = False
67
68
@@ -198,10 +197,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
197 lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
198 ("true", "1")),
199
201 - # If set, allowing the use of deprecated beam search implementation
202 - "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH":
203 - lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BEAM_SEARCH", "0") == "1",
204 -
200 # Internal flag to enable Dynamo graph capture
201 "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
202 lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
vllm/model_executor/layers/sampler.py
+2 -7
@@ -947,8 +947,6 @@ def get_logprobs(
947 # largest num logprobs in this API. If every logprobs is None, it will be
948 # set to -1.
949 largest_num_logprobs = -1
950 - # If beam search is enabled.
951 - use_beam_search = False
950
951 # Select indices to compute logprob from, ranks of token ids, and the top
952 # k token ids from logprobs.
@@ -981,8 +979,6 @@ def get_logprobs(
979 largest_num_logprobs = max(largest_num_logprobs,
980 sampling_params.logprobs)
981
984 - use_beam_search = use_beam_search or sampling_params.use_beam_search
985 -
982 assert len(next_token_ids) == len(query_indices)
983
984 if len(query_indices) == 0:
@@ -995,7 +991,7 @@ def get_logprobs(
991
992 # If largest_num_logprobs == -1, i.e. no logprobs are requested, we can
993 # skip the whole logprob calculation.
998 - if largest_num_logprobs >= 0 or use_beam_search:
994 + if largest_num_logprobs >= 0:
995 query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
996 next_token_ids_gpu = torch.tensor(next_token_ids,
997 device=logprobs.device)
@@ -1121,13 +1117,12 @@ def _get_sampled_logprob_if_needed(
1117 """Compute the sample logprob if needed."""
1118 seq_ids = seq_group.seq_ids
1119 num_logprobs = seq_group.sampling_params.logprobs
1124 - use_beam_search = seq_group.sampling_params.use_beam_search
1120 sampled_logprobs: SampleLogprobs = []
1121 next_token_ids, parent_seq_ids = sample_result
1122
1123 if seq_group.do_sample:
1124 assert len(next_token_ids) > 0
1130 - if num_logprobs is None and not use_beam_search:
1125 + if num_logprobs is None:
1126 for next_token_id in next_token_ids:
1127 # Use a dummy logprob
1128 sampled_logprobs.append({next_token_id: Logprob(inf)})
vllm/outputs.py
+1 -5
@@ -142,11 +142,7 @@ class RequestOutput:
142 else:
143 # Get the top-n sequences.
144 n = sampling_params.n
145 - if sampling_params.use_beam_search:
146 - sorting_key = lambda seq: seq.get_beam_search_score(
147 - sampling_params.length_penalty)
148 - else:
149 - sorting_key = lambda seq: seq.get_cumulative_logprob()
145 + sorting_key = lambda seq: seq.get_cumulative_logprob()
146 sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
147 top_n_seqs = sorted_seqs[:n]
148
vllm/sampling_params.py
+8 -65
@@ -10,7 +10,6 @@ import torch
10 from pydantic import BaseModel
11 from typing_extensions import Annotated
12
13 -import vllm.envs as envs
13 from vllm.logger import init_logger
14
15 logger = init_logger(__name__)
@@ -23,7 +22,6 @@ class SamplingType(IntEnum):
22 GREEDY = 0
23 RANDOM = 1
24 RANDOM_SEED = 2
26 - BEAM = 3
25
26
27 LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
@@ -134,16 +132,6 @@ class SamplingParams(
132 considered, relative to the probability of the most likely token.
133 Must be in [0, 1]. Set to 0 to disable this.
134 seed: Random seed to use for the generation.
137 - use_beam_search: Whether to use beam search instead of sampling.
138 - length_penalty: Float that penalizes sequences based on their length.
139 - Used in beam search.
140 - early_stopping: Controls the stopping condition for beam search. It
141 - accepts the following values: `True`, where the generation stops as
142 - soon as there are `best_of` complete candidates; `False`, where an
143 - heuristic is applied and the generation stops when is it very
144 - unlikely to find better candidates; `"never"`, where the beam search
145 - procedure only stops when there cannot be better candidates
146 - (canonical beam search algorithm).
135 stop: List of strings that stop the generation when they are generated.
136 The returned output will not contain the stop strings.
137 stop_token_ids: List of tokens that stop the generation when they are
@@ -193,9 +181,6 @@ class SamplingParams(
181 top_k: int = -1
182 min_p: float = 0.0
183 seed: Optional[int] = None
196 - use_beam_search: bool = False
197 - length_penalty: float = 1.0
198 - early_stopping: Union[bool, str] = False
184 stop: Optional[Union[str, List[str]]] = None
185 stop_token_ids: Optional[List[int]] = None
186 ignore_eos: bool = False
@@ -238,9 +223,6 @@ class SamplingParams(
223 top_k: int = -1,
224 min_p: float = 0.0,
225 seed: Optional[int] = None,
241 - use_beam_search: bool = False,
242 - length_penalty: float = 1.0,
243 - early_stopping: Union[bool, str] = False,
226 stop: Optional[Union[str, List[str]]] = None,
227 stop_token_ids: Optional[List[int]] = None,
228 include_stop_str_in_output: bool = False,
@@ -280,9 +262,6 @@ class SamplingParams(
262 top_k=top_k,
263 min_p=min_p,
264 seed=seed,
283 - use_beam_search=use_beam_search,
284 - length_penalty=length_penalty,
285 - early_stopping=early_stopping,
265 stop=stop,
266 stop_token_ids=stop_token_ids,
267 include_stop_str_in_output=include_stop_str_in_output,
@@ -334,20 +313,13 @@ class SamplingParams(
313 self.output_text_buffer_length = max(len(s) for s in self.stop) - 1
314
315 self._verify_args()
337 - if self.use_beam_search:
338 - if not envs.VLLM_ALLOW_DEPRECATED_BEAM_SEARCH:
339 - raise ValueError(
340 - "Using beam search as a sampling parameter is deprecated, and will be removed in the future release. Please use the `vllm.LLM.use_beam_search` method for dedicated beam search instead, or set the environment variable `VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1` to suppress this error. For more details, see https://github.com/vllm-project/vllm/issues/8306 ." # noqa
341 - )
342 - self._verify_beam_search()
343 - else:
344 - self._verify_non_beam_search()
345 - if self.temperature < _SAMPLING_EPS:
346 - # Zero temperature means greedy sampling.
347 - self.top_p = 1.0
348 - self.top_k = -1
349 - self.min_p = 0.0
350 - self._verify_greedy_sampling()
316 +
317 + if self.temperature < _SAMPLING_EPS:
318 + # Zero temperature means greedy sampling.
319 + self.top_p = 1.0
320 + self.top_k = -1
321 + self.min_p = 0.0
322 + self._verify_greedy_sampling()
323 # eos_token_id is added to this by the engine
324 self._all_stop_token_ids = set(self.stop_token_ids)
325
@@ -417,31 +389,6 @@ class SamplingParams(
389 RequestOutputKind.DELTA):
390 raise ValueError("best_of must equal n to use output_kind=DELTA")
391
420 - def _verify_beam_search(self) -> None:
421 - if self.best_of == 1:
422 - raise ValueError("best_of must be greater than 1 when using beam "
423 - f"search. Got {self.best_of}.")
424 - if self.temperature > _SAMPLING_EPS:
425 - raise ValueError("temperature must be 0 when using beam search.")
426 - if self.top_p < 1.0 - _SAMPLING_EPS:
427 - raise ValueError("top_p must be 1 when using beam search.")
428 - if self.top_k != -1:
429 - raise ValueError("top_k must be -1 when using beam search.")
430 - if self.early_stopping not in [True, False, "never"]:
431 - raise ValueError(
432 - f"early_stopping must be True, False, or 'never', "
433 - f"got {self.early_stopping}.")
434 -
435 - def _verify_non_beam_search(self) -> None:
436 - if self.early_stopping is not False:
437 - raise ValueError("early_stopping is not effective and must be "
438 - "False when not using beam search.")
439 - if (self.length_penalty < 1.0 - _SAMPLING_EPS
440 - or self.length_penalty > 1.0 + _SAMPLING_EPS):
441 - raise ValueError(
442 - "length_penalty is not effective and must be the "
443 - "default value of 1.0 when not using beam search.")
444 -
392 def _verify_greedy_sampling(self) -> None:
393 assert isinstance(self.best_of, int)
394 if self.best_of > 1:
@@ -476,8 +423,6 @@ class SamplingParams(
423
424 @cached_property
425 def sampling_type(self) -> SamplingType:
479 - if self.use_beam_search:
480 - return SamplingType.BEAM
426 if self.temperature < _SAMPLING_EPS:
427 return SamplingType.GREEDY
428 if self.seed is not None:
@@ -514,9 +459,6 @@ class SamplingParams(
459 f"top_k={self.top_k}, "
460 f"min_p={self.min_p}, "
461 f"seed={self.seed}, "
517 - f"use_beam_search={self.use_beam_search}, "
518 - f"length_penalty={self.length_penalty}, "
519 - f"early_stopping={self.early_stopping}, "
462 f"stop={self.stop}, "
463 f"stop_token_ids={self.stop_token_ids}, "
464 f"include_stop_str_in_output={self.include_stop_str_in_output}, "
@@ -542,3 +484,4 @@ class BeamSearchParams(
484 max_tokens: int
485 ignore_eos: bool = False
486 temperature: float = 0.0
487 + length_penalty: float = 1.0
vllm/sequence.py
+10 -36
@@ -577,25 +577,6 @@ class Sequence:
577 def get_cumulative_logprob(self) -> float:
578 return self.data.cumulative_logprob
579
580 - def get_beam_search_score(self,
581 - length_penalty: float = 1.0,
582 - seq_len: Optional[int] = None,
583 - eos_token_id: Optional[int] = None) -> float:
584 - """Calculate the beam search score with length penalty.
585 -
586 - Adapted from
587 -
588 - https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
589 - """
590 - if seq_len is None:
591 - seq_len = self.get_len()
592 - # NOTE: HF implementation does not count the EOS token
593 - # towards the length, we align with that here for testing.
594 - if (eos_token_id is not None
595 - and self.get_last_token_id() == eos_token_id):
596 - seq_len -= 1
597 - return self.get_cumulative_logprob() / (seq_len**length_penalty)
598 -
580 def is_finished(self) -> bool:
581 return SequenceStatus.is_finished(self.status)
582
@@ -809,25 +790,18 @@ class SequenceGroup:
790 def get_max_num_running_seqs(self) -> int:
791 """The maximum number of sequences running in parallel in the remaining
792 lifetime of the request."""
812 - if self.sampling_params and self.sampling_params.use_beam_search:
813 - # For beam search, maximally there will always be `best_of` beam
814 - # candidates running in the future.
793 + if self.sampling_params:
794 best_of = self.sampling_params.best_of
795 assert isinstance(best_of, int)
817 - return best_of
818 - else:
819 - if self.sampling_params:
820 - best_of = self.sampling_params.best_of
821 - assert isinstance(best_of, int)
822 - if best_of > self.num_seqs():
823 - # At prompt stage, the sequence group is not yet filled up
824 - # and only have one sequence running. However, in the
825 - # generation stage, we will have `best_of` sequences
826 - # running.
827 - return best_of
828 - # At sampling stages, return the number of actual sequences
829 - # that are not finished yet.
830 - return self.num_unfinished_seqs()
796 + if best_of > self.num_seqs():
797 + # At prompt stage, the sequence group is not yet filled up
798 + # and only have one sequence running. However, in the
799 + # generation stage, we will have `best_of` sequences
800 + # running.
801 + return best_of
802 + # At sampling stages, return the number of actual sequences
803 + # that are not finished yet.
804 + return self.num_unfinished_seqs()
805
806 def get_seqs(
807 self,
vllm/utils.py
+19
@@ -1361,3 +1361,22 @@ class AtomicCounter:
1361 @property
1362 def value(self):
1363 return self._value
1364 +
1365 +
1366 +def get_beam_search_score(
1367 + tokens: List[int],
1368 + cumulative_logprob: float,
1369 + eos_token_id: int,
1370 + length_penalty: float = 1.0,
1371 +) -> float:
1372 + """Calculate the beam search score with length penalty.
1373 +
1374 + Adapted from
1375 +
1376 + https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
1377 + """
1378 + seq_len = len(tokens)
1379 + if tokens[-1] == eos_token_id:
1380 + seq_len -= 1
1381 +
1382 + return cumulative_logprob / (seq_len**length_penalty)
vllm/worker/tpu_model_runner.py
-3
@@ -453,9 +453,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
453 f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
454 "backend.")
455 best_of.append(sampling_params.best_of)
456 - if sampling_params.use_beam_search:
457 - raise NotImplementedError(
458 - "Beam search is not supported by the TPU backend.")
456 if sampling_params.logprobs is not None:
457 raise NotImplementedError(
458 "logprobs is not currently supported by the TPU backend.")