A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
re-implement beam search on top of vllm core (#8726)
Co-authored-by: Brendan Wong <bjwpokemon@gmail.com>
youkaichao committed
Sep 23, 2024 at 22:08 UTC
0250dd68c5df12ead29d2ec7d922855c9a257b06
4 files changed
+171
-9
benchmarks/benchmark_throughput.py
+20
-4
@@ -90,6 +90,7 @@ def run_vllm(
90
download_dir: Optional[str] = None,
91
load_format: str = EngineArgs.load_format,
92
disable_async_output_proc: bool = False,
93
+ use_new_beam_search_impl: bool = False,
94
) -> float:
95
from vllm import LLM, SamplingParams
96
llm = LLM(
@@ -132,9 +133,23 @@ def run_vllm(
133
max_tokens=output_len,
134
))
135
135
- start = time.perf_counter()
136
- llm.generate(prompts, sampling_params, use_tqdm=True)
137
- end = time.perf_counter()
136
+ if not use_new_beam_search_impl:
137
+ start = time.perf_counter()
138
+ llm.generate(prompts, sampling_params, use_tqdm=True)
139
+ end = time.perf_counter()
140
+ else:
141
+ assert use_beam_search
142
+ prompts = [prompt for prompt, _, _ in requests]
143
+ # output_len should be the same for all requests.
144
+ output_len = requests[0][2]
145
+ for prompt, input_len, _output_len in requests:
146
+ assert _output_len == output_len
147
+ start = time.perf_counter()
148
+ llm.beam_search(prompts,
149
+ beam_width=n,
150
+ max_tokens=output_len,
151
+ ignore_eos=True)
152
+ end = time.perf_counter()
153
return end - start
154
155
@@ -336,7 +351,7 @@ def main(args: argparse.Namespace):
351
run_args.append(args.disable_frontend_multiprocessing)
352
elapsed_time = uvloop.run(run_vllm_async(*run_args))
353
else:
339
- elapsed_time = run_vllm(*run_args)
354
+ elapsed_time = run_vllm(*run_args, args.use_new_beam_search_impl)
355
elif args.backend == "hf":
356
assert args.tensor_parallel_size == 1
357
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -396,6 +411,7 @@ if __name__ == "__main__":
411
default=1,
412
help="Number of generated sequences per prompt.")
413
parser.add_argument("--use-beam-search", action="store_true")
414
+ parser.add_argument("--use-new-beam-search-impl", action="store_true")
415
parser.add_argument("--num-prompts",
416
type=int,
417
default=1000,
tests/conftest.py
+14
@@ -798,6 +798,20 @@ class VllmRunner:
798
outputs = self.generate(prompts, beam_search_params)
799
return outputs
800
801
+ def generate_beam_search_new(
802
+ self,
803
+ prompts: Union[List[str], List[List[int]]],
804
+ beam_width: int,
805
+ max_tokens: int,
806
+ ) -> List[Tuple[List[List[int]], List[str]]]:
807
+ outputs = self.model.beam_search(prompts, beam_width, max_tokens)
808
+ returned_outputs = []
809
+ for output in outputs:
810
+ token_ids = [x.tokens for x in output.sequences]
811
+ texts = [x.text for x in output.sequences]
812
+ returned_outputs.append((token_ids, texts))
813
+ return returned_outputs
814
+
815
def encode(self, prompts: List[str]) -> List[List[float]]:
816
req_outputs = self.model.encode(prompts)
817
outputs = []
tests/samplers/test_beam_search.py
+3
-3
@@ -9,7 +9,7 @@ import pytest
9
# 1. Increase max_tokens to 256.
10
# 2. Increase beam_width to 8.
11
# 3. Use the model "huggyllama/llama-7b".
12
-MAX_TOKENS = [128]
12
+MAX_TOKENS = [64]
13
BEAM_WIDTHS = [4]
14
MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
15
@@ -33,8 +33,8 @@ def test_beam_search_single_input(
33
max_tokens)
34
35
with vllm_runner(model, dtype=dtype) as vllm_model:
36
- vllm_outputs = vllm_model.generate_beam_search(example_prompts,
37
- beam_width, max_tokens)
36
+ vllm_outputs = vllm_model.generate_beam_search_new(
37
+ example_prompts, beam_width, max_tokens)
38
39
for i in range(len(example_prompts)):
40
hf_output_ids, hf_output_texts = hf_outputs[i]
vllm/entrypoints/llm.py
+134
-2
@@ -1,6 +1,8 @@
1
+import itertools
2
from contextlib import contextmanager
2
-from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Union, cast,
3
- overload)
3
+from dataclasses import dataclass
4
+from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
5
+ Union, cast, overload)
6
7
from tqdm import tqdm
8
@@ -30,6 +32,37 @@ from vllm.utils import Counter, deprecate_kwargs, is_list_of
32
logger = init_logger(__name__)
33
34
35
+@dataclass
36
+class BeamSearchSequence:
37
+ """A sequence for beam search.
38
+ It keeps track of the tokens and the log probability of the sequence.
39
+ The text field is optional and will only be filled when the sequence is
40
+ about to be returned to the user.
41
+ """
42
+ # The tokens includes the prompt.
43
+ tokens: List[int]
44
+ cum_logprob: float = 0.0
45
+ text: Optional[str] = None
46
+
47
+
48
+@dataclass
49
+class BeamSearchOutput:
50
+ """The output of beam search.
51
+ It contains the list of the best beam search sequences.
52
+ The length of the list is equal to the beam width.
53
+ """
54
+ sequences: List[BeamSearchSequence]
55
+
56
+
57
+class BeamSearchInstance:
58
+
59
+ def __init__(self, prompt_tokens: List[int]):
60
+ self.beams: List[BeamSearchSequence] = [
61
+ BeamSearchSequence(tokens=prompt_tokens)
62
+ ]
63
+ self.completed: List[BeamSearchSequence] = []
64
+
65
+
66
class LLM:
67
"""An LLM for generating texts from given prompts and sampling parameters.
68
@@ -354,6 +387,105 @@ class LLM:
387
outputs = self._run_engine(use_tqdm=use_tqdm)
388
return LLMEngine.validate_outputs(outputs, RequestOutput)
389
390
+ def beam_search(
391
+ self,
392
+ prompts: List[Union[str, List[int]]],
393
+ beam_width: int,
394
+ max_tokens: int,
395
+ ignore_eos: bool = False,
396
+ ) -> List[BeamSearchOutput]:
397
+ """
398
+ Generate sequences using beam search.
399
+
400
+ Args:
401
+ prompts: A list of prompts. Each prompt can be a string or a list
402
+ of token IDs.
403
+ beam_width: The number of beams to keep at each step.
404
+ max_tokens: The max number of tokens to generate for each prompt.
405
+
406
+ TODO: how does beam search work together with length penalty, frequency
407
+ penalty, and stopping criteria, etc.?
408
+ """
409
+
410
+ tokenizer = self.get_tokenizer()
411
+ # generate 2 * beam_width candidates at each step
412
+ # following the huggingface transformers implementation
413
+ # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
414
+ beam_search_params = SamplingParams(logprobs=2 * beam_width,
415
+ max_tokens=1,
416
+ temperature=0.0)
417
+ instances: List[BeamSearchInstance] = []
418
+
419
+ for prompt in prompts:
420
+ prompt_tokens = prompt if isinstance(
421
+ prompt, list) else tokenizer.encode(prompt)
422
+ instances.append(BeamSearchInstance(prompt_tokens))
423
+
424
+ for _ in range(max_tokens):
425
+ all_beams: List[BeamSearchSequence] = list(
426
+ sum((instance.beams for instance in instances), []))
427
+ pos = [0] + list(
428
+ itertools.accumulate(
429
+ len(instance.beams) for instance in instances))
430
+ instance_start_and_end: List[Tuple[int, int]] = list(
431
+ zip(pos[:-1], pos[1:]))
432
+
433
+ if len(all_beams) == 0:
434
+ break
435
+
436
+ prompts_batch = [
437
+ TokensPrompt(prompt_token_ids=beam.tokens)
438
+ for beam in all_beams
439
+ ]
440
+
441
+ # only runs for one step
442
+ # we don't need to use tqdm here
443
+ output = self.generate(prompts_batch,
444
+ sampling_params=beam_search_params,
445
+ use_tqdm=False)
446
+
447
+ for (start, end), instance in zip(instance_start_and_end,
448
+ instances):
449
+ instance_new_beams = []
450
+ for i in range(start, end):
451
+ current_beam = all_beams[i]
452
+ result = output[i]
453
+
454
+ if result.outputs[0].logprobs is not None:
455
+ # if `result.outputs[0].logprobs` is None, it means
456
+ # the sequence is completed because of the max-model-len
457
+ # or abortion. we don't need to add it to the new beams.
458
+ logprobs = result.outputs[0].logprobs[0]
459
+ for token_id, logprob_obj in logprobs.items():
460
+ new_beam = BeamSearchSequence(
461
+ tokens=current_beam.tokens + [token_id],
462
+ cum_logprob=current_beam.cum_logprob +
463
+ logprob_obj.logprob)
464
+
465
+ if token_id == tokenizer.eos_token_id and \
466
+ not ignore_eos:
467
+ instance.completed.append(new_beam)
468
+ else:
469
+ instance_new_beams.append(new_beam)
470
+ sorted_beams = sorted(instance_new_beams,
471
+ key=lambda x: x.cum_logprob,
472
+ reverse=True)
473
+ instance.beams = sorted_beams[:beam_width]
474
+
475
+ outputs = []
476
+ for instance in instances:
477
+ instance.completed.extend(instance.beams)
478
+ sorted_completed = sorted(instance.completed,
479
+ key=lambda x: x.cum_logprob,
480
+ reverse=True)
481
+ best_beams = sorted_completed[:beam_width]
482
+
483
+ for beam in best_beams:
484
+ beam.text = tokenizer.decode(beam.tokens)
485
+ outputs.append(BeamSearchOutput(sequences=best_beams))
486
+
487
+ return outputs
488
+
489
def chat(
490
self,
491
messages: List[ChatCompletionMessageParam],