@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0250dd68c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

re-implement beam search on top of vllm core (#8726)

Co-authored-by: Brendan Wong <bjwpokemon@gmail.com>

youkaichao committed Sep 23, 2024 at 22:08 UTC 0250dd68c5df12ead29d2ec7d922855c9a257b06
4 files changed +171 -9
benchmarks/benchmark_throughput.py
+20 -4
@@ -90,6 +90,7 @@ def run_vllm(
90 download_dir: Optional[str] = None,
91 load_format: str = EngineArgs.load_format,
92 disable_async_output_proc: bool = False,
93 + use_new_beam_search_impl: bool = False,
94 ) -> float:
95 from vllm import LLM, SamplingParams
96 llm = LLM(
@@ -132,9 +133,23 @@ def run_vllm(
133 max_tokens=output_len,
134 ))
135
135 - start = time.perf_counter()
136 - llm.generate(prompts, sampling_params, use_tqdm=True)
137 - end = time.perf_counter()
136 + if not use_new_beam_search_impl:
137 + start = time.perf_counter()
138 + llm.generate(prompts, sampling_params, use_tqdm=True)
139 + end = time.perf_counter()
140 + else:
141 + assert use_beam_search
142 + prompts = [prompt for prompt, _, _ in requests]
143 + # output_len should be the same for all requests.
144 + output_len = requests[0][2]
145 + for prompt, input_len, _output_len in requests:
146 + assert _output_len == output_len
147 + start = time.perf_counter()
148 + llm.beam_search(prompts,
149 + beam_width=n,
150 + max_tokens=output_len,
151 + ignore_eos=True)
152 + end = time.perf_counter()
153 return end - start
154
155
@@ -336,7 +351,7 @@ def main(args: argparse.Namespace):
351 run_args.append(args.disable_frontend_multiprocessing)
352 elapsed_time = uvloop.run(run_vllm_async(*run_args))
353 else:
339 - elapsed_time = run_vllm(*run_args)
354 + elapsed_time = run_vllm(*run_args, args.use_new_beam_search_impl)
355 elif args.backend == "hf":
356 assert args.tensor_parallel_size == 1
357 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -396,6 +411,7 @@ if __name__ == "__main__":
411 default=1,
412 help="Number of generated sequences per prompt.")
413 parser.add_argument("--use-beam-search", action="store_true")
414 + parser.add_argument("--use-new-beam-search-impl", action="store_true")
415 parser.add_argument("--num-prompts",
416 type=int,
417 default=1000,
tests/conftest.py
+14
@@ -798,6 +798,20 @@ class VllmRunner:
798 outputs = self.generate(prompts, beam_search_params)
799 return outputs
800
801 + def generate_beam_search_new(
802 + self,
803 + prompts: Union[List[str], List[List[int]]],
804 + beam_width: int,
805 + max_tokens: int,
806 + ) -> List[Tuple[List[List[int]], List[str]]]:
807 + outputs = self.model.beam_search(prompts, beam_width, max_tokens)
808 + returned_outputs = []
809 + for output in outputs:
810 + token_ids = [x.tokens for x in output.sequences]
811 + texts = [x.text for x in output.sequences]
812 + returned_outputs.append((token_ids, texts))
813 + return returned_outputs
814 +
815 def encode(self, prompts: List[str]) -> List[List[float]]:
816 req_outputs = self.model.encode(prompts)
817 outputs = []
tests/samplers/test_beam_search.py
+3 -3
@@ -9,7 +9,7 @@ import pytest
9 # 1. Increase max_tokens to 256.
10 # 2. Increase beam_width to 8.
11 # 3. Use the model "huggyllama/llama-7b".
12 -MAX_TOKENS = [128]
12 +MAX_TOKENS = [64]
13 BEAM_WIDTHS = [4]
14 MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
15
@@ -33,8 +33,8 @@ def test_beam_search_single_input(
33 max_tokens)
34
35 with vllm_runner(model, dtype=dtype) as vllm_model:
36 - vllm_outputs = vllm_model.generate_beam_search(example_prompts,
37 - beam_width, max_tokens)
36 + vllm_outputs = vllm_model.generate_beam_search_new(
37 + example_prompts, beam_width, max_tokens)
38
39 for i in range(len(example_prompts)):
40 hf_output_ids, hf_output_texts = hf_outputs[i]
vllm/entrypoints/llm.py
+134 -2
@@ -1,6 +1,8 @@
1 +import itertools
2 from contextlib import contextmanager
2 -from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Union, cast,
3 - overload)
3 +from dataclasses import dataclass
4 +from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
5 + Union, cast, overload)
6
7 from tqdm import tqdm
8
@@ -30,6 +32,37 @@ from vllm.utils import Counter, deprecate_kwargs, is_list_of
32 logger = init_logger(__name__)
33
34
35 +@dataclass
36 +class BeamSearchSequence:
37 + """A sequence for beam search.
38 + It keeps track of the tokens and the log probability of the sequence.
39 + The text field is optional and will only be filled when the sequence is
40 + about to be returned to the user.
41 + """
42 + # The tokens includes the prompt.
43 + tokens: List[int]
44 + cum_logprob: float = 0.0
45 + text: Optional[str] = None
46 +
47 +
48 +@dataclass
49 +class BeamSearchOutput:
50 + """The output of beam search.
51 + It contains the list of the best beam search sequences.
52 + The length of the list is equal to the beam width.
53 + """
54 + sequences: List[BeamSearchSequence]
55 +
56 +
57 +class BeamSearchInstance:
58 +
59 + def __init__(self, prompt_tokens: List[int]):
60 + self.beams: List[BeamSearchSequence] = [
61 + BeamSearchSequence(tokens=prompt_tokens)
62 + ]
63 + self.completed: List[BeamSearchSequence] = []
64 +
65 +
66 class LLM:
67 """An LLM for generating texts from given prompts and sampling parameters.
68
@@ -354,6 +387,105 @@ class LLM:
387 outputs = self._run_engine(use_tqdm=use_tqdm)
388 return LLMEngine.validate_outputs(outputs, RequestOutput)
389
390 + def beam_search(
391 + self,
392 + prompts: List[Union[str, List[int]]],
393 + beam_width: int,
394 + max_tokens: int,
395 + ignore_eos: bool = False,
396 + ) -> List[BeamSearchOutput]:
397 + """
398 + Generate sequences using beam search.
399 +
400 + Args:
401 + prompts: A list of prompts. Each prompt can be a string or a list
402 + of token IDs.
403 + beam_width: The number of beams to keep at each step.
404 + max_tokens: The max number of tokens to generate for each prompt.
405 +
406 + TODO: how does beam search work together with length penalty, frequency
407 + penalty, and stopping criteria, etc.?
408 + """
409 +
410 + tokenizer = self.get_tokenizer()
411 + # generate 2 * beam_width candidates at each step
412 + # following the huggingface transformers implementation
413 + # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
414 + beam_search_params = SamplingParams(logprobs=2 * beam_width,
415 + max_tokens=1,
416 + temperature=0.0)
417 + instances: List[BeamSearchInstance] = []
418 +
419 + for prompt in prompts:
420 + prompt_tokens = prompt if isinstance(
421 + prompt, list) else tokenizer.encode(prompt)
422 + instances.append(BeamSearchInstance(prompt_tokens))
423 +
424 + for _ in range(max_tokens):
425 + all_beams: List[BeamSearchSequence] = list(
426 + sum((instance.beams for instance in instances), []))
427 + pos = [0] + list(
428 + itertools.accumulate(
429 + len(instance.beams) for instance in instances))
430 + instance_start_and_end: List[Tuple[int, int]] = list(
431 + zip(pos[:-1], pos[1:]))
432 +
433 + if len(all_beams) == 0:
434 + break
435 +
436 + prompts_batch = [
437 + TokensPrompt(prompt_token_ids=beam.tokens)
438 + for beam in all_beams
439 + ]
440 +
441 + # only runs for one step
442 + # we don't need to use tqdm here
443 + output = self.generate(prompts_batch,
444 + sampling_params=beam_search_params,
445 + use_tqdm=False)
446 +
447 + for (start, end), instance in zip(instance_start_and_end,
448 + instances):
449 + instance_new_beams = []
450 + for i in range(start, end):
451 + current_beam = all_beams[i]
452 + result = output[i]
453 +
454 + if result.outputs[0].logprobs is not None:
455 + # if `result.outputs[0].logprobs` is None, it means
456 + # the sequence is completed because of the max-model-len
457 + # or abortion. we don't need to add it to the new beams.
458 + logprobs = result.outputs[0].logprobs[0]
459 + for token_id, logprob_obj in logprobs.items():
460 + new_beam = BeamSearchSequence(
461 + tokens=current_beam.tokens + [token_id],
462 + cum_logprob=current_beam.cum_logprob +
463 + logprob_obj.logprob)
464 +
465 + if token_id == tokenizer.eos_token_id and \
466 + not ignore_eos:
467 + instance.completed.append(new_beam)
468 + else:
469 + instance_new_beams.append(new_beam)
470 + sorted_beams = sorted(instance_new_beams,
471 + key=lambda x: x.cum_logprob,
472 + reverse=True)
473 + instance.beams = sorted_beams[:beam_width]
474 +
475 + outputs = []
476 + for instance in instances:
477 + instance.completed.extend(instance.beams)
478 + sorted_completed = sorted(instance.completed,
479 + key=lambda x: x.cum_logprob,
480 + reverse=True)
481 + best_beams = sorted_completed[:beam_width]
482 +
483 + for beam in best_beams:
484 + beam.text = tokenizer.decode(beam.tokens)
485 + outputs.append(BeamSearchOutput(sequences=best_beams))
486 +
487 + return outputs
488 +
489 def chat(
490 self,
491 messages: List[ChatCompletionMessageParam],