@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d967acb45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] fix beam search input errors and latency benchmark script (#11875)

Signed-off-by: Ye Qi <yeq@meta.com> Co-authored-by: yeq <yeq@devgpu004.lla3.facebook.com>

Ye (Charlotte) Qi committed Jan 9, 2025 at 01:36 UTC 1d967acb45d5d18434409b822f105f087e379eee
2 files changed +23 -10
benchmarks/benchmark_latency.py
+17 -6
@@ -13,6 +13,7 @@ from tqdm import tqdm
13 from vllm import LLM, SamplingParams
14 from vllm.engine.arg_utils import EngineArgs
15 from vllm.inputs import PromptType
16 +from vllm.sampling_params import BeamSearchParams
17 from vllm.utils import FlexibleArgumentParser
18
19
@@ -40,6 +41,20 @@ def main(args: argparse.Namespace):
41 "prompt_token_ids": batch
42 } for batch in dummy_prompt_token_ids.tolist()]
43
44 + def llm_generate():
45 + if not args.use_beam_search:
46 + llm.generate(dummy_prompts,
47 + sampling_params=sampling_params,
48 + use_tqdm=False)
49 + else:
50 + llm.beam_search(
51 + dummy_prompts,
52 + BeamSearchParams(
53 + beam_width=args.n,
54 + max_tokens=args.output_len,
55 + ignore_eos=True,
56 + ))
57 +
58 def run_to_completion(profile_dir: Optional[str] = None):
59 if profile_dir:
60 with torch.profiler.profile(
@@ -49,15 +64,11 @@ def main(args: argparse.Namespace):
64 ],
65 on_trace_ready=torch.profiler.tensorboard_trace_handler(
66 str(profile_dir))) as p:
52 - llm.generate(dummy_prompts,
53 - sampling_params=sampling_params,
54 - use_tqdm=False)
67 + llm_generate()
68 print(p.key_averages().table(sort_by="self_cuda_time_total"))
69 else:
70 start_time = time.perf_counter()
58 - llm.generate(dummy_prompts,
59 - sampling_params=sampling_params,
60 - use_tqdm=False)
71 + llm_generate()
72 end_time = time.perf_counter()
73 latency = end_time - start_time
74 return latency
vllm/entrypoints/llm.py
+6 -4
@@ -21,7 +21,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
21 parse_chat_messages,
22 resolve_chat_template_content_format)
23 from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
24 -from vllm.inputs.parse import parse_and_batch_prompt
24 +from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
25 from vllm.logger import init_logger
26 from vllm.lora.request import LoRARequest
27 from vllm.model_executor.guided_decoding.guided_fields import (
@@ -457,7 +457,7 @@ class LLM:
457
458 def beam_search(
459 self,
460 - prompts: List[Union[str, List[int]]],
460 + prompts: List[Union[TokensPrompt, TextPrompt]],
461 params: BeamSearchParams,
462 ) -> List[BeamSearchOutput]:
463 """
@@ -493,8 +493,10 @@ class LLM:
493 instances: List[BeamSearchInstance] = []
494
495 for prompt in prompts:
496 - prompt_tokens = prompt if isinstance(
497 - prompt, list) else tokenizer.encode(prompt)
496 + if is_token_prompt(prompt):
497 + prompt_tokens = prompt["prompt_token_ids"]
498 + else:
499 + prompt_tokens = tokenizer.encode(prompt["prompt"])
500 instances.append(BeamSearchInstance(prompt_tokens))
501
502 for _ in range(max_tokens):