A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] fix beam search input errors and latency benchmark script (#11875)
Signed-off-by: Ye Qi <yeq@meta.com> Co-authored-by: yeq <yeq@devgpu004.lla3.facebook.com>
Ye (Charlotte) Qi committed
Jan 9, 2025 at 01:36 UTC
1d967acb45d5d18434409b822f105f087e379eee
2 files changed
+23
-10
benchmarks/benchmark_latency.py
+17
-6
@@ -13,6 +13,7 @@ from tqdm import tqdm
13
from vllm import LLM, SamplingParams
14
from vllm.engine.arg_utils import EngineArgs
15
from vllm.inputs import PromptType
16
+from vllm.sampling_params import BeamSearchParams
17
from vllm.utils import FlexibleArgumentParser
18
19
@@ -40,6 +41,20 @@ def main(args: argparse.Namespace):
41
"prompt_token_ids": batch
42
} for batch in dummy_prompt_token_ids.tolist()]
43
44
+ def llm_generate():
45
+ if not args.use_beam_search:
46
+ llm.generate(dummy_prompts,
47
+ sampling_params=sampling_params,
48
+ use_tqdm=False)
49
+ else:
50
+ llm.beam_search(
51
+ dummy_prompts,
52
+ BeamSearchParams(
53
+ beam_width=args.n,
54
+ max_tokens=args.output_len,
55
+ ignore_eos=True,
56
+ ))
57
+
58
def run_to_completion(profile_dir: Optional[str] = None):
59
if profile_dir:
60
with torch.profiler.profile(
@@ -49,15 +64,11 @@ def main(args: argparse.Namespace):
64
],
65
on_trace_ready=torch.profiler.tensorboard_trace_handler(
66
str(profile_dir))) as p:
52
- llm.generate(dummy_prompts,
53
- sampling_params=sampling_params,
54
- use_tqdm=False)
67
+ llm_generate()
68
print(p.key_averages().table(sort_by="self_cuda_time_total"))
69
else:
70
start_time = time.perf_counter()
58
- llm.generate(dummy_prompts,
59
- sampling_params=sampling_params,
60
- use_tqdm=False)
71
+ llm_generate()
72
end_time = time.perf_counter()
73
latency = end_time - start_time
74
return latency
vllm/entrypoints/llm.py
+6
-4
@@ -21,7 +21,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
21
parse_chat_messages,
22
resolve_chat_template_content_format)
23
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
24
-from vllm.inputs.parse import parse_and_batch_prompt
24
+from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
25
from vllm.logger import init_logger
26
from vllm.lora.request import LoRARequest
27
from vllm.model_executor.guided_decoding.guided_fields import (
@@ -457,7 +457,7 @@ class LLM:
457
458
def beam_search(
459
self,
460
- prompts: List[Union[str, List[int]]],
460
+ prompts: List[Union[TokensPrompt, TextPrompt]],
461
params: BeamSearchParams,
462
) -> List[BeamSearchOutput]:
463
"""
@@ -493,8 +493,10 @@ class LLM:
493
instances: List[BeamSearchInstance] = []
494
495
for prompt in prompts:
496
- prompt_tokens = prompt if isinstance(
497
- prompt, list) else tokenizer.encode(prompt)
496
+ if is_token_prompt(prompt):
497
+ prompt_tokens = prompt["prompt_token_ids"]
498
+ else:
499
+ prompt_tokens = tokenizer.encode(prompt["prompt"])
500
instances.append(BeamSearchInstance(prompt_tokens))
501
502
for _ in range(max_tokens):