@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ac8d36f3e5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Refactor LLMEngine demo script for clarity and modularity (#1413)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

iongpt committed Oct 30, 2023 at 18:14 UTC ac8d36f3e5776e020eb32a08eb3a2f9c60a49344
1 file changed +25 -14
examples/llm_engine_example.py
+25 -14
@@ -1,15 +1,12 @@
1 import argparse
2 +from typing import List, Tuple
3
3 -from vllm import EngineArgs, LLMEngine, SamplingParams
4 +from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
5
6
6 -def main(args: argparse.Namespace):
7 - # Parse the CLI argument and initialize the engine.
8 - engine_args = EngineArgs.from_cli_args(args)
9 - engine = LLMEngine.from_engine_args(engine_args)
10 -
11 - # Test the following prompts.
12 - test_prompts = [
7 +def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
8 + """Create a list of test prompts with their sampling parameters."""
9 + return [
10 ("A robot may not injure a human being",
11 SamplingParams(temperature=0.0, logprobs=1, prompt_logprobs=1)),
12 ("To be or not to be,",
@@ -25,22 +22,36 @@ def main(args: argparse.Namespace):
22 temperature=0.0)),
23 ]
24
28 - # Run the engine by calling `engine.step()` manually.
25 +
26 +def process_requests(engine: LLMEngine,
27 + test_prompts: List[Tuple[str, SamplingParams]]):
28 + """Continuously process a list of prompts and handle the outputs."""
29 request_id = 0
30 - while True:
31 - # To test continuous batching, we add one request at each step.
30 +
31 + while test_prompts or engine.has_unfinished_requests():
32 if test_prompts:
33 prompt, sampling_params = test_prompts.pop(0)
34 engine.add_request(str(request_id), prompt, sampling_params)
35 request_id += 1
36
37 - request_outputs = engine.step()
37 + request_outputs: List[RequestOutput] = engine.step()
38 +
39 for request_output in request_outputs:
40 if request_output.finished:
41 print(request_output)
42
42 - if not (engine.has_unfinished_requests() or test_prompts):
43 - break
43 +
44 +def initialize_engine(args: argparse.Namespace) -> LLMEngine:
45 + """Initialize the LLMEngine from the command line arguments."""
46 + engine_args = EngineArgs.from_cli_args(args)
47 + return LLMEngine.from_engine_args(engine_args)
48 +
49 +
50 +def main(args: argparse.Namespace):
51 + """Main function that sets up and runs the prompt processing."""
52 + engine = initialize_engine(args)
53 + test_prompts = create_test_prompts()
54 + process_requests(engine, test_prompts)
55
56
57 if __name__ == '__main__':