@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa16389a2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Clean up the server script

Woosuk Kwon committed Feb 24, 2023 at 11:56 UTC fa16389a2e2fa8207f2489e64657897eec17662c
1 file changed +25 -64
server.py
+25 -64
@@ -1,6 +1,7 @@
1 import argparse
2 from typing import List
3
4 +from cacheflow.master.frontend import Frontend
5 from cacheflow.master.scheduler import Scheduler
6 from cacheflow.worker.controller import Controller
7
@@ -8,15 +9,15 @@ parser = argparse.ArgumentParser(description='CacheFlow server')
9 parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
10 parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
11 parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
11 -parser.add_argument('--block-size', type=int, default=8, help='block size')
12 -parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks')
13 -parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks')
12 +parser.add_argument('--block-size', type=int, default=8, help='token block size')
13 +# TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
14 +parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
15 +parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks (per GPU)')
16 +args = parser.parse_args()
17
18
19 def main():
17 - args = parser.parse_args()
18 -
19 - # Create controllers.
20 + # Create a controller for each node.
21 controllers: List[Controller] = []
22 for i in range(args.num_nodes):
23 controller = Controller(
@@ -26,12 +27,18 @@ def main():
27 block_size=args.block_size,
28 num_gpu_blocks=args.num_gpu_blocks,
29 num_cpu_blocks=args.num_cpu_blocks,
29 - dtype='float',
30 )
31 controllers.append(controller)
32
33 + # Create a frontend.
34 + frontend = Frontend(
35 + model_name=args.model,
36 + block_size=args.block_size,
37 + )
38 +
39 # Create a scheduler.
40 scheduler = Scheduler(
41 + frontend=frontend,
42 controllers=controllers,
43 block_size=args.block_size,
44 num_gpu_blocks=args.num_gpu_blocks,
@@ -42,65 +49,19 @@ def main():
49 controllers[i].set_next(controllers[i + 1])
50 controllers[-1].set_next(scheduler)
51
45 - # seq_groups, max_num_steps, stop_token_ids = generate_inputs(1000, args.block_size)
46 - seq_groups, max_num_steps, stop_token_ids = test_inputs(args.block_size)
47 - scheduler.pending.extend(seq_groups)
48 - scheduler.max_num_steps.update(max_num_steps)
49 - scheduler.stop_token_ids.update(stop_token_ids)
52 + test_inputs = [
53 + 'Ion Stoica is a',
54 + 'UC Berkeley is',
55 + 'The future of cloud computing is',
56 + ]
57 + for prompt in test_inputs:
58 + frontend.query(prompt)
59
51 - while scheduler.pending or scheduler.running:
52 - scheduler.prepare()
60 + # FIXME
61 + while True:
62 scheduler.step()
54 -
55 -
56 -def test_inputs(block_size):
57 - from cacheflow.sequence import Sequence
58 - from cacheflow.sequence import SequenceGroup
59 - from cacheflow.utils import Counter
60 -
61 - from transformers import AutoTokenizer
62 - tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
63 - prompt = "Hello, I'm am conscious and"
64 - prompt_tokens = tokenizer.encode(prompt)
65 -
66 - seq = Sequence(0, prompt_tokens, block_size=block_size)
67 - seq_group = SequenceGroup(0, [seq])
68 - seq_groups = [seq_group]
69 - max_num_steps = {0: 8}
70 - stop_token_ids = {0: []}
71 - return seq_groups, max_num_steps, stop_token_ids
72 -
73 -
74 -def generate_inputs(num_inputs, block_size):
75 - import random
76 - random.seed(0)
77 -
78 - from cacheflow.sequence import Sequence
79 - from cacheflow.sequence import SequenceGroup
80 - from cacheflow.utils import Counter
81 -
82 - seq_group_counter = Counter()
83 - seq_counter = Counter()
84 -
85 - max_num_steps = {}
86 - stop_token_ids = {}
87 - seq_groups = []
88 - for _ in range(num_inputs):
89 - seq_group_id = next(seq_group_counter)
90 -
91 - prompt_len = random.randint(16, 128)
92 - max_num_steps[seq_group_id] = random.randint(32, 1024)
93 - stop_token_ids[seq_group_id] = []
94 -
95 - seqs = []
96 - for _ in range(2):
97 - seq_id = next(seq_counter)
98 - seq = Sequence(seq_id, [0] * prompt_len, block_size=block_size)
99 - seqs.append(seq)
100 - seq_group = SequenceGroup(seq_group_id, seqs)
101 - seq_groups.append(seq_group)
102 -
103 - return seq_groups, max_num_steps, stop_token_ids
63 + if not scheduler.pending and not scheduler.running:
64 + break
65
66
67 if __name__ == '__main__':