@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / afdbe5d373
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

[WIP] Add server script

Woosuk Kwon committed Feb 24, 2023 at 01:33 UTC afdbe5d3736f156e2a2c0afd13891f47a416baf5
2 files changed +113
cacheflow/master/scheduler.py
+6
@@ -236,6 +236,12 @@ class Scheduler:
236 del self.max_num_steps[seq_group.group_id]
237 del self.stop_token_ids[seq_group.group_id]
238 # TODO: Return the seq_group to the client.
239 + from transformers import AutoTokenizer
240 + tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
241 + for seq in seq_group.seqs:
242 + token_ids = seq.get_token_ids()
243 + output = tokenizer.decode(token_ids, skip_special_tokens=True)
244 + print(f'Seq {seq.seq_id}: {output}')
245 else:
246 running.append(seq_group)
247 self.running = running
server.py new
+107
@@ -0,0 +1,107 @@
1 +import argparse
2 +from typing import List
3 +
4 +from cacheflow.master.scheduler import Scheduler
5 +from cacheflow.worker.controller import Controller
6 +
7 +parser = argparse.ArgumentParser(description='CacheFlow server')
8 +parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
9 +parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
10 +parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
11 +parser.add_argument('--block-size', type=int, default=8, help='block size')
12 +parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks')
13 +parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks')
14 +
15 +
16 +def main():
17 + args = parser.parse_args()
18 +
19 + # Create controllers.
20 + controllers: List[Controller] = []
21 + for i in range(args.num_nodes):
22 + controller = Controller(
23 + node_id=i,
24 + num_workers=args.num_workers,
25 + model_name=args.model,
26 + block_size=args.block_size,
27 + num_gpu_blocks=args.num_gpu_blocks,
28 + num_cpu_blocks=args.num_cpu_blocks,
29 + dtype='float',
30 + )
31 + controllers.append(controller)
32 +
33 + # Create a scheduler.
34 + scheduler = Scheduler(
35 + controllers=controllers,
36 + block_size=args.block_size,
37 + num_gpu_blocks=args.num_gpu_blocks,
38 + num_cpu_blocks=args.num_cpu_blocks,
39 + )
40 + # Connect the controllers.
41 + for i in range(len(controllers) - 1):
42 + controllers[i].set_next(controllers[i + 1])
43 + controllers[-1].set_next(scheduler)
44 +
45 + # seq_groups, max_num_steps, stop_token_ids = generate_inputs(1000, args.block_size)
46 + seq_groups, max_num_steps, stop_token_ids = test_inputs(args.block_size)
47 + scheduler.pending.extend(seq_groups)
48 + scheduler.max_num_steps.update(max_num_steps)
49 + scheduler.stop_token_ids.update(stop_token_ids)
50 +
51 + while scheduler.pending or scheduler.running:
52 + scheduler.prepare()
53 + scheduler.step()
54 +
55 +
56 +def test_inputs(block_size):
57 + from cacheflow.sequence import Sequence
58 + from cacheflow.sequence import SequenceGroup
59 + from cacheflow.utils import Counter
60 +
61 + from transformers import AutoTokenizer
62 + tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
63 + prompt = "Hello, I'm am conscious and"
64 + prompt_tokens = tokenizer.encode(prompt)
65 +
66 + seq = Sequence(0, prompt_tokens, block_size=block_size)
67 + seq_group = SequenceGroup(0, [seq])
68 + seq_groups = [seq_group]
69 + max_num_steps = {0: 8}
70 + stop_token_ids = {0: []}
71 + return seq_groups, max_num_steps, stop_token_ids
72 +
73 +
74 +def generate_inputs(num_inputs, block_size):
75 + import random
76 + random.seed(0)
77 +
78 + from cacheflow.sequence import Sequence
79 + from cacheflow.sequence import SequenceGroup
80 + from cacheflow.utils import Counter
81 +
82 + seq_group_counter = Counter()
83 + seq_counter = Counter()
84 +
85 + max_num_steps = {}
86 + stop_token_ids = {}
87 + seq_groups = []
88 + for _ in range(num_inputs):
89 + seq_group_id = next(seq_group_counter)
90 +
91 + prompt_len = random.randint(16, 128)
92 + max_num_steps[seq_group_id] = random.randint(32, 1024)
93 + stop_token_ids[seq_group_id] = []
94 +
95 + seqs = []
96 + for _ in range(2):
97 + seq_id = next(seq_counter)
98 + seq = Sequence(seq_id, [0] * prompt_len, block_size=block_size)
99 + seqs.append(seq)
100 + seq_group = SequenceGroup(seq_group_id, seqs)
101 + seq_groups.append(seq_group)
102 +
103 + return seq_groups, max_num_steps, stop_token_ids
104 +
105 +
106 +if __name__ == '__main__':
107 + main()