A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
[WIP] Add server script
Woosuk Kwon committed
Feb 24, 2023 at 01:33 UTC
afdbe5d3736f156e2a2c0afd13891f47a416baf5
2 files changed
+113
cacheflow/master/scheduler.py
+6
@@ -236,6 +236,12 @@ class Scheduler:
236
del self.max_num_steps[seq_group.group_id]
237
del self.stop_token_ids[seq_group.group_id]
238
# TODO: Return the seq_group to the client.
239
+ from transformers import AutoTokenizer
240
+ tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
241
+ for seq in seq_group.seqs:
242
+ token_ids = seq.get_token_ids()
243
+ output = tokenizer.decode(token_ids, skip_special_tokens=True)
244
+ print(f'Seq {seq.seq_id}: {output}')
245
else:
246
running.append(seq_group)
247
self.running = running
server.py
new
+107
@@ -0,0 +1,107 @@
1
+import argparse
2
+from typing import List
3
+
4
+from cacheflow.master.scheduler import Scheduler
5
+from cacheflow.worker.controller import Controller
6
+
7
+parser = argparse.ArgumentParser(description='CacheFlow server')
8
+parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
9
+parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
10
+parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
11
+parser.add_argument('--block-size', type=int, default=8, help='block size')
12
+parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks')
13
+parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks')
14
+
15
+
16
+def main():
17
+ args = parser.parse_args()
18
+
19
+ # Create controllers.
20
+ controllers: List[Controller] = []
21
+ for i in range(args.num_nodes):
22
+ controller = Controller(
23
+ node_id=i,
24
+ num_workers=args.num_workers,
25
+ model_name=args.model,
26
+ block_size=args.block_size,
27
+ num_gpu_blocks=args.num_gpu_blocks,
28
+ num_cpu_blocks=args.num_cpu_blocks,
29
+ dtype='float',
30
+ )
31
+ controllers.append(controller)
32
+
33
+ # Create a scheduler.
34
+ scheduler = Scheduler(
35
+ controllers=controllers,
36
+ block_size=args.block_size,
37
+ num_gpu_blocks=args.num_gpu_blocks,
38
+ num_cpu_blocks=args.num_cpu_blocks,
39
+ )
40
+ # Connect the controllers.
41
+ for i in range(len(controllers) - 1):
42
+ controllers[i].set_next(controllers[i + 1])
43
+ controllers[-1].set_next(scheduler)
44
+
45
+ # seq_groups, max_num_steps, stop_token_ids = generate_inputs(1000, args.block_size)
46
+ seq_groups, max_num_steps, stop_token_ids = test_inputs(args.block_size)
47
+ scheduler.pending.extend(seq_groups)
48
+ scheduler.max_num_steps.update(max_num_steps)
49
+ scheduler.stop_token_ids.update(stop_token_ids)
50
+
51
+ while scheduler.pending or scheduler.running:
52
+ scheduler.prepare()
53
+ scheduler.step()
54
+
55
+
56
+def test_inputs(block_size):
57
+ from cacheflow.sequence import Sequence
58
+ from cacheflow.sequence import SequenceGroup
59
+ from cacheflow.utils import Counter
60
+
61
+ from transformers import AutoTokenizer
62
+ tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
63
+ prompt = "Hello, I'm am conscious and"
64
+ prompt_tokens = tokenizer.encode(prompt)
65
+
66
+ seq = Sequence(0, prompt_tokens, block_size=block_size)
67
+ seq_group = SequenceGroup(0, [seq])
68
+ seq_groups = [seq_group]
69
+ max_num_steps = {0: 8}
70
+ stop_token_ids = {0: []}
71
+ return seq_groups, max_num_steps, stop_token_ids
72
+
73
+
74
+def generate_inputs(num_inputs, block_size):
75
+ import random
76
+ random.seed(0)
77
+
78
+ from cacheflow.sequence import Sequence
79
+ from cacheflow.sequence import SequenceGroup
80
+ from cacheflow.utils import Counter
81
+
82
+ seq_group_counter = Counter()
83
+ seq_counter = Counter()
84
+
85
+ max_num_steps = {}
86
+ stop_token_ids = {}
87
+ seq_groups = []
88
+ for _ in range(num_inputs):
89
+ seq_group_id = next(seq_group_counter)
90
+
91
+ prompt_len = random.randint(16, 128)
92
+ max_num_steps[seq_group_id] = random.randint(32, 1024)
93
+ stop_token_ids[seq_group_id] = []
94
+
95
+ seqs = []
96
+ for _ in range(2):
97
+ seq_id = next(seq_counter)
98
+ seq = Sequence(seq_id, [0] * prompt_len, block_size=block_size)
99
+ seqs.append(seq)
100
+ seq_group = SequenceGroup(seq_group_id, seqs)
101
+ seq_groups.append(seq_group)
102
+
103
+ return seq_groups, max_num_steps, stop_token_ids
104
+
105
+
106
+if __name__ == '__main__':
107
+ main()