A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Clean up the server script
Woosuk Kwon committed
Feb 24, 2023 at 11:56 UTC
fa16389a2e2fa8207f2489e64657897eec17662c
1 file changed
+25
-64
server.py
+25
-64
@@ -1,6 +1,7 @@
1
import argparse
2
from typing import List
3
4
+from cacheflow.master.frontend import Frontend
5
from cacheflow.master.scheduler import Scheduler
6
from cacheflow.worker.controller import Controller
7
@@ -8,15 +9,15 @@ parser = argparse.ArgumentParser(description='CacheFlow server')
9
parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
10
parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
11
parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
11
-parser.add_argument('--block-size', type=int, default=8, help='block size')
12
-parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks')
13
-parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks')
12
+parser.add_argument('--block-size', type=int, default=8, help='token block size')
13
+# TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
14
+parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
15
+parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks (per GPU)')
16
+args = parser.parse_args()
17
18
19
def main():
17
- args = parser.parse_args()
18
-
19
- # Create controllers.
20
+ # Create a controller for each node.
21
controllers: List[Controller] = []
22
for i in range(args.num_nodes):
23
controller = Controller(
@@ -26,12 +27,18 @@ def main():
27
block_size=args.block_size,
28
num_gpu_blocks=args.num_gpu_blocks,
29
num_cpu_blocks=args.num_cpu_blocks,
29
- dtype='float',
30
)
31
controllers.append(controller)
32
33
+ # Create a frontend.
34
+ frontend = Frontend(
35
+ model_name=args.model,
36
+ block_size=args.block_size,
37
+ )
38
+
39
# Create a scheduler.
40
scheduler = Scheduler(
41
+ frontend=frontend,
42
controllers=controllers,
43
block_size=args.block_size,
44
num_gpu_blocks=args.num_gpu_blocks,
@@ -42,65 +49,19 @@ def main():
49
controllers[i].set_next(controllers[i + 1])
50
controllers[-1].set_next(scheduler)
51
45
- # seq_groups, max_num_steps, stop_token_ids = generate_inputs(1000, args.block_size)
46
- seq_groups, max_num_steps, stop_token_ids = test_inputs(args.block_size)
47
- scheduler.pending.extend(seq_groups)
48
- scheduler.max_num_steps.update(max_num_steps)
49
- scheduler.stop_token_ids.update(stop_token_ids)
52
+ test_inputs = [
53
+ 'Ion Stoica is a',
54
+ 'UC Berkeley is',
55
+ 'The future of cloud computing is',
56
+ ]
57
+ for prompt in test_inputs:
58
+ frontend.query(prompt)
59
51
- while scheduler.pending or scheduler.running:
52
- scheduler.prepare()
60
+ # FIXME
61
+ while True:
62
scheduler.step()
54
-
55
-
56
-def test_inputs(block_size):
57
- from cacheflow.sequence import Sequence
58
- from cacheflow.sequence import SequenceGroup
59
- from cacheflow.utils import Counter
60
-
61
- from transformers import AutoTokenizer
62
- tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
63
- prompt = "Hello, I'm am conscious and"
64
- prompt_tokens = tokenizer.encode(prompt)
65
-
66
- seq = Sequence(0, prompt_tokens, block_size=block_size)
67
- seq_group = SequenceGroup(0, [seq])
68
- seq_groups = [seq_group]
69
- max_num_steps = {0: 8}
70
- stop_token_ids = {0: []}
71
- return seq_groups, max_num_steps, stop_token_ids
72
-
73
-
74
-def generate_inputs(num_inputs, block_size):
75
- import random
76
- random.seed(0)
77
-
78
- from cacheflow.sequence import Sequence
79
- from cacheflow.sequence import SequenceGroup
80
- from cacheflow.utils import Counter
81
-
82
- seq_group_counter = Counter()
83
- seq_counter = Counter()
84
-
85
- max_num_steps = {}
86
- stop_token_ids = {}
87
- seq_groups = []
88
- for _ in range(num_inputs):
89
- seq_group_id = next(seq_group_counter)
90
-
91
- prompt_len = random.randint(16, 128)
92
- max_num_steps[seq_group_id] = random.randint(32, 1024)
93
- stop_token_ids[seq_group_id] = []
94
-
95
- seqs = []
96
- for _ in range(2):
97
- seq_id = next(seq_counter)
98
- seq = Sequence(seq_id, [0] * prompt_len, block_size=block_size)
99
- seqs.append(seq)
100
- seq_group = SequenceGroup(seq_group_id, seqs)
101
- seq_groups.append(seq_group)
102
-
103
- return seq_groups, max_num_steps, stop_token_ids
63
+ if not scheduler.pending and not scheduler.running:
64
+ break
65
66
67
if __name__ == '__main__':