A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add controller
Woosuk Kwon committed
Feb 23, 2023 at 09:32 UTC
1f6c7ef437b7b3ce5b83ca1b5a1de54183331bdd
1 file changed
+75
cacheflow/worker/controller.py
new
+75
@@ -0,0 +1,75 @@
1
+from typing import Dict, List, Union
2
+
3
+from cacheflow.master.scheduler import Scheduler
4
+from cacheflow.worker.worker import Worker
5
+
6
+
7
+class Controller:
8
+
9
+ def __init__(
10
+ self,
11
+ node_id: int,
12
+ num_workers: int,
13
+ model_name: str,
14
+ block_size: int,
15
+ num_gpu_blocks: int,
16
+ num_cpu_blocks: int,
17
+ ) -> None:
18
+ self.node_id = node_id
19
+ self.num_workers = num_workers
20
+ self.model_name = model_name
21
+ self.block_size = block_size
22
+ self.num_gpu_blocks = num_gpu_blocks
23
+ self.num_cpu_blocks = num_cpu_blocks
24
+
25
+ # Which pipeline stage is this node assigned to?
26
+ self.is_first_stage = node_id == 0
27
+ self.is_last_stage = False
28
+
29
+ self.workers: List[Worker] = []
30
+ for i in range(num_workers):
31
+ worker = Worker(
32
+ worker_id=node_id + i,
33
+ gpu_id=i,
34
+ model_name=model_name,
35
+ block_size=block_size,
36
+ num_gpu_blocks=num_gpu_blocks,
37
+ num_cpu_blocks=num_cpu_blocks,
38
+ )
39
+ self.workers.append(worker)
40
+
41
+ def set_next(
42
+ self,
43
+ next_node: Union['Controller', 'Scheduler'],
44
+ ) -> None:
45
+ self.next_node = next_node
46
+ self.is_last_stage = isinstance(next_node, Scheduler)
47
+
48
+ def execute_stage(
49
+ self,
50
+ prompt_tokens: Dict[int, List[int]],
51
+ generation_tokens: Dict[int, int],
52
+ context_lens: Dict[int, int],
53
+ block_tables: Dict[int, List[int]],
54
+ blocks_to_swap_in: Dict[int, int],
55
+ blocks_to_swap_out: Dict[int, int],
56
+ blocks_to_copy: Dict[int, int],
57
+ ) -> None:
58
+ # FIXME: Support tensor parallelism.
59
+ assert len(self.workers) == 1
60
+ worker = self.workers[0]
61
+ output = worker.execute_stage(
62
+ prompt_tokens,
63
+ generation_tokens,
64
+ context_lens,
65
+ block_tables,
66
+ blocks_to_swap_in,
67
+ blocks_to_swap_out,
68
+ blocks_to_copy,
69
+ )
70
+
71
+ if self.is_last_stage:
72
+ self.next_node.post_step(output)
73
+ else:
74
+ # TODO: Support pipeline parallelism.
75
+ assert False