@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f6c7ef437
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add controller

Woosuk Kwon committed Feb 23, 2023 at 09:32 UTC 1f6c7ef437b7b3ce5b83ca1b5a1de54183331bdd
1 file changed +75
cacheflow/worker/controller.py new
+75
@@ -0,0 +1,75 @@
1 +from typing import Dict, List, Union
2 +
3 +from cacheflow.master.scheduler import Scheduler
4 +from cacheflow.worker.worker import Worker
5 +
6 +
7 +class Controller:
8 +
9 + def __init__(
10 + self,
11 + node_id: int,
12 + num_workers: int,
13 + model_name: str,
14 + block_size: int,
15 + num_gpu_blocks: int,
16 + num_cpu_blocks: int,
17 + ) -> None:
18 + self.node_id = node_id
19 + self.num_workers = num_workers
20 + self.model_name = model_name
21 + self.block_size = block_size
22 + self.num_gpu_blocks = num_gpu_blocks
23 + self.num_cpu_blocks = num_cpu_blocks
24 +
25 + # Which pipeline stage is this node assigned to?
26 + self.is_first_stage = node_id == 0
27 + self.is_last_stage = False
28 +
29 + self.workers: List[Worker] = []
30 + for i in range(num_workers):
31 + worker = Worker(
32 + worker_id=node_id + i,
33 + gpu_id=i,
34 + model_name=model_name,
35 + block_size=block_size,
36 + num_gpu_blocks=num_gpu_blocks,
37 + num_cpu_blocks=num_cpu_blocks,
38 + )
39 + self.workers.append(worker)
40 +
41 + def set_next(
42 + self,
43 + next_node: Union['Controller', 'Scheduler'],
44 + ) -> None:
45 + self.next_node = next_node
46 + self.is_last_stage = isinstance(next_node, Scheduler)
47 +
48 + def execute_stage(
49 + self,
50 + prompt_tokens: Dict[int, List[int]],
51 + generation_tokens: Dict[int, int],
52 + context_lens: Dict[int, int],
53 + block_tables: Dict[int, List[int]],
54 + blocks_to_swap_in: Dict[int, int],
55 + blocks_to_swap_out: Dict[int, int],
56 + blocks_to_copy: Dict[int, int],
57 + ) -> None:
58 + # FIXME: Support tensor parallelism.
59 + assert len(self.workers) == 1
60 + worker = self.workers[0]
61 + output = worker.execute_stage(
62 + prompt_tokens,
63 + generation_tokens,
64 + context_lens,
65 + block_tables,
66 + blocks_to_swap_in,
67 + blocks_to_swap_out,
68 + blocks_to_copy,
69 + )
70 +
71 + if self.is_last_stage:
72 + self.next_node.post_step(output)
73 + else:
74 + # TODO: Support pipeline parallelism.
75 + assert False