@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4858f3bb45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add an option to launch cacheflow without ray (#51)

Zhuohan Li committed Apr 30, 2023 at 15:42 UTC 4858f3bb45ec62fab1fc32dc26eb1e2a8e1df14b
7 files changed +102 -28
.gitignore
+3
@@ -3,8 +3,11 @@
3 *.egg-info/
4 *.eggs/
5 *.so
6 +*.log
7 +*.csv
8 build/
9
10 *.pkl
11 *.png
12 **/log.txt
13 +.vscode/
benchmark/benchmark_latency.py
+8 -4
@@ -8,7 +8,8 @@ import torch
8
9 from cacheflow.master.simple_frontend import SimpleFrontend
10 from cacheflow.master.server import (Server, add_server_arguments,
11 - initialize_ray_cluster)
11 + process_server_arguments,
12 + initialize_cluster)
13 from cacheflow.sampling_params import SamplingParams
14 from cacheflow.utils import get_gpu_memory, get_cpu_memory
15
@@ -20,8 +21,8 @@ def main(args: argparse.Namespace):
21
22 (num_nodes, num_devices_per_node, distributed_init_method,
23 all_stage_devices) = (
23 - initialize_ray_cluster(
24 - address='local',
24 + initialize_cluster(
25 + use_ray=args.use_ray,
26 pipeline_parallel_size=args.pipeline_parallel_size,
27 tensor_parallel_size=args.tensor_parallel_size))
28
@@ -44,6 +45,7 @@ def main(args: argparse.Namespace):
45 all_stage_devices=all_stage_devices,
46 gpu_memory=get_gpu_memory(),
47 cpu_memory=get_cpu_memory(),
48 + use_ray=args.use_ray,
49 )
50
51 # Create a frontend.
@@ -91,7 +93,8 @@ def main(args: argparse.Namespace):
93
94
95 if __name__ == '__main__':
94 - parser = argparse.ArgumentParser(description='CacheFlow simple server.')
96 + parser = argparse.ArgumentParser(
97 + description='Benchmark the latency of decoding a single sentence.')
98 parser = add_server_arguments(parser)
99 parser.add_argument('--input-len', type=int, default=32)
100 parser.add_argument('--output-len', type=int, default=128)
@@ -99,6 +102,7 @@ if __name__ == '__main__':
102 parser.add_argument('--n', type=int, default=1)
103 parser.add_argument('--use-beam-search', action='store_true')
104 args = parser.parse_args()
105 + args = process_server_arguments(args)
106 args.max_num_batched_tokens = max(
107 args.max_num_batched_tokens, args.batch_size * args.input_len)
108 print(args)
benchmark/benchmark_text_completion.py
+10 -6
@@ -11,7 +11,8 @@ from transformers import AutoConfig
11 from benchmark.trace import generate_text_completion_requests
12 from cacheflow.master.simple_frontend import SimpleFrontend
13 from cacheflow.master.server import (Server, add_server_arguments,
14 - initialize_ray_cluster)
14 + process_server_arguments,
15 + initialize_cluster)
16 from cacheflow.sampling_params import SamplingParams
17 from cacheflow.utils import get_gpu_memory, get_cpu_memory
18
@@ -25,8 +26,8 @@ def main(args: argparse.Namespace):
26
27 (num_nodes, num_devices_per_node, distributed_init_method,
28 all_stage_devices) = (
28 - initialize_ray_cluster(
29 - address='local',
29 + initialize_cluster(
30 + use_ray=args.use_ray,
31 pipeline_parallel_size=args.pipeline_parallel_size,
32 tensor_parallel_size=args.tensor_parallel_size))
33
@@ -49,6 +50,7 @@ def main(args: argparse.Namespace):
50 all_stage_devices=all_stage_devices,
51 gpu_memory=get_gpu_memory(),
52 cpu_memory=get_cpu_memory(),
53 + use_ray=args.use_ray,
54 collect_stats=True,
55 do_memory_analysis=args.do_memory_analysis,
56 )
@@ -134,7 +136,7 @@ def main(args: argparse.Namespace):
136 finished.append({
137 'group_id': seq_group.group_id,
138 'seq_id': seq.seq_id,
137 - 'arrival_time': arrival_time,
139 + 'arrival_time': arrival_time,
140 'finish_time': finish_time,
141 'prompt_len': seq.prompt_len,
142 'output_len': output_len,
@@ -225,8 +227,9 @@ def get_sampling_dir_name(
227
228
229 if __name__ == '__main__':
228 - parser = argparse.ArgumentParser(description='CacheFlow simple server.')
229 - parser = add_server_arguments(parser)
230 + parser = argparse.ArgumentParser(
231 + description='Benchmark the performance on a series of requests.')
232 + parser = add_server_arguments(parser)
233 parser.add_argument('--output-dir', type=str, help='path to output directory', default=None)
234
235 parser.add_argument('--dataset', type=str, help='path to dataset', required=True)
@@ -246,6 +249,7 @@ if __name__ == '__main__':
249 parser.add_argument('--n6-beam', type=float, help='ratio of requests with n=6 & beam search', default=0.0)
250 parser.add_argument('--n8-beam', type=float, help='ratio of requests with n=8 & beam search', default=0.0)
251 args = parser.parse_args()
252 + args = process_server_arguments(args)
253 if args.n1 + args.n2 + args.n3 + args.n4 + args.n6 + args.n2_beam + args.n4_beam + args.n6_beam + args.n8_beam != 1.0:
254 raise ValueError('The ratios of requests must sum to 1.')
255
cacheflow/http_frontend/fastapi_frontend.py
+15 -3
@@ -13,7 +13,8 @@ import uvicorn
13 from cacheflow.sampling_params import SamplingParams
14 from cacheflow.sequence import Sequence, SequenceGroup
15 from cacheflow.master.server import (Server, add_server_arguments,
16 - initialize_ray_cluster)
16 + process_server_arguments,
17 + initialize_cluster)
18 from cacheflow.worker.controller import DeviceID
19 from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
20
@@ -33,17 +34,22 @@ class FastAPIFrontend:
34 seed: int,
35 swap_space: int,
36 max_num_batched_tokens: int,
37 + max_num_sequences: int,
38 num_nodes: int,
39 num_devices_per_node: int,
40 distributed_init_method: str,
41 all_stage_devices: List[List[DeviceID]],
42 + server_use_ray: bool,
43 ):
44 self.block_size = block_size
45
46 self.tokenizer = AutoTokenizer.from_pretrained(model)
47 self.seq_group_counter = Counter()
48 self.seq_counter = Counter()
46 - remote_server_class = ray.remote(num_cpus=0)(Server)
49 + if server_use_ray:
50 + remote_server_class = ray.remote(num_cpus=0)(Server)
51 + else:
52 + remote_server_class = ray.remote(num_gpus=1)(Server)
53 self.server = remote_server_class.remote(
54 model=model,
55 model_path=model_path,
@@ -55,12 +61,14 @@ class FastAPIFrontend:
61 seed=seed,
62 swap_space=swap_space,
63 max_num_batched_tokens=max_num_batched_tokens,
64 + max_num_sequences=max_num_sequences,
65 num_nodes=num_nodes,
66 num_devices_per_node=num_devices_per_node,
67 distributed_init_method=distributed_init_method,
68 all_stage_devices=all_stage_devices,
69 gpu_memory=get_gpu_memory(),
70 cpu_memory=get_cpu_memory(),
71 + use_ray=server_use_ray,
72 )
73
74 self.running_seq_groups: Dict[int, SequenceGroup] = {}
@@ -149,6 +157,7 @@ if __name__ == "__main__":
157 parser.add_argument("--port", type=int, default=10002)
158 parser = add_server_arguments(parser)
159 args = parser.parse_args()
160 + args = process_server_arguments(args)
161
162 # TODO(zhuohan): Support pipeline parallelism.
163 assert args.pipeline_parallel_size == 1, (
@@ -156,7 +165,8 @@ if __name__ == "__main__":
165
166 (num_nodes, num_devices_per_node, distributed_init_method,
167 all_stage_devices) = (
159 - initialize_ray_cluster(
168 + initialize_cluster(
169 + use_ray=True,
170 pipeline_parallel_size=args.pipeline_parallel_size,
171 tensor_parallel_size=args.tensor_parallel_size))
172
@@ -170,10 +180,12 @@ if __name__ == "__main__":
180 seed=args.seed,
181 swap_space=args.swap_space,
182 max_num_batched_tokens=args.max_num_batched_tokens,
183 + max_num_sequences=args.max_num_sequences,
184 num_nodes=num_nodes,
185 num_devices_per_node=num_devices_per_node,
186 distributed_init_method=distributed_init_method,
187 all_stage_devices=all_stage_devices,
188 + server_use_ray=args.use_ray,
189 )
190
191 uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/master/server.py
+39 -4
@@ -1,8 +1,12 @@
1 import argparse
2 -from typing import List, Tuple
2 +from typing import List, Tuple, Optional
3 import random
4
5 -import ray
5 +import torch
6 +try:
7 + import ray
8 +except ImportError:
9 + ray = None
10
11 from cacheflow.master.scheduler import Scheduler
12 from cacheflow.models import get_memory_analyzer
@@ -31,6 +35,7 @@ class Server:
35 all_stage_devices: List[List[DeviceID]],
36 gpu_memory: int,
37 cpu_memory: int,
38 + use_ray: bool,
39 collect_stats: bool = False,
40 do_memory_analysis: bool = False,
41 ):
@@ -38,6 +43,10 @@ class Server:
43 self.num_devices_per_node = num_devices_per_node
44 self.world_size = pipeline_parallel_size * tensor_parallel_size
45
46 + if not use_ray:
47 + assert self.world_size == 1, (
48 + "Only support single GPU without Ray.")
49 +
50 self.memory_analyzer = get_memory_analyzer(
51 model_name=model,
52 block_size=block_size,
@@ -72,6 +81,7 @@ class Server:
81 model_path=model_path,
82 use_dummy_weights=use_dummy_weights,
83 max_num_batched_tokens=max_num_batched_tokens,
84 + use_ray=use_ray,
85 )
86 self.controllers.append(controller)
87
@@ -105,11 +115,30 @@ class Server:
115 self.scheduler.swapped)
116
117
108 -def initialize_ray_cluster(
109 - address: str = 'auto',
118 +def initialize_cluster(
119 + use_ray: bool = False,
120 + address: Optional[str] = None,
121 pipeline_parallel_size: int = 1,
122 tensor_parallel_size: int = 1,
123 ) -> Tuple[int, int, str, List[List[DeviceID]]]:
124 + # Initialize cluster locally.
125 + if not use_ray:
126 + assert pipeline_parallel_size * tensor_parallel_size == 1, (
127 + "Only support single GPU without Ray.")
128 + num_nodes = 1
129 + num_devices_per_node = torch.cuda.device_count()
130 + port = random.randint(10000, 20000)
131 + # We need to setup the distributed init method to make sure
132 + # the distributed megatron code (e.g., get world size) works correctly.
133 + distributed_init_method = f"tcp://localhost:{port}"
134 + all_stage_devices = [[(0, None, 0)]]
135 + return (num_nodes, num_devices_per_node, distributed_init_method,
136 + all_stage_devices)
137 +
138 + assert ray is not None, (
139 + "Ray is not installed. Please install Ray to use distributed "
140 + "serving.")
141 +
142 # Connect to a ray cluster.
143 ray.init(address=address)
144
@@ -177,6 +206,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
206 parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
207 help='model path to download and load the weights')
208 # Parallel arguments
209 + parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
210 parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
211 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
212 # KV cache arguments
@@ -190,3 +220,8 @@ def add_server_arguments(parser: argparse.ArgumentParser):
220 parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
221 parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
222 return parser
223 +
224 +def process_server_arguments(args: argparse.Namespace):
225 + if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
226 + args.use_ray = True
227 + return args
cacheflow/worker/controller.py
+21 -9
@@ -1,6 +1,9 @@
1 from typing import Dict, List, Union, Tuple
2
3 -import ray
3 +try:
4 + import ray
5 +except ImportError:
6 + ray = None
7
8 from cacheflow.master.scheduler import Scheduler
9 from cacheflow.sequence import SequenceGroupInputs
@@ -29,6 +32,7 @@ class Controller:
32 model_path: str,
33 use_dummy_weights: bool,
34 max_num_batched_tokens: int,
35 + use_ray: bool,
36 ) -> None:
37 self.stage_id = stage_id
38 self.stage_devices = stage_devices
@@ -36,6 +40,7 @@ class Controller:
40 self.block_size = block_size
41 self.num_gpu_blocks = num_gpu_blocks
42 self.num_cpu_blocks = num_cpu_blocks
43 + self.use_ray = use_ray
44
45 # Which pipeline stage is this node assigned to?
46 self.is_first_stage = stage_id == 0
@@ -43,10 +48,13 @@ class Controller:
48
49 self.workers: List[Worker] = []
50 for rank, node_resource, device_id in stage_devices:
46 - worker_cls = ray.remote(num_cpus=0,
47 - num_gpus=1,
48 - resources={node_resource: 1e-5})(Worker)
49 - worker = worker_cls.remote(
51 + if self.use_ray:
52 + worker_cls = ray.remote(num_cpus=0,
53 + num_gpus=1,
54 + resources={node_resource: 1e-5})(Worker).remote
55 + else:
56 + worker_cls = Worker
57 + worker = worker_cls(
58 model_name=model_name,
59 block_size=block_size,
60 num_gpu_blocks=num_gpu_blocks,
@@ -78,17 +86,21 @@ class Controller:
86 blocks_to_swap_out: Dict[int, int],
87 blocks_to_copy: Dict[int, List[int]],
88 ) -> None:
81 - futures = []
89 + all_outputs = []
90 for worker in self.workers:
83 - future = worker.execute_stage.remote(
91 + executor = (worker.execute_stage.remote
92 + if self.use_ray else worker.execute_stage)
93 + output = executor(
94 input_seq_groups,
95 blocks_to_swap_in,
96 blocks_to_swap_out,
97 blocks_to_copy,
98 )
89 - futures.append(future)
99 + all_outputs.append(output)
100 +
101 + if self.use_ray:
102 + all_outputs = ray.get(all_outputs)
103
91 - all_outputs = ray.get(futures)
104 # Make sure all workers have the same results.
105 output = all_outputs[0]
106 for other_output in all_outputs[1:]:
simple_server.py
+6 -2
@@ -3,7 +3,8 @@ from typing import List
3
4 from cacheflow.master.simple_frontend import SimpleFrontend
5 from cacheflow.master.server import (Server, add_server_arguments,
6 - initialize_ray_cluster)
6 + process_server_arguments,
7 + initialize_cluster)
8 from cacheflow.sampling_params import SamplingParams
9 from cacheflow.utils import get_gpu_memory, get_cpu_memory
10
@@ -14,7 +15,8 @@ def main(args: argparse.Namespace):
15
16 (num_nodes, num_devices_per_node, distributed_init_method,
17 all_stage_devices) = (
17 - initialize_ray_cluster(
18 + initialize_cluster(
19 + use_ray=args.use_ray,
20 pipeline_parallel_size=args.pipeline_parallel_size,
21 tensor_parallel_size=args.tensor_parallel_size))
22
@@ -37,6 +39,7 @@ def main(args: argparse.Namespace):
39 all_stage_devices=all_stage_devices,
40 gpu_memory=get_gpu_memory(),
41 cpu_memory=get_cpu_memory(),
42 + use_ray=args.use_ray,
43 )
44
45 # Create a frontend.
@@ -70,4 +73,5 @@ if __name__ == '__main__':
73 parser = argparse.ArgumentParser(description='CacheFlow simple server.')
74 parser = add_server_arguments(parser)
75 args = parser.parse_args()
76 + args = process_server_arguments(args)
77 main(args)