A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add an option to launch cacheflow without ray (#51)
Zhuohan Li committed
Apr 30, 2023 at 15:42 UTC
4858f3bb45ec62fab1fc32dc26eb1e2a8e1df14b
7 files changed
+102
-28
.gitignore
+3
@@ -3,8 +3,11 @@
3
*.egg-info/
4
*.eggs/
5
*.so
6
+*.log
7
+*.csv
8
build/
9
10
*.pkl
11
*.png
12
**/log.txt
13
+.vscode/
benchmark/benchmark_latency.py
+8
-4
@@ -8,7 +8,8 @@ import torch
8
9
from cacheflow.master.simple_frontend import SimpleFrontend
10
from cacheflow.master.server import (Server, add_server_arguments,
11
- initialize_ray_cluster)
11
+ process_server_arguments,
12
+ initialize_cluster)
13
from cacheflow.sampling_params import SamplingParams
14
from cacheflow.utils import get_gpu_memory, get_cpu_memory
15
@@ -20,8 +21,8 @@ def main(args: argparse.Namespace):
21
22
(num_nodes, num_devices_per_node, distributed_init_method,
23
all_stage_devices) = (
23
- initialize_ray_cluster(
24
- address='local',
24
+ initialize_cluster(
25
+ use_ray=args.use_ray,
26
pipeline_parallel_size=args.pipeline_parallel_size,
27
tensor_parallel_size=args.tensor_parallel_size))
28
@@ -44,6 +45,7 @@ def main(args: argparse.Namespace):
45
all_stage_devices=all_stage_devices,
46
gpu_memory=get_gpu_memory(),
47
cpu_memory=get_cpu_memory(),
48
+ use_ray=args.use_ray,
49
)
50
51
# Create a frontend.
@@ -91,7 +93,8 @@ def main(args: argparse.Namespace):
93
94
95
if __name__ == '__main__':
94
- parser = argparse.ArgumentParser(description='CacheFlow simple server.')
96
+ parser = argparse.ArgumentParser(
97
+ description='Benchmark the latency of decoding a single sentence.')
98
parser = add_server_arguments(parser)
99
parser.add_argument('--input-len', type=int, default=32)
100
parser.add_argument('--output-len', type=int, default=128)
@@ -99,6 +102,7 @@ if __name__ == '__main__':
102
parser.add_argument('--n', type=int, default=1)
103
parser.add_argument('--use-beam-search', action='store_true')
104
args = parser.parse_args()
105
+ args = process_server_arguments(args)
106
args.max_num_batched_tokens = max(
107
args.max_num_batched_tokens, args.batch_size * args.input_len)
108
print(args)
benchmark/benchmark_text_completion.py
+10
-6
@@ -11,7 +11,8 @@ from transformers import AutoConfig
11
from benchmark.trace import generate_text_completion_requests
12
from cacheflow.master.simple_frontend import SimpleFrontend
13
from cacheflow.master.server import (Server, add_server_arguments,
14
- initialize_ray_cluster)
14
+ process_server_arguments,
15
+ initialize_cluster)
16
from cacheflow.sampling_params import SamplingParams
17
from cacheflow.utils import get_gpu_memory, get_cpu_memory
18
@@ -25,8 +26,8 @@ def main(args: argparse.Namespace):
26
27
(num_nodes, num_devices_per_node, distributed_init_method,
28
all_stage_devices) = (
28
- initialize_ray_cluster(
29
- address='local',
29
+ initialize_cluster(
30
+ use_ray=args.use_ray,
31
pipeline_parallel_size=args.pipeline_parallel_size,
32
tensor_parallel_size=args.tensor_parallel_size))
33
@@ -49,6 +50,7 @@ def main(args: argparse.Namespace):
50
all_stage_devices=all_stage_devices,
51
gpu_memory=get_gpu_memory(),
52
cpu_memory=get_cpu_memory(),
53
+ use_ray=args.use_ray,
54
collect_stats=True,
55
do_memory_analysis=args.do_memory_analysis,
56
)
@@ -134,7 +136,7 @@ def main(args: argparse.Namespace):
136
finished.append({
137
'group_id': seq_group.group_id,
138
'seq_id': seq.seq_id,
137
- 'arrival_time': arrival_time,
139
+ 'arrival_time': arrival_time,
140
'finish_time': finish_time,
141
'prompt_len': seq.prompt_len,
142
'output_len': output_len,
@@ -225,8 +227,9 @@ def get_sampling_dir_name(
227
228
229
if __name__ == '__main__':
228
- parser = argparse.ArgumentParser(description='CacheFlow simple server.')
229
- parser = add_server_arguments(parser)
230
+ parser = argparse.ArgumentParser(
231
+ description='Benchmark the performance on a series of requests.')
232
+ parser = add_server_arguments(parser)
233
parser.add_argument('--output-dir', type=str, help='path to output directory', default=None)
234
235
parser.add_argument('--dataset', type=str, help='path to dataset', required=True)
@@ -246,6 +249,7 @@ if __name__ == '__main__':
249
parser.add_argument('--n6-beam', type=float, help='ratio of requests with n=6 & beam search', default=0.0)
250
parser.add_argument('--n8-beam', type=float, help='ratio of requests with n=8 & beam search', default=0.0)
251
args = parser.parse_args()
252
+ args = process_server_arguments(args)
253
if args.n1 + args.n2 + args.n3 + args.n4 + args.n6 + args.n2_beam + args.n4_beam + args.n6_beam + args.n8_beam != 1.0:
254
raise ValueError('The ratios of requests must sum to 1.')
255
cacheflow/http_frontend/fastapi_frontend.py
+15
-3
@@ -13,7 +13,8 @@ import uvicorn
13
from cacheflow.sampling_params import SamplingParams
14
from cacheflow.sequence import Sequence, SequenceGroup
15
from cacheflow.master.server import (Server, add_server_arguments,
16
- initialize_ray_cluster)
16
+ process_server_arguments,
17
+ initialize_cluster)
18
from cacheflow.worker.controller import DeviceID
19
from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
20
@@ -33,17 +34,22 @@ class FastAPIFrontend:
34
seed: int,
35
swap_space: int,
36
max_num_batched_tokens: int,
37
+ max_num_sequences: int,
38
num_nodes: int,
39
num_devices_per_node: int,
40
distributed_init_method: str,
41
all_stage_devices: List[List[DeviceID]],
42
+ server_use_ray: bool,
43
):
44
self.block_size = block_size
45
46
self.tokenizer = AutoTokenizer.from_pretrained(model)
47
self.seq_group_counter = Counter()
48
self.seq_counter = Counter()
46
- remote_server_class = ray.remote(num_cpus=0)(Server)
49
+ if server_use_ray:
50
+ remote_server_class = ray.remote(num_cpus=0)(Server)
51
+ else:
52
+ remote_server_class = ray.remote(num_gpus=1)(Server)
53
self.server = remote_server_class.remote(
54
model=model,
55
model_path=model_path,
@@ -55,12 +61,14 @@ class FastAPIFrontend:
61
seed=seed,
62
swap_space=swap_space,
63
max_num_batched_tokens=max_num_batched_tokens,
64
+ max_num_sequences=max_num_sequences,
65
num_nodes=num_nodes,
66
num_devices_per_node=num_devices_per_node,
67
distributed_init_method=distributed_init_method,
68
all_stage_devices=all_stage_devices,
69
gpu_memory=get_gpu_memory(),
70
cpu_memory=get_cpu_memory(),
71
+ use_ray=server_use_ray,
72
)
73
74
self.running_seq_groups: Dict[int, SequenceGroup] = {}
@@ -149,6 +157,7 @@ if __name__ == "__main__":
157
parser.add_argument("--port", type=int, default=10002)
158
parser = add_server_arguments(parser)
159
args = parser.parse_args()
160
+ args = process_server_arguments(args)
161
162
# TODO(zhuohan): Support pipeline parallelism.
163
assert args.pipeline_parallel_size == 1, (
@@ -156,7 +165,8 @@ if __name__ == "__main__":
165
166
(num_nodes, num_devices_per_node, distributed_init_method,
167
all_stage_devices) = (
159
- initialize_ray_cluster(
168
+ initialize_cluster(
169
+ use_ray=True,
170
pipeline_parallel_size=args.pipeline_parallel_size,
171
tensor_parallel_size=args.tensor_parallel_size))
172
@@ -170,10 +180,12 @@ if __name__ == "__main__":
180
seed=args.seed,
181
swap_space=args.swap_space,
182
max_num_batched_tokens=args.max_num_batched_tokens,
183
+ max_num_sequences=args.max_num_sequences,
184
num_nodes=num_nodes,
185
num_devices_per_node=num_devices_per_node,
186
distributed_init_method=distributed_init_method,
187
all_stage_devices=all_stage_devices,
188
+ server_use_ray=args.use_ray,
189
)
190
191
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/master/server.py
+39
-4
@@ -1,8 +1,12 @@
1
import argparse
2
-from typing import List, Tuple
2
+from typing import List, Tuple, Optional
3
import random
4
5
-import ray
5
+import torch
6
+try:
7
+ import ray
8
+except ImportError:
9
+ ray = None
10
11
from cacheflow.master.scheduler import Scheduler
12
from cacheflow.models import get_memory_analyzer
@@ -31,6 +35,7 @@ class Server:
35
all_stage_devices: List[List[DeviceID]],
36
gpu_memory: int,
37
cpu_memory: int,
38
+ use_ray: bool,
39
collect_stats: bool = False,
40
do_memory_analysis: bool = False,
41
):
@@ -38,6 +43,10 @@ class Server:
43
self.num_devices_per_node = num_devices_per_node
44
self.world_size = pipeline_parallel_size * tensor_parallel_size
45
46
+ if not use_ray:
47
+ assert self.world_size == 1, (
48
+ "Only support single GPU without Ray.")
49
+
50
self.memory_analyzer = get_memory_analyzer(
51
model_name=model,
52
block_size=block_size,
@@ -72,6 +81,7 @@ class Server:
81
model_path=model_path,
82
use_dummy_weights=use_dummy_weights,
83
max_num_batched_tokens=max_num_batched_tokens,
84
+ use_ray=use_ray,
85
)
86
self.controllers.append(controller)
87
@@ -105,11 +115,30 @@ class Server:
115
self.scheduler.swapped)
116
117
108
-def initialize_ray_cluster(
109
- address: str = 'auto',
118
+def initialize_cluster(
119
+ use_ray: bool = False,
120
+ address: Optional[str] = None,
121
pipeline_parallel_size: int = 1,
122
tensor_parallel_size: int = 1,
123
) -> Tuple[int, int, str, List[List[DeviceID]]]:
124
+ # Initialize cluster locally.
125
+ if not use_ray:
126
+ assert pipeline_parallel_size * tensor_parallel_size == 1, (
127
+ "Only support single GPU without Ray.")
128
+ num_nodes = 1
129
+ num_devices_per_node = torch.cuda.device_count()
130
+ port = random.randint(10000, 20000)
131
+ # We need to setup the distributed init method to make sure
132
+ # the distributed megatron code (e.g., get world size) works correctly.
133
+ distributed_init_method = f"tcp://localhost:{port}"
134
+ all_stage_devices = [[(0, None, 0)]]
135
+ return (num_nodes, num_devices_per_node, distributed_init_method,
136
+ all_stage_devices)
137
+
138
+ assert ray is not None, (
139
+ "Ray is not installed. Please install Ray to use distributed "
140
+ "serving.")
141
+
142
# Connect to a ray cluster.
143
ray.init(address=address)
144
@@ -177,6 +206,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
206
parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
207
help='model path to download and load the weights')
208
# Parallel arguments
209
+ parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
210
parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
211
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
212
# KV cache arguments
@@ -190,3 +220,8 @@ def add_server_arguments(parser: argparse.ArgumentParser):
220
parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
221
parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
222
return parser
223
+
224
+def process_server_arguments(args: argparse.Namespace):
225
+ if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
226
+ args.use_ray = True
227
+ return args
cacheflow/worker/controller.py
+21
-9
@@ -1,6 +1,9 @@
1
from typing import Dict, List, Union, Tuple
2
3
-import ray
3
+try:
4
+ import ray
5
+except ImportError:
6
+ ray = None
7
8
from cacheflow.master.scheduler import Scheduler
9
from cacheflow.sequence import SequenceGroupInputs
@@ -29,6 +32,7 @@ class Controller:
32
model_path: str,
33
use_dummy_weights: bool,
34
max_num_batched_tokens: int,
35
+ use_ray: bool,
36
) -> None:
37
self.stage_id = stage_id
38
self.stage_devices = stage_devices
@@ -36,6 +40,7 @@ class Controller:
40
self.block_size = block_size
41
self.num_gpu_blocks = num_gpu_blocks
42
self.num_cpu_blocks = num_cpu_blocks
43
+ self.use_ray = use_ray
44
45
# Which pipeline stage is this node assigned to?
46
self.is_first_stage = stage_id == 0
@@ -43,10 +48,13 @@ class Controller:
48
49
self.workers: List[Worker] = []
50
for rank, node_resource, device_id in stage_devices:
46
- worker_cls = ray.remote(num_cpus=0,
47
- num_gpus=1,
48
- resources={node_resource: 1e-5})(Worker)
49
- worker = worker_cls.remote(
51
+ if self.use_ray:
52
+ worker_cls = ray.remote(num_cpus=0,
53
+ num_gpus=1,
54
+ resources={node_resource: 1e-5})(Worker).remote
55
+ else:
56
+ worker_cls = Worker
57
+ worker = worker_cls(
58
model_name=model_name,
59
block_size=block_size,
60
num_gpu_blocks=num_gpu_blocks,
@@ -78,17 +86,21 @@ class Controller:
86
blocks_to_swap_out: Dict[int, int],
87
blocks_to_copy: Dict[int, List[int]],
88
) -> None:
81
- futures = []
89
+ all_outputs = []
90
for worker in self.workers:
83
- future = worker.execute_stage.remote(
91
+ executor = (worker.execute_stage.remote
92
+ if self.use_ray else worker.execute_stage)
93
+ output = executor(
94
input_seq_groups,
95
blocks_to_swap_in,
96
blocks_to_swap_out,
97
blocks_to_copy,
98
)
89
- futures.append(future)
99
+ all_outputs.append(output)
100
+
101
+ if self.use_ray:
102
+ all_outputs = ray.get(all_outputs)
103
91
- all_outputs = ray.get(futures)
104
# Make sure all workers have the same results.
105
output = all_outputs[0]
106
for other_output in all_outputs[1:]:
simple_server.py
+6
-2
@@ -3,7 +3,8 @@ from typing import List
3
4
from cacheflow.master.simple_frontend import SimpleFrontend
5
from cacheflow.master.server import (Server, add_server_arguments,
6
- initialize_ray_cluster)
6
+ process_server_arguments,
7
+ initialize_cluster)
8
from cacheflow.sampling_params import SamplingParams
9
from cacheflow.utils import get_gpu_memory, get_cpu_memory
10
@@ -14,7 +15,8 @@ def main(args: argparse.Namespace):
15
16
(num_nodes, num_devices_per_node, distributed_init_method,
17
all_stage_devices) = (
17
- initialize_ray_cluster(
18
+ initialize_cluster(
19
+ use_ray=args.use_ray,
20
pipeline_parallel_size=args.pipeline_parallel_size,
21
tensor_parallel_size=args.tensor_parallel_size))
22
@@ -37,6 +39,7 @@ def main(args: argparse.Namespace):
39
all_stage_devices=all_stage_devices,
40
gpu_memory=get_gpu_memory(),
41
cpu_memory=get_cpu_memory(),
42
+ use_ray=args.use_ray,
43
)
44
45
# Create a frontend.
@@ -70,4 +73,5 @@ if __name__ == '__main__':
73
parser = argparse.ArgumentParser(description='CacheFlow simple server.')
74
parser = add_server_arguments(parser)
75
args = parser.parse_args()
76
+ args = process_server_arguments(args)
77
main(args)