A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Collect system stats in scheduler & Add scripts for experiments (#30)
Woosuk Kwon committed
Apr 12, 2023 at 15:03 UTC
84eee24e20ff4c0fc1b126289265f560089efa47
12 files changed
+830
-3
.gitignore
+4
@@ -4,3 +4,7 @@
4
*.eggs/
5
*.so
6
build/
7
+
8
+*.pkl
9
+*.png
10
+**/log.txt
benchmark/benchmark_latency.py
+1
@@ -37,6 +37,7 @@ def main(args: argparse.Namespace):
37
seed=args.seed,
38
swap_space=args.swap_space,
39
max_num_batched_tokens=args.max_num_batched_tokens,
40
+ max_num_sequences=args.max_num_sequences,
41
num_nodes=num_nodes,
42
num_devices_per_node=num_devices_per_node,
43
distributed_init_method=distributed_init_method,
benchmark/benchmark_text_completion.py
new
+289
@@ -0,0 +1,289 @@
1
+import argparse
2
+import logging
3
+import os
4
+import pickle
5
+import time
6
+from typing import List
7
+
8
+from tqdm import tqdm
9
+from transformers import AutoConfig
10
+
11
+from benchmark.trace import generate_text_completion_requests
12
+from cacheflow.master.simple_frontend import SimpleFrontend
13
+from cacheflow.master.server import (Server, add_server_arguments,
14
+ initialize_ray_cluster)
15
+from cacheflow.sampling_params import SamplingParams
16
+from cacheflow.utils import get_gpu_memory, get_cpu_memory
17
+
18
+
19
+logger = logging.getLogger(__name__)
20
+
21
+
22
+def main(args: argparse.Namespace):
23
+ assert args.pipeline_parallel_size == 1, (
24
+ 'Pipeline parallelism is not supported yet.')
25
+
26
+ (num_nodes, num_devices_per_node, distributed_init_method,
27
+ all_stage_devices) = (
28
+ initialize_ray_cluster(
29
+ address='local',
30
+ pipeline_parallel_size=args.pipeline_parallel_size,
31
+ tensor_parallel_size=args.tensor_parallel_size))
32
+
33
+ # Create a server.
34
+ server = Server(
35
+ model=args.model,
36
+ model_path=args.model_path,
37
+ use_dummy_weights=args.use_dummy_weights,
38
+ pipeline_parallel_size=args.pipeline_parallel_size,
39
+ tensor_parallel_size=args.tensor_parallel_size,
40
+ block_size=args.block_size,
41
+ dtype=args.dtype,
42
+ seed=args.seed,
43
+ swap_space=args.swap_space,
44
+ max_num_batched_tokens=args.max_num_batched_tokens,
45
+ max_num_sequences=args.max_num_sequences,
46
+ num_nodes=num_nodes,
47
+ num_devices_per_node=num_devices_per_node,
48
+ distributed_init_method=distributed_init_method,
49
+ all_stage_devices=all_stage_devices,
50
+ gpu_memory=get_gpu_memory(),
51
+ cpu_memory=get_cpu_memory(),
52
+ collect_stats=True,
53
+ do_memory_analysis=args.do_memory_analysis,
54
+ )
55
+
56
+ # Create a frontend.
57
+ frontend = SimpleFrontend(
58
+ model_name=args.model,
59
+ block_size=args.block_size,
60
+ )
61
+ # Generate requests.
62
+ requests = generate_text_completion_requests(
63
+ args.dataset,
64
+ args.request_rate,
65
+ args.duration,
66
+ args.seed,
67
+ args.n1,
68
+ args.n2,
69
+ args.n3,
70
+ args.n4,
71
+ args.n6,
72
+ args.n2_beam,
73
+ args.n4_beam,
74
+ args.n6_beam,
75
+ args.n8_beam,
76
+ )
77
+
78
+ # Warm up.
79
+ logger.info('Warming up.')
80
+ num_warmup_requests = 8
81
+ warmup_input_len = 8
82
+ warmup_output_len = 32
83
+ warmup_sampling_params = SamplingParams(
84
+ n=1,
85
+ temperature=1.0,
86
+ top_p=0.99,
87
+ max_num_steps=warmup_output_len,
88
+ use_beam_search=False,
89
+ stop_token_ids=set(),
90
+ num_logprobs=0,
91
+ context_window_size=None,
92
+ )
93
+ for _ in range(num_warmup_requests):
94
+ frontend._add_query([0] * warmup_input_len, warmup_sampling_params)
95
+ server.add_sequence_groups(frontend.get_inputs())
96
+ while True:
97
+ server.step()
98
+ if not server.has_unfinished_requests():
99
+ break
100
+
101
+ # Start benchmarking.
102
+ logger.info('Start benchmarking.')
103
+ # Initialize tqdm.
104
+ pbar = tqdm(total=len(requests), desc='Finished requests')
105
+
106
+ finished = []
107
+ server.scheduler.reset_stats()
108
+ start_time = time.time()
109
+ while True:
110
+ now = time.time()
111
+ if args.timeout is not None and now - start_time > args.timeout:
112
+ logger.info('Timeout. Stop benchmarking.')
113
+ break
114
+
115
+ while requests:
116
+ if requests[0][0] <= now - start_time:
117
+ request_time, input_tokens, sampling_params = requests.pop(0)
118
+ frontend._add_query(
119
+ input_tokens, sampling_params, arrival_time=start_time + request_time)
120
+ else:
121
+ break
122
+ server.add_sequence_groups(frontend.get_inputs())
123
+ updated_seq_groups = server.step()
124
+
125
+ now = time.time()
126
+ for seq_group in updated_seq_groups:
127
+ if not seq_group.is_finished():
128
+ continue
129
+ arrival_time = seq_group.arrival_time
130
+ finish_time = now
131
+ for seq in seq_group.get_seqs():
132
+ seq_len = seq.get_len()
133
+ output_len = seq_len - seq.prompt_len
134
+ finished.append({
135
+ 'group_id': seq_group.group_id,
136
+ 'seq_id': seq.seq_id,
137
+ 'arrival_time': arrival_time,
138
+ 'finish_time': finish_time,
139
+ 'prompt_len': seq.prompt_len,
140
+ 'output_len': output_len,
141
+ })
142
+ pbar.update(1)
143
+
144
+ if not (requests or server.has_unfinished_requests()):
145
+ break
146
+ pbar.close()
147
+ logger.info('Finish benchmarking. Saving stats.')
148
+ server.scheduler.save_stats(args.output_dir)
149
+ with open(os.path.join(args.output_dir, 'sequences.pkl'), 'wb') as f:
150
+ pickle.dump(finished, f)
151
+ logger.info('Done.')
152
+
153
+
154
+def get_model_name(model: str) -> str:
155
+ OPT_MODELS = [
156
+ 'opt-125m',
157
+ 'opt-350m',
158
+ 'opt-1.3b',
159
+ 'opt-2.7b',
160
+ 'opt-6.7b',
161
+ 'opt-13b',
162
+ 'opt-30b',
163
+ 'opt-66b',
164
+ 'opt-175b',
165
+ ]
166
+ for opt_model in OPT_MODELS:
167
+ if opt_model in model:
168
+ return opt_model
169
+
170
+ config = AutoConfig.from_pretrained(model)
171
+ assert config.model_type == 'llama'
172
+ hidden_size = config.hidden_size
173
+ if hidden_size == 4096:
174
+ return 'llama-7b'
175
+ elif hidden_size == 5120:
176
+ return 'llama-13b'
177
+ elif hidden_size == 6656:
178
+ return 'llama-30b'
179
+ elif hidden_size == 8192:
180
+ return 'llama-65b'
181
+ else:
182
+ raise ValueError(f'Unknown model: {model}')
183
+
184
+
185
+def get_dataset_name(dataset: str) -> str:
186
+ if 'sharegpt' in dataset.lower():
187
+ return 'sharegpt'
188
+ elif 'alpaca' in dataset.lower():
189
+ return 'alpaca'
190
+ else:
191
+ raise ValueError(f'Unknown dataset: {dataset}')
192
+
193
+
194
+def get_sampling_dir_name(
195
+ n1: float,
196
+ n2: float,
197
+ n3: float,
198
+ n4: float,
199
+ n6: float,
200
+ n2_beam: float,
201
+ n4_beam: float,
202
+ n6_beam: float,
203
+ n8_beam: float,
204
+) -> str:
205
+ method = ''
206
+ if n1 > 0.0:
207
+ method = 'n1' if n1 == 1.0 else method + f'n1-{n1}-'
208
+ if n2 > 0.0:
209
+ method = 'n2' if n2 == 1.0 else method + f'n2-{n2}-'
210
+ if n3 > 0.0:
211
+ method = 'n3' if n3 == 1.0 else method + f'n3-{n3}-'
212
+ if n4 > 0.0:
213
+ method = 'n4' if n4 == 1.0 else method + f'n4-{n4}-'
214
+ if n6 > 0.0:
215
+ method = 'n6' if n6 == 1.0 else method + f'n6-{n6}-'
216
+ if n2_beam > 0.0:
217
+ method = 'n2-beam' if n2_beam == 1.0 else method + f'n2-beam-{n2_beam}-'
218
+ if n4_beam > 0.0:
219
+ method = 'n4-beam' if n4_beam == 1.0 else method + f'n4-beam-{n4_beam}-'
220
+ if n6_beam > 0.0:
221
+ method = 'n6-beam' if n6_beam == 1.0 else method + f'n6-beam-{n6_beam}-'
222
+ if n8_beam > 0.0:
223
+ method = 'n8-beam' if n8_beam == 1.0 else method + f'n8-beam-{n8_beam}-'
224
+ return method[:-1] if method.endswith('-') else method
225
+
226
+
227
+if __name__ == '__main__':
228
+ parser = argparse.ArgumentParser(description='CacheFlow simple server.')
229
+ parser = add_server_arguments(parser)
230
+ parser.add_argument('--output-dir', type=str, help='path to output directory', default=None)
231
+
232
+ parser.add_argument('--dataset', type=str, help='path to dataset', required=True)
233
+ parser.add_argument('--request-rate', type=float, help='reqs/sec', required=True)
234
+ parser.add_argument('--duration', type=int, help='duration in seconds', required=True)
235
+ parser.add_argument('--do-memory-analysis', action='store_true',
236
+ help='do memory analysis (This will lower the throughput. Use this only for analysis.)')
237
+ parser.add_argument('--timeout', type=int, help='time out in seconds', default=None)
238
+
239
+ parser.add_argument('--n1', type=float, help='ratio of requests with n=1', default=0.0)
240
+ parser.add_argument('--n2', type=float, help='ratio of requests with n=2', default=0.0)
241
+ parser.add_argument('--n3', type=float, help='ratio of requests with n=3', default=0.0)
242
+ parser.add_argument('--n4', type=float, help='ratio of requests with n=4', default=0.0)
243
+ parser.add_argument('--n6', type=float, help='ratio of requests with n=6', default=0.0)
244
+ parser.add_argument('--n2-beam', type=float, help='ratio of requests with n=2 & beam search', default=0.0)
245
+ parser.add_argument('--n4-beam', type=float, help='ratio of requests with n=4 & beam search', default=0.0)
246
+ parser.add_argument('--n6-beam', type=float, help='ratio of requests with n=6 & beam search', default=0.0)
247
+ parser.add_argument('--n8-beam', type=float, help='ratio of requests with n=8 & beam search', default=0.0)
248
+ args = parser.parse_args()
249
+ if args.n1 + args.n2 + args.n3 + args.n4 + args.n6 + args.n2_beam + args.n4_beam + args.n6_beam + args.n8_beam != 1.0:
250
+ raise ValueError('The ratios of requests must sum to 1.')
251
+
252
+ model_name = get_model_name(args.model)
253
+ dataset_name = get_dataset_name(args.dataset)
254
+ if 'opt' in model_name:
255
+ if 'opt' not in args.dataset.lower():
256
+ raise ValueError(f'OPT models can only be used with OPT datasets.')
257
+ elif 'llama' in model_name:
258
+ if 'llama' not in args.dataset.lower():
259
+ raise ValueError(f'Llama models can only be used with Llama datasets.')
260
+
261
+ dataset_name = 'sharegpt' if 'sharegpt' in args.dataset else 'alpaca'
262
+ sample_dir = get_sampling_dir_name(
263
+ args.n1, args.n2, args.n3, args.n4, args.n6, args.n2_beam, args.n4_beam, args.n6_beam, args.n8_beam)
264
+ if args.output_dir is None:
265
+ args.output_dir = os.path.join(
266
+ '../exp',
267
+ dataset_name,
268
+ f'{model_name}-tp{args.tensor_parallel_size}',
269
+ sample_dir,
270
+ 'cacheflow',
271
+ f'req-rate-{args.request_rate}',
272
+ f'seed{args.seed}',
273
+ f'duration-{args.duration}',
274
+ )
275
+ os.makedirs(args.output_dir, exist_ok=True)
276
+
277
+ # Set up logging.
278
+ logging.basicConfig(
279
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
280
+ datefmt="%m/%d/%Y %H:%M:%S",
281
+ level=logging.INFO,
282
+ handlers=[
283
+ logging.StreamHandler(),
284
+ logging.FileHandler(os.path.join(args.output_dir, 'log.txt')),
285
+ ],
286
+ )
287
+ logger.info(args)
288
+
289
+ main(args)
benchmark/trace.py
new
+116
@@ -0,0 +1,116 @@
1
+import pickle
2
+import random
3
+from typing import List, Tuple
4
+
5
+import numpy as np
6
+
7
+from cacheflow.sampling_params import SamplingParams
8
+
9
+
10
+def generate_text_completion_requests(
11
+ dataset: str,
12
+ request_rate: float,
13
+ duration: int,
14
+ seed: int,
15
+ n1: float = 0.0,
16
+ n2: float = 0.0,
17
+ n3: float = 0.0,
18
+ n4: float = 0.0,
19
+ n6: float = 0.0,
20
+ n2_beam: float = 0.0,
21
+ n4_beam: float = 0.0,
22
+ n6_beam: float = 0.0,
23
+ n8_beam: float = 0.0,
24
+ max_seq_len: int = 2048,
25
+ time_quantum: int = 10,
26
+) -> List[Tuple[float, List[int], SamplingParams]]:
27
+ random.seed(seed)
28
+ np.random.seed(seed)
29
+
30
+ # Generate timestamps for requests using Poisson distribution.
31
+ lam = request_rate * (time_quantum / 1000)
32
+ quantums_per_sec = 1000 / time_quantum
33
+ arrival_times = np.random.poisson(
34
+ lam=lam, size=int(duration * quantums_per_sec))
35
+ timestamps = []
36
+ for i, n in enumerate(arrival_times):
37
+ timestamps += [i * (time_quantum / 1000)] * n
38
+
39
+ # Load and shuffle the dataset.
40
+ num_requests = len(timestamps)
41
+ with open(dataset, 'rb') as f:
42
+ data = pickle.load(f)
43
+
44
+ filtered = []
45
+ for pair in data:
46
+ input_tokens, output_tokens = pair
47
+ input_len = len(input_tokens)
48
+ output_len = len(output_tokens)
49
+ # Filter out too long sequences.
50
+ if input_len + output_len < max_seq_len:
51
+ # Output tokens are not needed for the benchmark.
52
+ filtered.append((input_tokens, output_len))
53
+
54
+ data = []
55
+ while len(data) < num_requests:
56
+ data += filtered
57
+ data = data[:num_requests]
58
+ # Shuffle the data.
59
+ assert len(data) == len(timestamps)
60
+ random.shuffle(data)
61
+
62
+ random_sampling_params_dict = {
63
+ 'temperature': 1.0,
64
+ 'top_p': 1.0,
65
+ 'use_beam_search': False,
66
+ 'stop_token_ids': set(),
67
+ 'num_logprobs': 0,
68
+ 'context_window_size': None,
69
+ }
70
+ beam_search_params_dict = {
71
+ 'temperature': 0.0,
72
+ 'top_p': 1.0,
73
+ 'use_beam_search': True,
74
+ 'stop_token_ids': set(),
75
+ 'num_logprobs': 0,
76
+ 'context_window_size': None,
77
+ }
78
+
79
+ # Generate requests based on the sampling parameter ratio.
80
+ requests = []
81
+ assert n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam == 1.0
82
+ cum_sum = 0
83
+ for timestamp, pair in zip(timestamps, data):
84
+ input_tokens, output_len = pair
85
+ if cum_sum < n1 * num_requests:
86
+ sampling_params = SamplingParams(
87
+ n=1, max_num_steps=output_len, **random_sampling_params_dict)
88
+ elif cum_sum < (n1 + n2) * num_requests:
89
+ sampling_params = SamplingParams(
90
+ n=2, max_num_steps=output_len, **random_sampling_params_dict)
91
+ elif cum_sum < (n1 + n2 + n3) * num_requests:
92
+ sampling_params = SamplingParams(
93
+ n=3, max_num_steps=output_len, **random_sampling_params_dict)
94
+ elif cum_sum < (n1 + n2 + n3 + n4) * num_requests:
95
+ sampling_params = SamplingParams(
96
+ n=4, max_num_steps=output_len, **random_sampling_params_dict)
97
+ elif cum_sum < (n1 + n2 + n3 + n4 + n6) * num_requests:
98
+ sampling_params = SamplingParams(
99
+ n=6, max_num_steps=output_len, **random_sampling_params_dict)
100
+ elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam) * num_requests:
101
+ sampling_params = SamplingParams(
102
+ n=2, max_num_steps=output_len, **beam_search_params_dict)
103
+ elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam) * num_requests:
104
+ sampling_params = SamplingParams(
105
+ n=4, max_num_steps=output_len, **beam_search_params_dict)
106
+ elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam) * num_requests:
107
+ sampling_params = SamplingParams(
108
+ n=6, max_num_steps=output_len, **beam_search_params_dict)
109
+ elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam) * num_requests:
110
+ sampling_params = SamplingParams(
111
+ n=8, max_num_steps=output_len, **beam_search_params_dict)
112
+ else:
113
+ raise ValueError('Invalid request ratio.')
114
+ cum_sum += 1
115
+ requests.append((timestamp, input_tokens, sampling_params))
116
+ return requests
cacheflow/master/block_manager.py
+6
@@ -241,3 +241,9 @@ class BlockSpaceManager:
241
def get_block_table(self, seq: Sequence) -> List[int]:
242
block_table = self.block_tables[seq.seq_id]
243
return [block.block_number for block in block_table]
244
+
245
+ def get_num_free_gpu_blocks(self) -> int:
246
+ return self.gpu_allocator.get_num_free_blocks()
247
+
248
+ def get_num_free_cpu_blocks(self) -> int:
249
+ return self.cpu_allocator.get_num_free_blocks()
cacheflow/master/scheduler.py
+146
-1
@@ -1,6 +1,8 @@
1
import enum
2
+import os
3
+import pickle
4
import time
3
-from typing import Dict, List, Optional, Tuple
5
+from typing import Any, Dict, List, Optional, Tuple
6
7
from cacheflow.master.block_manager import BlockSpaceManager
8
from cacheflow.master.policy import PolicyFactory
@@ -34,12 +36,18 @@ class Scheduler:
36
num_gpu_blocks: int,
37
num_cpu_blocks: int,
38
max_num_batched_tokens: int,
39
+ max_num_sequences: int,
40
+ collect_stats: bool,
41
+ do_memory_analysis: bool = False,
42
) -> None:
43
self.controllers = controllers
44
self.block_size = block_size
45
self.num_gpu_blocks = num_gpu_blocks
46
self.num_cpu_blocks = num_cpu_blocks
47
self.max_num_batched_tokens = max_num_batched_tokens
48
+ self.max_num_sequences = max_num_sequences
49
+ self.collect_stats = collect_stats
50
+ self.do_memory_analysis = do_memory_analysis
51
52
# Instantiate the scheduling policy.
53
self.policy = PolicyFactory.get_policy(policy_name='fcfs')
@@ -61,6 +69,9 @@ class Scheduler:
69
# Sequence groups in the SWAPPED state.
70
self.swapped: List[SequenceGroup] = []
71
72
+ # Performance-related statistics.
73
+ self.stats = Stats(num_gpu_blocks, num_cpu_blocks)
74
+
75
def add_sequence_groups(
76
self,
77
seq_groups: List[Tuple[SequenceGroup, SamplingParams]],
@@ -123,6 +134,12 @@ class Scheduler:
134
if not self.block_manager.can_swap_in(seq_group):
135
break
136
137
+ # The total number of sequences in the RUNNING state should not
138
+ # exceed the maximum number of sequences.
139
+ num_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
140
+ if len(self.running) + num_seqs > self.max_num_sequences:
141
+ break
142
+
143
seq_group = self.swapped.pop(0)
144
self._swap_in(seq_group, blocks_to_swap_in)
145
self._append(seq_group, blocks_to_copy)
@@ -156,12 +173,68 @@ class Scheduler:
173
> self.max_num_batched_tokens):
174
break
175
176
+ # The total number of sequences in the RUNNING state should not
177
+ # exceed the maximum number of sequences.
178
+ num_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
179
+ if len(self.running) + num_seqs > self.max_num_sequences:
180
+ break
181
+
182
seq_group = self.waiting.pop(0)
183
self._allocate(seq_group)
184
self.running.append(seq_group)
185
num_batched_tokens += num_prompt_tokens
186
prompt_group_ids.append(seq_group.group_id)
187
188
+ if self.collect_stats:
189
+ if self.running or blocks_to_swap_in or blocks_to_swap_out:
190
+ self.stats.timestamps.append(now - self.stats.start_time)
191
+ self.stats.input_lens.append(num_batched_tokens)
192
+ self.stats.swap_out_lens.append(len(blocks_to_swap_out) * self.block_size)
193
+ self.stats.swap_in_lens.append(len(blocks_to_swap_in) * self.block_size)
194
+ self.stats.num_preemption.append(len(preempted))
195
+ self.stats.num_swapped.append(len(self.swapped))
196
+ self.stats.num_running.append(len(self.running))
197
+ self.stats.num_waiting.append(len(self.waiting))
198
+
199
+ num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
200
+ num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
201
+ self.stats.gpu_cache_usage.append(num_used_gpu_blocks / self.num_gpu_blocks)
202
+ num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
203
+ num_used_cpu_blocks = self.num_cpu_blocks - num_free_cpu_blocks
204
+ self.stats.cpu_cache_usage.append(num_used_cpu_blocks / self.num_cpu_blocks)
205
+
206
+ if self.do_memory_analysis:
207
+ block_tables = self.block_manager.block_tables
208
+ num_logical_blocks = 0
209
+ num_logical_tokens = 0
210
+ num_physical_blocks = 0
211
+ num_physical_tokens = 0
212
+ physical_block_numbers = set()
213
+ num_reserved_tokens = 0
214
+ for seq_group in self.running:
215
+ group_id = seq_group.group_id
216
+ sampling_params = self.sampling_params[group_id]
217
+ max_num_steps = sampling_params.max_num_steps
218
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
219
+ num_logical_blocks += len(seq.logical_token_blocks)
220
+ num_logical_tokens += seq.get_len()
221
+
222
+ seq_id = seq.seq_id
223
+ block_table = block_tables[seq_id]
224
+ for i, block in enumerate(block_table):
225
+ if block.block_number in physical_block_numbers:
226
+ continue
227
+ physical_block_numbers.add(block.block_number)
228
+ num_physical_blocks += 1
229
+ num_physical_tokens += seq.logical_token_blocks[i].num_tokens
230
+
231
+ assert num_physical_blocks == num_used_gpu_blocks
232
+ self.stats.num_logical_blocks.append(num_logical_blocks)
233
+ self.stats.num_logical_tokens.append(num_logical_tokens)
234
+ self.stats.num_physical_blocks.append(num_physical_blocks)
235
+ self.stats.num_physical_tokens.append(num_physical_tokens)
236
+ self.stats.num_reserved_tokens.append(num_reserved_tokens)
237
+
238
return (blocks_to_swap_in,
239
blocks_to_swap_out,
240
blocks_to_copy,
@@ -381,3 +454,75 @@ class Scheduler:
454
blocks_to_swap_out.update(mapping)
455
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
456
seq.status = SequenceStatus.SWAPPED
457
+
458
+ def reset_stats(self) -> None:
459
+ self.stats.reset(self.num_gpu_blocks, self.num_cpu_blocks)
460
+
461
+ def save_stats(
462
+ self,
463
+ output_dir: str,
464
+ ) -> None:
465
+ assert self.collect_stats, 'Statistics collection is disabled.'
466
+ self.stats.save(output_dir)
467
+
468
+
469
+class Stats:
470
+
471
+ def __init__(
472
+ self,
473
+ num_gpu_blocks: int,
474
+ num_cpu_blocks: int,
475
+ ) -> None:
476
+ self.start_time: float = time.time()
477
+ self.num_gpu_blocks = num_gpu_blocks
478
+ self.num_cpu_blocks = num_cpu_blocks
479
+
480
+ self.timestamps: List[float] = []
481
+ self.input_lens: List[int] = []
482
+ self.swap_out_lens: List[int] = []
483
+ self.swap_in_lens: List[int] = []
484
+ self.num_preemption: List[int] = []
485
+ self.num_waiting: List[int] = []
486
+ self.num_running: List[int] = []
487
+ self.num_swapped: List[int] = []
488
+ self.gpu_cache_usage: List[float] = []
489
+ self.cpu_cache_usage: List[float] = []
490
+
491
+ self.num_logical_blocks: List[int] = []
492
+ self.num_logical_tokens: List[int] = []
493
+ self.num_physical_blocks: List[int] = []
494
+ self.num_physical_tokens: List[int] = []
495
+ self.num_reserved_tokens: List[int] = []
496
+
497
+ def reset(
498
+ self,
499
+ num_gpu_blocks: int,
500
+ num_cpu_blocks: int,
501
+ ) -> None:
502
+ self.__init__(num_gpu_blocks, num_cpu_blocks)
503
+
504
+ def to_dict(self) -> Dict[str, Any]:
505
+ return {
506
+ 'start_time': self.start_time,
507
+ 'num_gpu_blocks': self.num_gpu_blocks,
508
+ 'num_cpu_blocks': self.num_cpu_blocks,
509
+ 'timestamps': self.timestamps,
510
+ 'input_lens': self.input_lens,
511
+ 'swap_out_lens': self.swap_out_lens,
512
+ 'swap_in_lens': self.swap_in_lens,
513
+ 'num_preemption': self.num_preemption,
514
+ 'num_waiting': self.num_waiting,
515
+ 'num_running': self.num_running,
516
+ 'num_swapped': self.num_swapped,
517
+ 'gpu_cache_usage': self.gpu_cache_usage,
518
+ 'cpu_cache_usage': self.cpu_cache_usage,
519
+ 'num_logical_blocks': self.num_logical_blocks,
520
+ 'num_logical_tokens': self.num_logical_tokens,
521
+ 'num_physical_blocks': self.num_physical_blocks,
522
+ 'num_physical_tokens': self.num_physical_tokens,
523
+ 'num_reserved_tokens': self.num_reserved_tokens,
524
+ }
525
+
526
+ def save(self, output_dir: str) -> None:
527
+ with open(os.path.join(output_dir, 'stats.pkl'), 'wb') as f:
528
+ pickle.dump(self.to_dict(), f)
cacheflow/master/server.py
+8
-1
@@ -24,12 +24,15 @@ class Server:
24
seed: int,
25
swap_space: int,
26
max_num_batched_tokens: int,
27
+ max_num_sequences: int,
28
num_nodes: int,
29
num_devices_per_node: int,
30
distributed_init_method: str,
31
all_stage_devices: List[List[DeviceID]],
32
gpu_memory: int,
33
cpu_memory: int,
34
+ collect_stats: bool = False,
35
+ do_memory_analysis: bool = False,
36
):
37
self.num_nodes = num_nodes
38
self.num_devices_per_node = num_devices_per_node
@@ -79,6 +82,9 @@ class Server:
82
num_gpu_blocks=self.num_gpu_blocks,
83
num_cpu_blocks=self.num_cpu_blocks,
84
max_num_batched_tokens=max_num_batched_tokens,
85
+ max_num_sequences=max_num_sequences,
86
+ collect_stats=collect_stats,
87
+ do_memory_analysis=do_memory_analysis,
88
)
89
# Connect the controllers.
90
for i in range(len(self.controllers) - 1):
@@ -180,6 +186,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
186
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
187
parser.add_argument('--seed', type=int, default=0, help='random seed')
188
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
183
- parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens')
189
+ parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
190
+ parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
191
parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
192
return parser
cacheflow/master/simple_frontend.py
+3
-1
@@ -39,8 +39,10 @@ class SimpleFrontend:
39
self,
40
token_ids: List[int],
41
sampling_params: SamplingParams,
42
+ arrival_time: Optional[float] = None,
43
) -> None:
43
- arrival_time = time.time()
44
+ if arrival_time is None:
45
+ arrival_time = time.time()
46
seqs: List[Sequence] = []
47
for _ in range(sampling_params.n):
48
seq_id = next(self.seq_counter)
cacheflow/sequence.py
+1
@@ -28,6 +28,7 @@ class Sequence:
28
# Initialize the logical token blocks with the given token ids.
29
self.add(token_ids)
30
31
+ self.prompt_len = len(token_ids)
32
self.status = SequenceStatus.WAITING
33
self.output_logprobs: List[Dict[int, float]] = []
34
self.cumulative_logprobs = 0.0
plot/plot_normalized_latency.py
new
+203
@@ -0,0 +1,203 @@
1
+import argparse
2
+import os
3
+import pickle
4
+from typing import Any, Dict, List, Optional, Tuple
5
+
6
+import matplotlib.pyplot as plt
7
+import numpy as np
8
+
9
+
10
+SYSTEMS = [
11
+ 'orca-constant',
12
+ 'orca-power2',
13
+ 'orca-oracle',
14
+ 'cacheflow',
15
+]
16
+
17
+SYSTEM_TO_LABEL = {
18
+ 'orca-constant': 'Orca (Max)',
19
+ 'orca-power2': 'Orca (Next power of 2)',
20
+ 'orca-oracle': 'Orca (Oracle)',
21
+ 'cacheflow': 'CacheFlow',
22
+}
23
+
24
+SYSTEM_TO_COLOR = {
25
+ 'orca-constant': 'red',
26
+ 'orca-power2': 'orange',
27
+ 'orca-oracle': 'green',
28
+ 'cacheflow': 'blue',
29
+}
30
+
31
+SYSTEM_TO_MARKER = {
32
+ 'orca-constant': 'x',
33
+ 'orca-power2': '^',
34
+ 'orca-oracle': 's',
35
+ 'cacheflow': 'o',
36
+}
37
+
38
+
39
+def get_results(save_dir: str) -> List[Dict[str, Any]]:
40
+ with open(os.path.join(save_dir, 'sequences.pkl'), 'rb') as f:
41
+ results = pickle.load(f)
42
+ return results
43
+
44
+
45
+def get_request_rate(save_dir: str) -> float:
46
+ """Get request rate from save_dir name."""
47
+ # Directory name format:
48
+ # .../req-rate-{req_rate}/seed-{seed}/duration-{duration}
49
+ save_dir = os.path.abspath(save_dir)
50
+ dir_names = save_dir.split('/')
51
+
52
+ request_rate = None
53
+ for dir_name in dir_names:
54
+ if dir_name.startswith('req-rate-'):
55
+ if request_rate is not None:
56
+ raise ValueError(f'Found multiple request rates in {save_dir}')
57
+ request_rate = float(dir_name.split('-')[-1])
58
+ if request_rate is None:
59
+ raise ValueError(f'Cannot find request rate in {save_dir}')
60
+ return request_rate
61
+
62
+
63
+def get_model(save_dir: str) -> Tuple[str, int]:
64
+ save_dir = os.path.abspath(save_dir)
65
+ dir_names = save_dir.split('/')
66
+
67
+ model = None
68
+ for dir_name in dir_names:
69
+ if '-tp' in dir_name:
70
+ if model is not None:
71
+ raise ValueError(f'Found multiple models in {save_dir}')
72
+ model = dir_name.split('-tp')[0]
73
+ tp = int(dir_name.split('-tp')[-1])
74
+ if model is None:
75
+ raise ValueError(f'Cannot find model in {save_dir}')
76
+ return model, tp
77
+
78
+
79
+def get_system(save_dir: str) -> str:
80
+ save_dir = os.path.abspath(save_dir)
81
+ dir_names = save_dir.split('/')
82
+
83
+ for dir_name in dir_names:
84
+ if dir_name.startswith('orca-'):
85
+ return dir_name
86
+ if dir_name == 'cacheflow':
87
+ return dir_name
88
+ raise ValueError(f'Cannot find system in {save_dir}')
89
+
90
+
91
+def get_sampling(save_dir: str) -> str:
92
+ save_dir = os.path.abspath(save_dir)
93
+ dir_names = save_dir.split('/')
94
+
95
+ for dir_name in dir_names:
96
+ if dir_name.startswith('n'):
97
+ if dir_name.endswith('-beam'):
98
+ return dir_name
99
+ if dir_name[1:].isdigit():
100
+ return dir_name
101
+ raise ValueError(f'Cannot find sampling method in {save_dir}')
102
+
103
+
104
+def plot_normalized_latency(
105
+ exp_dir: str,
106
+ duration: int,
107
+ seed: int,
108
+ warmup: int,
109
+ xlim: Optional[float],
110
+ ylim: Optional[float],
111
+ log_scale: bool,
112
+ format: str,
113
+) -> None:
114
+ # Get leaf directories.
115
+ save_dirs = []
116
+ for root, dirs, files in os.walk(exp_dir):
117
+ if dirs:
118
+ continue
119
+ if 'sequences.pkl' not in files:
120
+ continue
121
+ if f'seed{seed}' not in root:
122
+ continue
123
+ if f'duration-{duration}' not in root:
124
+ continue
125
+ save_dirs.append(root)
126
+
127
+ # Plot normalized latency.
128
+ perf_per_system: Dict[str, Tuple[List[float], List[float]]] = {}
129
+ for save_dir in save_dirs:
130
+ per_seq_norm_latencies = []
131
+ results = get_results(save_dir)
132
+ for seq in results:
133
+ arrival_time = seq['arrival_time']
134
+ finish_time = seq['finish_time']
135
+ output_len = seq['output_len']
136
+ if arrival_time < warmup:
137
+ continue
138
+ latency = finish_time - arrival_time
139
+ norm_latency = latency / output_len
140
+ per_seq_norm_latencies.append(norm_latency)
141
+
142
+ request_rate = get_request_rate(save_dir)
143
+ normalized_latency = np.mean(per_seq_norm_latencies)
144
+ system_name = get_system(save_dir)
145
+ if system_name not in perf_per_system:
146
+ perf_per_system[system_name] = ([], [])
147
+ perf_per_system[system_name][0].append(request_rate)
148
+ perf_per_system[system_name][1].append(normalized_latency)
149
+
150
+ print('#seqs', len(per_seq_norm_latencies))
151
+ print(f'{save_dir}: {normalized_latency:.3f} s')
152
+
153
+
154
+ # Plot normalized latency.
155
+ plt.figure(figsize=(6, 4))
156
+ for system_name in reversed(SYSTEMS):
157
+ if system_name not in perf_per_system:
158
+ continue
159
+ # Sort by request rate.
160
+ request_rates, normalized_latencies = perf_per_system[system_name]
161
+ request_rates, normalized_latencies = zip(*sorted(zip(request_rates, normalized_latencies)))
162
+ label = SYSTEM_TO_LABEL[system_name]
163
+ color = SYSTEM_TO_COLOR[system_name]
164
+ marker = SYSTEM_TO_MARKER[system_name]
165
+ plt.plot(request_rates, normalized_latencies, label=label, color=color, marker=marker)
166
+
167
+ # plt.legend()
168
+ plt.xlabel('Request rate (req/s)', fontsize=12)
169
+ plt.ylabel('Normalized latency (s/token)', fontsize=12)
170
+
171
+ if log_scale:
172
+ plt.yscale('log')
173
+ if xlim is not None:
174
+ plt.xlim(left=0, right=xlim)
175
+ if ylim is not None:
176
+ if log_scale:
177
+ plt.ylim(top=ylim)
178
+ else:
179
+ plt.ylim(bottom=0, top=ylim)
180
+
181
+ # Save figure.
182
+ model, tp = get_model(exp_dir)
183
+ sampling = get_sampling(exp_dir)
184
+ figname = f'{model}-tp{tp}-{sampling}.{format}'
185
+ os.makedirs('./figures', exist_ok=True)
186
+ plt.savefig(os.path.join('figures', figname), bbox_inches='tight')
187
+ print(f'Saved figure to ./figures/{figname}')
188
+
189
+
190
+if __name__ == '__main__':
191
+ parser = argparse.ArgumentParser()
192
+ parser.add_argument('exp_dir', type=str)
193
+ parser.add_argument('--duration', type=int, required=True)
194
+ parser.add_argument('--seed', type=int, default=0)
195
+ parser.add_argument('--warmup', type=int, default=60)
196
+ parser.add_argument('--xlim', type=float, required=False, default=None)
197
+ parser.add_argument('--ylim', type=float, required=False, default=None)
198
+ parser.add_argument('--log', action='store_true')
199
+ parser.add_argument('--format', choices=['png', 'pdf'], default='png')
200
+ args = parser.parse_args()
201
+
202
+ plot_normalized_latency(
203
+ args.exp_dir, args.duration, args.seed, args.warmup, args.xlim, args.ylim, args.log, args.format)
plot/plot_stats.py
new
+52
@@ -0,0 +1,52 @@
1
+import os
2
+import pickle
3
+
4
+import matplotlib.pyplot as plt
5
+
6
+STAT_NAMES = [
7
+ 'input_lens',
8
+ 'num_running',
9
+ 'num_waiting',
10
+ 'num_preemption',
11
+ 'gpu_cache_usage',
12
+ 'cpu_cache_usage',
13
+ 'num_swapped',
14
+ 'swap_in_lens',
15
+ 'swap_out_lens',
16
+]
17
+
18
+
19
+def plot_stats(output_dir: str):
20
+ # Get stats.
21
+ with open(os.path.join(output_dir, 'stats.pkl'), 'rb') as f:
22
+ stats = pickle.load(f)
23
+ timestamps = stats['timestamps']
24
+
25
+ # Draw one figure for each stat.
26
+ num_stats = len(STAT_NAMES)
27
+ COLORS = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'orange', 'purple', 'pink', 'brown', 'gray']
28
+ fig, axs = plt.subplots(num_stats, 1, figsize=(10, 2 * num_stats))
29
+ for i, stat in enumerate(STAT_NAMES):
30
+ data = stats[stat]
31
+ if stat in ['gpu_cache_usage', 'cpu_cache_usage']:
32
+ data = [x * 100 for x in data]
33
+ stat = stat + ' (%)'
34
+ axs[i].plot(timestamps, data, color=COLORS[i % len(COLORS)])
35
+ axs[i].set_ylabel(stat.replace('_', ' '), fontdict={'fontsize': 12})
36
+ axs[i].set_ylim(bottom=0)
37
+
38
+ plt.xlabel('Time (s)')
39
+ plt.tight_layout()
40
+ fig_path = os.path.join(output_dir, 'stats.png')
41
+ plt.savefig(fig_path)
42
+ print(f'Saved stats to {fig_path}')
43
+
44
+
45
+if __name__ == '__main__':
46
+ import argparse
47
+
48
+ parser = argparse.ArgumentParser()
49
+ parser.add_argument('output_dir', type=str, help='Output directory.')
50
+ args = parser.parse_args()
51
+
52
+ plot_stats(args.output_dir)
simple_server.py
+1
@@ -30,6 +30,7 @@ def main(args: argparse.Namespace):
30
seed=args.seed,
31
swap_space=args.swap_space,
32
max_num_batched_tokens=args.max_num_batched_tokens,
33
+ max_num_sequences=args.max_num_sequences,
34
num_nodes=num_nodes,
35
num_devices_per_node=num_devices_per_node,
36
distributed_init_method=distributed_init_method,