@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 84eee24e20
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Collect system stats in scheduler & Add scripts for experiments (#30)

Woosuk Kwon committed Apr 12, 2023 at 15:03 UTC 84eee24e20ff4c0fc1b126289265f560089efa47
12 files changed +830 -3
.gitignore
+4
@@ -4,3 +4,7 @@
4 *.eggs/
5 *.so
6 build/
7 +
8 +*.pkl
9 +*.png
10 +**/log.txt
benchmark/benchmark_latency.py
+1
@@ -37,6 +37,7 @@ def main(args: argparse.Namespace):
37 seed=args.seed,
38 swap_space=args.swap_space,
39 max_num_batched_tokens=args.max_num_batched_tokens,
40 + max_num_sequences=args.max_num_sequences,
41 num_nodes=num_nodes,
42 num_devices_per_node=num_devices_per_node,
43 distributed_init_method=distributed_init_method,
benchmark/benchmark_text_completion.py new
+289
@@ -0,0 +1,289 @@
1 +import argparse
2 +import logging
3 +import os
4 +import pickle
5 +import time
6 +from typing import List
7 +
8 +from tqdm import tqdm
9 +from transformers import AutoConfig
10 +
11 +from benchmark.trace import generate_text_completion_requests
12 +from cacheflow.master.simple_frontend import SimpleFrontend
13 +from cacheflow.master.server import (Server, add_server_arguments,
14 + initialize_ray_cluster)
15 +from cacheflow.sampling_params import SamplingParams
16 +from cacheflow.utils import get_gpu_memory, get_cpu_memory
17 +
18 +
19 +logger = logging.getLogger(__name__)
20 +
21 +
22 +def main(args: argparse.Namespace):
23 + assert args.pipeline_parallel_size == 1, (
24 + 'Pipeline parallelism is not supported yet.')
25 +
26 + (num_nodes, num_devices_per_node, distributed_init_method,
27 + all_stage_devices) = (
28 + initialize_ray_cluster(
29 + address='local',
30 + pipeline_parallel_size=args.pipeline_parallel_size,
31 + tensor_parallel_size=args.tensor_parallel_size))
32 +
33 + # Create a server.
34 + server = Server(
35 + model=args.model,
36 + model_path=args.model_path,
37 + use_dummy_weights=args.use_dummy_weights,
38 + pipeline_parallel_size=args.pipeline_parallel_size,
39 + tensor_parallel_size=args.tensor_parallel_size,
40 + block_size=args.block_size,
41 + dtype=args.dtype,
42 + seed=args.seed,
43 + swap_space=args.swap_space,
44 + max_num_batched_tokens=args.max_num_batched_tokens,
45 + max_num_sequences=args.max_num_sequences,
46 + num_nodes=num_nodes,
47 + num_devices_per_node=num_devices_per_node,
48 + distributed_init_method=distributed_init_method,
49 + all_stage_devices=all_stage_devices,
50 + gpu_memory=get_gpu_memory(),
51 + cpu_memory=get_cpu_memory(),
52 + collect_stats=True,
53 + do_memory_analysis=args.do_memory_analysis,
54 + )
55 +
56 + # Create a frontend.
57 + frontend = SimpleFrontend(
58 + model_name=args.model,
59 + block_size=args.block_size,
60 + )
61 + # Generate requests.
62 + requests = generate_text_completion_requests(
63 + args.dataset,
64 + args.request_rate,
65 + args.duration,
66 + args.seed,
67 + args.n1,
68 + args.n2,
69 + args.n3,
70 + args.n4,
71 + args.n6,
72 + args.n2_beam,
73 + args.n4_beam,
74 + args.n6_beam,
75 + args.n8_beam,
76 + )
77 +
78 + # Warm up.
79 + logger.info('Warming up.')
80 + num_warmup_requests = 8
81 + warmup_input_len = 8
82 + warmup_output_len = 32
83 + warmup_sampling_params = SamplingParams(
84 + n=1,
85 + temperature=1.0,
86 + top_p=0.99,
87 + max_num_steps=warmup_output_len,
88 + use_beam_search=False,
89 + stop_token_ids=set(),
90 + num_logprobs=0,
91 + context_window_size=None,
92 + )
93 + for _ in range(num_warmup_requests):
94 + frontend._add_query([0] * warmup_input_len, warmup_sampling_params)
95 + server.add_sequence_groups(frontend.get_inputs())
96 + while True:
97 + server.step()
98 + if not server.has_unfinished_requests():
99 + break
100 +
101 + # Start benchmarking.
102 + logger.info('Start benchmarking.')
103 + # Initialize tqdm.
104 + pbar = tqdm(total=len(requests), desc='Finished requests')
105 +
106 + finished = []
107 + server.scheduler.reset_stats()
108 + start_time = time.time()
109 + while True:
110 + now = time.time()
111 + if args.timeout is not None and now - start_time > args.timeout:
112 + logger.info('Timeout. Stop benchmarking.')
113 + break
114 +
115 + while requests:
116 + if requests[0][0] <= now - start_time:
117 + request_time, input_tokens, sampling_params = requests.pop(0)
118 + frontend._add_query(
119 + input_tokens, sampling_params, arrival_time=start_time + request_time)
120 + else:
121 + break
122 + server.add_sequence_groups(frontend.get_inputs())
123 + updated_seq_groups = server.step()
124 +
125 + now = time.time()
126 + for seq_group in updated_seq_groups:
127 + if not seq_group.is_finished():
128 + continue
129 + arrival_time = seq_group.arrival_time
130 + finish_time = now
131 + for seq in seq_group.get_seqs():
132 + seq_len = seq.get_len()
133 + output_len = seq_len - seq.prompt_len
134 + finished.append({
135 + 'group_id': seq_group.group_id,
136 + 'seq_id': seq.seq_id,
137 + 'arrival_time': arrival_time,
138 + 'finish_time': finish_time,
139 + 'prompt_len': seq.prompt_len,
140 + 'output_len': output_len,
141 + })
142 + pbar.update(1)
143 +
144 + if not (requests or server.has_unfinished_requests()):
145 + break
146 + pbar.close()
147 + logger.info('Finish benchmarking. Saving stats.')
148 + server.scheduler.save_stats(args.output_dir)
149 + with open(os.path.join(args.output_dir, 'sequences.pkl'), 'wb') as f:
150 + pickle.dump(finished, f)
151 + logger.info('Done.')
152 +
153 +
154 +def get_model_name(model: str) -> str:
155 + OPT_MODELS = [
156 + 'opt-125m',
157 + 'opt-350m',
158 + 'opt-1.3b',
159 + 'opt-2.7b',
160 + 'opt-6.7b',
161 + 'opt-13b',
162 + 'opt-30b',
163 + 'opt-66b',
164 + 'opt-175b',
165 + ]
166 + for opt_model in OPT_MODELS:
167 + if opt_model in model:
168 + return opt_model
169 +
170 + config = AutoConfig.from_pretrained(model)
171 + assert config.model_type == 'llama'
172 + hidden_size = config.hidden_size
173 + if hidden_size == 4096:
174 + return 'llama-7b'
175 + elif hidden_size == 5120:
176 + return 'llama-13b'
177 + elif hidden_size == 6656:
178 + return 'llama-30b'
179 + elif hidden_size == 8192:
180 + return 'llama-65b'
181 + else:
182 + raise ValueError(f'Unknown model: {model}')
183 +
184 +
185 +def get_dataset_name(dataset: str) -> str:
186 + if 'sharegpt' in dataset.lower():
187 + return 'sharegpt'
188 + elif 'alpaca' in dataset.lower():
189 + return 'alpaca'
190 + else:
191 + raise ValueError(f'Unknown dataset: {dataset}')
192 +
193 +
194 +def get_sampling_dir_name(
195 + n1: float,
196 + n2: float,
197 + n3: float,
198 + n4: float,
199 + n6: float,
200 + n2_beam: float,
201 + n4_beam: float,
202 + n6_beam: float,
203 + n8_beam: float,
204 +) -> str:
205 + method = ''
206 + if n1 > 0.0:
207 + method = 'n1' if n1 == 1.0 else method + f'n1-{n1}-'
208 + if n2 > 0.0:
209 + method = 'n2' if n2 == 1.0 else method + f'n2-{n2}-'
210 + if n3 > 0.0:
211 + method = 'n3' if n3 == 1.0 else method + f'n3-{n3}-'
212 + if n4 > 0.0:
213 + method = 'n4' if n4 == 1.0 else method + f'n4-{n4}-'
214 + if n6 > 0.0:
215 + method = 'n6' if n6 == 1.0 else method + f'n6-{n6}-'
216 + if n2_beam > 0.0:
217 + method = 'n2-beam' if n2_beam == 1.0 else method + f'n2-beam-{n2_beam}-'
218 + if n4_beam > 0.0:
219 + method = 'n4-beam' if n4_beam == 1.0 else method + f'n4-beam-{n4_beam}-'
220 + if n6_beam > 0.0:
221 + method = 'n6-beam' if n6_beam == 1.0 else method + f'n6-beam-{n6_beam}-'
222 + if n8_beam > 0.0:
223 + method = 'n8-beam' if n8_beam == 1.0 else method + f'n8-beam-{n8_beam}-'
224 + return method[:-1] if method.endswith('-') else method
225 +
226 +
227 +if __name__ == '__main__':
228 + parser = argparse.ArgumentParser(description='CacheFlow simple server.')
229 + parser = add_server_arguments(parser)
230 + parser.add_argument('--output-dir', type=str, help='path to output directory', default=None)
231 +
232 + parser.add_argument('--dataset', type=str, help='path to dataset', required=True)
233 + parser.add_argument('--request-rate', type=float, help='reqs/sec', required=True)
234 + parser.add_argument('--duration', type=int, help='duration in seconds', required=True)
235 + parser.add_argument('--do-memory-analysis', action='store_true',
236 + help='do memory analysis (This will lower the throughput. Use this only for analysis.)')
237 + parser.add_argument('--timeout', type=int, help='time out in seconds', default=None)
238 +
239 + parser.add_argument('--n1', type=float, help='ratio of requests with n=1', default=0.0)
240 + parser.add_argument('--n2', type=float, help='ratio of requests with n=2', default=0.0)
241 + parser.add_argument('--n3', type=float, help='ratio of requests with n=3', default=0.0)
242 + parser.add_argument('--n4', type=float, help='ratio of requests with n=4', default=0.0)
243 + parser.add_argument('--n6', type=float, help='ratio of requests with n=6', default=0.0)
244 + parser.add_argument('--n2-beam', type=float, help='ratio of requests with n=2 & beam search', default=0.0)
245 + parser.add_argument('--n4-beam', type=float, help='ratio of requests with n=4 & beam search', default=0.0)
246 + parser.add_argument('--n6-beam', type=float, help='ratio of requests with n=6 & beam search', default=0.0)
247 + parser.add_argument('--n8-beam', type=float, help='ratio of requests with n=8 & beam search', default=0.0)
248 + args = parser.parse_args()
249 + if args.n1 + args.n2 + args.n3 + args.n4 + args.n6 + args.n2_beam + args.n4_beam + args.n6_beam + args.n8_beam != 1.0:
250 + raise ValueError('The ratios of requests must sum to 1.')
251 +
252 + model_name = get_model_name(args.model)
253 + dataset_name = get_dataset_name(args.dataset)
254 + if 'opt' in model_name:
255 + if 'opt' not in args.dataset.lower():
256 + raise ValueError(f'OPT models can only be used with OPT datasets.')
257 + elif 'llama' in model_name:
258 + if 'llama' not in args.dataset.lower():
259 + raise ValueError(f'Llama models can only be used with Llama datasets.')
260 +
261 + dataset_name = 'sharegpt' if 'sharegpt' in args.dataset else 'alpaca'
262 + sample_dir = get_sampling_dir_name(
263 + args.n1, args.n2, args.n3, args.n4, args.n6, args.n2_beam, args.n4_beam, args.n6_beam, args.n8_beam)
264 + if args.output_dir is None:
265 + args.output_dir = os.path.join(
266 + '../exp',
267 + dataset_name,
268 + f'{model_name}-tp{args.tensor_parallel_size}',
269 + sample_dir,
270 + 'cacheflow',
271 + f'req-rate-{args.request_rate}',
272 + f'seed{args.seed}',
273 + f'duration-{args.duration}',
274 + )
275 + os.makedirs(args.output_dir, exist_ok=True)
276 +
277 + # Set up logging.
278 + logging.basicConfig(
279 + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
280 + datefmt="%m/%d/%Y %H:%M:%S",
281 + level=logging.INFO,
282 + handlers=[
283 + logging.StreamHandler(),
284 + logging.FileHandler(os.path.join(args.output_dir, 'log.txt')),
285 + ],
286 + )
287 + logger.info(args)
288 +
289 + main(args)
benchmark/trace.py new
+116
@@ -0,0 +1,116 @@
1 +import pickle
2 +import random
3 +from typing import List, Tuple
4 +
5 +import numpy as np
6 +
7 +from cacheflow.sampling_params import SamplingParams
8 +
9 +
10 +def generate_text_completion_requests(
11 + dataset: str,
12 + request_rate: float,
13 + duration: int,
14 + seed: int,
15 + n1: float = 0.0,
16 + n2: float = 0.0,
17 + n3: float = 0.0,
18 + n4: float = 0.0,
19 + n6: float = 0.0,
20 + n2_beam: float = 0.0,
21 + n4_beam: float = 0.0,
22 + n6_beam: float = 0.0,
23 + n8_beam: float = 0.0,
24 + max_seq_len: int = 2048,
25 + time_quantum: int = 10,
26 +) -> List[Tuple[float, List[int], SamplingParams]]:
27 + random.seed(seed)
28 + np.random.seed(seed)
29 +
30 + # Generate timestamps for requests using Poisson distribution.
31 + lam = request_rate * (time_quantum / 1000)
32 + quantums_per_sec = 1000 / time_quantum
33 + arrival_times = np.random.poisson(
34 + lam=lam, size=int(duration * quantums_per_sec))
35 + timestamps = []
36 + for i, n in enumerate(arrival_times):
37 + timestamps += [i * (time_quantum / 1000)] * n
38 +
39 + # Load and shuffle the dataset.
40 + num_requests = len(timestamps)
41 + with open(dataset, 'rb') as f:
42 + data = pickle.load(f)
43 +
44 + filtered = []
45 + for pair in data:
46 + input_tokens, output_tokens = pair
47 + input_len = len(input_tokens)
48 + output_len = len(output_tokens)
49 + # Filter out too long sequences.
50 + if input_len + output_len < max_seq_len:
51 + # Output tokens are not needed for the benchmark.
52 + filtered.append((input_tokens, output_len))
53 +
54 + data = []
55 + while len(data) < num_requests:
56 + data += filtered
57 + data = data[:num_requests]
58 + # Shuffle the data.
59 + assert len(data) == len(timestamps)
60 + random.shuffle(data)
61 +
62 + random_sampling_params_dict = {
63 + 'temperature': 1.0,
64 + 'top_p': 1.0,
65 + 'use_beam_search': False,
66 + 'stop_token_ids': set(),
67 + 'num_logprobs': 0,
68 + 'context_window_size': None,
69 + }
70 + beam_search_params_dict = {
71 + 'temperature': 0.0,
72 + 'top_p': 1.0,
73 + 'use_beam_search': True,
74 + 'stop_token_ids': set(),
75 + 'num_logprobs': 0,
76 + 'context_window_size': None,
77 + }
78 +
79 + # Generate requests based on the sampling parameter ratio.
80 + requests = []
81 + assert n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam == 1.0
82 + cum_sum = 0
83 + for timestamp, pair in zip(timestamps, data):
84 + input_tokens, output_len = pair
85 + if cum_sum < n1 * num_requests:
86 + sampling_params = SamplingParams(
87 + n=1, max_num_steps=output_len, **random_sampling_params_dict)
88 + elif cum_sum < (n1 + n2) * num_requests:
89 + sampling_params = SamplingParams(
90 + n=2, max_num_steps=output_len, **random_sampling_params_dict)
91 + elif cum_sum < (n1 + n2 + n3) * num_requests:
92 + sampling_params = SamplingParams(
93 + n=3, max_num_steps=output_len, **random_sampling_params_dict)
94 + elif cum_sum < (n1 + n2 + n3 + n4) * num_requests:
95 + sampling_params = SamplingParams(
96 + n=4, max_num_steps=output_len, **random_sampling_params_dict)
97 + elif cum_sum < (n1 + n2 + n3 + n4 + n6) * num_requests:
98 + sampling_params = SamplingParams(
99 + n=6, max_num_steps=output_len, **random_sampling_params_dict)
100 + elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam) * num_requests:
101 + sampling_params = SamplingParams(
102 + n=2, max_num_steps=output_len, **beam_search_params_dict)
103 + elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam) * num_requests:
104 + sampling_params = SamplingParams(
105 + n=4, max_num_steps=output_len, **beam_search_params_dict)
106 + elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam) * num_requests:
107 + sampling_params = SamplingParams(
108 + n=6, max_num_steps=output_len, **beam_search_params_dict)
109 + elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam) * num_requests:
110 + sampling_params = SamplingParams(
111 + n=8, max_num_steps=output_len, **beam_search_params_dict)
112 + else:
113 + raise ValueError('Invalid request ratio.')
114 + cum_sum += 1
115 + requests.append((timestamp, input_tokens, sampling_params))
116 + return requests
cacheflow/master/block_manager.py
+6
@@ -241,3 +241,9 @@ class BlockSpaceManager:
241 def get_block_table(self, seq: Sequence) -> List[int]:
242 block_table = self.block_tables[seq.seq_id]
243 return [block.block_number for block in block_table]
244 +
245 + def get_num_free_gpu_blocks(self) -> int:
246 + return self.gpu_allocator.get_num_free_blocks()
247 +
248 + def get_num_free_cpu_blocks(self) -> int:
249 + return self.cpu_allocator.get_num_free_blocks()
cacheflow/master/scheduler.py
+146 -1
@@ -1,6 +1,8 @@
1 import enum
2 +import os
3 +import pickle
4 import time
3 -from typing import Dict, List, Optional, Tuple
5 +from typing import Any, Dict, List, Optional, Tuple
6
7 from cacheflow.master.block_manager import BlockSpaceManager
8 from cacheflow.master.policy import PolicyFactory
@@ -34,12 +36,18 @@ class Scheduler:
36 num_gpu_blocks: int,
37 num_cpu_blocks: int,
38 max_num_batched_tokens: int,
39 + max_num_sequences: int,
40 + collect_stats: bool,
41 + do_memory_analysis: bool = False,
42 ) -> None:
43 self.controllers = controllers
44 self.block_size = block_size
45 self.num_gpu_blocks = num_gpu_blocks
46 self.num_cpu_blocks = num_cpu_blocks
47 self.max_num_batched_tokens = max_num_batched_tokens
48 + self.max_num_sequences = max_num_sequences
49 + self.collect_stats = collect_stats
50 + self.do_memory_analysis = do_memory_analysis
51
52 # Instantiate the scheduling policy.
53 self.policy = PolicyFactory.get_policy(policy_name='fcfs')
@@ -61,6 +69,9 @@ class Scheduler:
69 # Sequence groups in the SWAPPED state.
70 self.swapped: List[SequenceGroup] = []
71
72 + # Performance-related statistics.
73 + self.stats = Stats(num_gpu_blocks, num_cpu_blocks)
74 +
75 def add_sequence_groups(
76 self,
77 seq_groups: List[Tuple[SequenceGroup, SamplingParams]],
@@ -123,6 +134,12 @@ class Scheduler:
134 if not self.block_manager.can_swap_in(seq_group):
135 break
136
137 + # The total number of sequences in the RUNNING state should not
138 + # exceed the maximum number of sequences.
139 + num_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
140 + if len(self.running) + num_seqs > self.max_num_sequences:
141 + break
142 +
143 seq_group = self.swapped.pop(0)
144 self._swap_in(seq_group, blocks_to_swap_in)
145 self._append(seq_group, blocks_to_copy)
@@ -156,12 +173,68 @@ class Scheduler:
173 > self.max_num_batched_tokens):
174 break
175
176 + # The total number of sequences in the RUNNING state should not
177 + # exceed the maximum number of sequences.
178 + num_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
179 + if len(self.running) + num_seqs > self.max_num_sequences:
180 + break
181 +
182 seq_group = self.waiting.pop(0)
183 self._allocate(seq_group)
184 self.running.append(seq_group)
185 num_batched_tokens += num_prompt_tokens
186 prompt_group_ids.append(seq_group.group_id)
187
188 + if self.collect_stats:
189 + if self.running or blocks_to_swap_in or blocks_to_swap_out:
190 + self.stats.timestamps.append(now - self.stats.start_time)
191 + self.stats.input_lens.append(num_batched_tokens)
192 + self.stats.swap_out_lens.append(len(blocks_to_swap_out) * self.block_size)
193 + self.stats.swap_in_lens.append(len(blocks_to_swap_in) * self.block_size)
194 + self.stats.num_preemption.append(len(preempted))
195 + self.stats.num_swapped.append(len(self.swapped))
196 + self.stats.num_running.append(len(self.running))
197 + self.stats.num_waiting.append(len(self.waiting))
198 +
199 + num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
200 + num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
201 + self.stats.gpu_cache_usage.append(num_used_gpu_blocks / self.num_gpu_blocks)
202 + num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
203 + num_used_cpu_blocks = self.num_cpu_blocks - num_free_cpu_blocks
204 + self.stats.cpu_cache_usage.append(num_used_cpu_blocks / self.num_cpu_blocks)
205 +
206 + if self.do_memory_analysis:
207 + block_tables = self.block_manager.block_tables
208 + num_logical_blocks = 0
209 + num_logical_tokens = 0
210 + num_physical_blocks = 0
211 + num_physical_tokens = 0
212 + physical_block_numbers = set()
213 + num_reserved_tokens = 0
214 + for seq_group in self.running:
215 + group_id = seq_group.group_id
216 + sampling_params = self.sampling_params[group_id]
217 + max_num_steps = sampling_params.max_num_steps
218 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
219 + num_logical_blocks += len(seq.logical_token_blocks)
220 + num_logical_tokens += seq.get_len()
221 +
222 + seq_id = seq.seq_id
223 + block_table = block_tables[seq_id]
224 + for i, block in enumerate(block_table):
225 + if block.block_number in physical_block_numbers:
226 + continue
227 + physical_block_numbers.add(block.block_number)
228 + num_physical_blocks += 1
229 + num_physical_tokens += seq.logical_token_blocks[i].num_tokens
230 +
231 + assert num_physical_blocks == num_used_gpu_blocks
232 + self.stats.num_logical_blocks.append(num_logical_blocks)
233 + self.stats.num_logical_tokens.append(num_logical_tokens)
234 + self.stats.num_physical_blocks.append(num_physical_blocks)
235 + self.stats.num_physical_tokens.append(num_physical_tokens)
236 + self.stats.num_reserved_tokens.append(num_reserved_tokens)
237 +
238 return (blocks_to_swap_in,
239 blocks_to_swap_out,
240 blocks_to_copy,
@@ -381,3 +454,75 @@ class Scheduler:
454 blocks_to_swap_out.update(mapping)
455 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
456 seq.status = SequenceStatus.SWAPPED
457 +
458 + def reset_stats(self) -> None:
459 + self.stats.reset(self.num_gpu_blocks, self.num_cpu_blocks)
460 +
461 + def save_stats(
462 + self,
463 + output_dir: str,
464 + ) -> None:
465 + assert self.collect_stats, 'Statistics collection is disabled.'
466 + self.stats.save(output_dir)
467 +
468 +
469 +class Stats:
470 +
471 + def __init__(
472 + self,
473 + num_gpu_blocks: int,
474 + num_cpu_blocks: int,
475 + ) -> None:
476 + self.start_time: float = time.time()
477 + self.num_gpu_blocks = num_gpu_blocks
478 + self.num_cpu_blocks = num_cpu_blocks
479 +
480 + self.timestamps: List[float] = []
481 + self.input_lens: List[int] = []
482 + self.swap_out_lens: List[int] = []
483 + self.swap_in_lens: List[int] = []
484 + self.num_preemption: List[int] = []
485 + self.num_waiting: List[int] = []
486 + self.num_running: List[int] = []
487 + self.num_swapped: List[int] = []
488 + self.gpu_cache_usage: List[float] = []
489 + self.cpu_cache_usage: List[float] = []
490 +
491 + self.num_logical_blocks: List[int] = []
492 + self.num_logical_tokens: List[int] = []
493 + self.num_physical_blocks: List[int] = []
494 + self.num_physical_tokens: List[int] = []
495 + self.num_reserved_tokens: List[int] = []
496 +
497 + def reset(
498 + self,
499 + num_gpu_blocks: int,
500 + num_cpu_blocks: int,
501 + ) -> None:
502 + self.__init__(num_gpu_blocks, num_cpu_blocks)
503 +
504 + def to_dict(self) -> Dict[str, Any]:
505 + return {
506 + 'start_time': self.start_time,
507 + 'num_gpu_blocks': self.num_gpu_blocks,
508 + 'num_cpu_blocks': self.num_cpu_blocks,
509 + 'timestamps': self.timestamps,
510 + 'input_lens': self.input_lens,
511 + 'swap_out_lens': self.swap_out_lens,
512 + 'swap_in_lens': self.swap_in_lens,
513 + 'num_preemption': self.num_preemption,
514 + 'num_waiting': self.num_waiting,
515 + 'num_running': self.num_running,
516 + 'num_swapped': self.num_swapped,
517 + 'gpu_cache_usage': self.gpu_cache_usage,
518 + 'cpu_cache_usage': self.cpu_cache_usage,
519 + 'num_logical_blocks': self.num_logical_blocks,
520 + 'num_logical_tokens': self.num_logical_tokens,
521 + 'num_physical_blocks': self.num_physical_blocks,
522 + 'num_physical_tokens': self.num_physical_tokens,
523 + 'num_reserved_tokens': self.num_reserved_tokens,
524 + }
525 +
526 + def save(self, output_dir: str) -> None:
527 + with open(os.path.join(output_dir, 'stats.pkl'), 'wb') as f:
528 + pickle.dump(self.to_dict(), f)
cacheflow/master/server.py
+8 -1
@@ -24,12 +24,15 @@ class Server:
24 seed: int,
25 swap_space: int,
26 max_num_batched_tokens: int,
27 + max_num_sequences: int,
28 num_nodes: int,
29 num_devices_per_node: int,
30 distributed_init_method: str,
31 all_stage_devices: List[List[DeviceID]],
32 gpu_memory: int,
33 cpu_memory: int,
34 + collect_stats: bool = False,
35 + do_memory_analysis: bool = False,
36 ):
37 self.num_nodes = num_nodes
38 self.num_devices_per_node = num_devices_per_node
@@ -79,6 +82,9 @@ class Server:
82 num_gpu_blocks=self.num_gpu_blocks,
83 num_cpu_blocks=self.num_cpu_blocks,
84 max_num_batched_tokens=max_num_batched_tokens,
85 + max_num_sequences=max_num_sequences,
86 + collect_stats=collect_stats,
87 + do_memory_analysis=do_memory_analysis,
88 )
89 # Connect the controllers.
90 for i in range(len(self.controllers) - 1):
@@ -180,6 +186,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
186 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
187 parser.add_argument('--seed', type=int, default=0, help='random seed')
188 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
183 - parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens')
189 + parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
190 + parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
191 parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
192 return parser
cacheflow/master/simple_frontend.py
+3 -1
@@ -39,8 +39,10 @@ class SimpleFrontend:
39 self,
40 token_ids: List[int],
41 sampling_params: SamplingParams,
42 + arrival_time: Optional[float] = None,
43 ) -> None:
43 - arrival_time = time.time()
44 + if arrival_time is None:
45 + arrival_time = time.time()
46 seqs: List[Sequence] = []
47 for _ in range(sampling_params.n):
48 seq_id = next(self.seq_counter)
cacheflow/sequence.py
+1
@@ -28,6 +28,7 @@ class Sequence:
28 # Initialize the logical token blocks with the given token ids.
29 self.add(token_ids)
30
31 + self.prompt_len = len(token_ids)
32 self.status = SequenceStatus.WAITING
33 self.output_logprobs: List[Dict[int, float]] = []
34 self.cumulative_logprobs = 0.0
plot/plot_normalized_latency.py new
+203
@@ -0,0 +1,203 @@
1 +import argparse
2 +import os
3 +import pickle
4 +from typing import Any, Dict, List, Optional, Tuple
5 +
6 +import matplotlib.pyplot as plt
7 +import numpy as np
8 +
9 +
10 +SYSTEMS = [
11 + 'orca-constant',
12 + 'orca-power2',
13 + 'orca-oracle',
14 + 'cacheflow',
15 +]
16 +
17 +SYSTEM_TO_LABEL = {
18 + 'orca-constant': 'Orca (Max)',
19 + 'orca-power2': 'Orca (Next power of 2)',
20 + 'orca-oracle': 'Orca (Oracle)',
21 + 'cacheflow': 'CacheFlow',
22 +}
23 +
24 +SYSTEM_TO_COLOR = {
25 + 'orca-constant': 'red',
26 + 'orca-power2': 'orange',
27 + 'orca-oracle': 'green',
28 + 'cacheflow': 'blue',
29 +}
30 +
31 +SYSTEM_TO_MARKER = {
32 + 'orca-constant': 'x',
33 + 'orca-power2': '^',
34 + 'orca-oracle': 's',
35 + 'cacheflow': 'o',
36 +}
37 +
38 +
39 +def get_results(save_dir: str) -> List[Dict[str, Any]]:
40 + with open(os.path.join(save_dir, 'sequences.pkl'), 'rb') as f:
41 + results = pickle.load(f)
42 + return results
43 +
44 +
45 +def get_request_rate(save_dir: str) -> float:
46 + """Get request rate from save_dir name."""
47 + # Directory name format:
48 + # .../req-rate-{req_rate}/seed-{seed}/duration-{duration}
49 + save_dir = os.path.abspath(save_dir)
50 + dir_names = save_dir.split('/')
51 +
52 + request_rate = None
53 + for dir_name in dir_names:
54 + if dir_name.startswith('req-rate-'):
55 + if request_rate is not None:
56 + raise ValueError(f'Found multiple request rates in {save_dir}')
57 + request_rate = float(dir_name.split('-')[-1])
58 + if request_rate is None:
59 + raise ValueError(f'Cannot find request rate in {save_dir}')
60 + return request_rate
61 +
62 +
63 +def get_model(save_dir: str) -> Tuple[str, int]:
64 + save_dir = os.path.abspath(save_dir)
65 + dir_names = save_dir.split('/')
66 +
67 + model = None
68 + for dir_name in dir_names:
69 + if '-tp' in dir_name:
70 + if model is not None:
71 + raise ValueError(f'Found multiple models in {save_dir}')
72 + model = dir_name.split('-tp')[0]
73 + tp = int(dir_name.split('-tp')[-1])
74 + if model is None:
75 + raise ValueError(f'Cannot find model in {save_dir}')
76 + return model, tp
77 +
78 +
79 +def get_system(save_dir: str) -> str:
80 + save_dir = os.path.abspath(save_dir)
81 + dir_names = save_dir.split('/')
82 +
83 + for dir_name in dir_names:
84 + if dir_name.startswith('orca-'):
85 + return dir_name
86 + if dir_name == 'cacheflow':
87 + return dir_name
88 + raise ValueError(f'Cannot find system in {save_dir}')
89 +
90 +
91 +def get_sampling(save_dir: str) -> str:
92 + save_dir = os.path.abspath(save_dir)
93 + dir_names = save_dir.split('/')
94 +
95 + for dir_name in dir_names:
96 + if dir_name.startswith('n'):
97 + if dir_name.endswith('-beam'):
98 + return dir_name
99 + if dir_name[1:].isdigit():
100 + return dir_name
101 + raise ValueError(f'Cannot find sampling method in {save_dir}')
102 +
103 +
104 +def plot_normalized_latency(
105 + exp_dir: str,
106 + duration: int,
107 + seed: int,
108 + warmup: int,
109 + xlim: Optional[float],
110 + ylim: Optional[float],
111 + log_scale: bool,
112 + format: str,
113 +) -> None:
114 + # Get leaf directories.
115 + save_dirs = []
116 + for root, dirs, files in os.walk(exp_dir):
117 + if dirs:
118 + continue
119 + if 'sequences.pkl' not in files:
120 + continue
121 + if f'seed{seed}' not in root:
122 + continue
123 + if f'duration-{duration}' not in root:
124 + continue
125 + save_dirs.append(root)
126 +
127 + # Plot normalized latency.
128 + perf_per_system: Dict[str, Tuple[List[float], List[float]]] = {}
129 + for save_dir in save_dirs:
130 + per_seq_norm_latencies = []
131 + results = get_results(save_dir)
132 + for seq in results:
133 + arrival_time = seq['arrival_time']
134 + finish_time = seq['finish_time']
135 + output_len = seq['output_len']
136 + if arrival_time < warmup:
137 + continue
138 + latency = finish_time - arrival_time
139 + norm_latency = latency / output_len
140 + per_seq_norm_latencies.append(norm_latency)
141 +
142 + request_rate = get_request_rate(save_dir)
143 + normalized_latency = np.mean(per_seq_norm_latencies)
144 + system_name = get_system(save_dir)
145 + if system_name not in perf_per_system:
146 + perf_per_system[system_name] = ([], [])
147 + perf_per_system[system_name][0].append(request_rate)
148 + perf_per_system[system_name][1].append(normalized_latency)
149 +
150 + print('#seqs', len(per_seq_norm_latencies))
151 + print(f'{save_dir}: {normalized_latency:.3f} s')
152 +
153 +
154 + # Plot normalized latency.
155 + plt.figure(figsize=(6, 4))
156 + for system_name in reversed(SYSTEMS):
157 + if system_name not in perf_per_system:
158 + continue
159 + # Sort by request rate.
160 + request_rates, normalized_latencies = perf_per_system[system_name]
161 + request_rates, normalized_latencies = zip(*sorted(zip(request_rates, normalized_latencies)))
162 + label = SYSTEM_TO_LABEL[system_name]
163 + color = SYSTEM_TO_COLOR[system_name]
164 + marker = SYSTEM_TO_MARKER[system_name]
165 + plt.plot(request_rates, normalized_latencies, label=label, color=color, marker=marker)
166 +
167 + # plt.legend()
168 + plt.xlabel('Request rate (req/s)', fontsize=12)
169 + plt.ylabel('Normalized latency (s/token)', fontsize=12)
170 +
171 + if log_scale:
172 + plt.yscale('log')
173 + if xlim is not None:
174 + plt.xlim(left=0, right=xlim)
175 + if ylim is not None:
176 + if log_scale:
177 + plt.ylim(top=ylim)
178 + else:
179 + plt.ylim(bottom=0, top=ylim)
180 +
181 + # Save figure.
182 + model, tp = get_model(exp_dir)
183 + sampling = get_sampling(exp_dir)
184 + figname = f'{model}-tp{tp}-{sampling}.{format}'
185 + os.makedirs('./figures', exist_ok=True)
186 + plt.savefig(os.path.join('figures', figname), bbox_inches='tight')
187 + print(f'Saved figure to ./figures/{figname}')
188 +
189 +
190 +if __name__ == '__main__':
191 + parser = argparse.ArgumentParser()
192 + parser.add_argument('exp_dir', type=str)
193 + parser.add_argument('--duration', type=int, required=True)
194 + parser.add_argument('--seed', type=int, default=0)
195 + parser.add_argument('--warmup', type=int, default=60)
196 + parser.add_argument('--xlim', type=float, required=False, default=None)
197 + parser.add_argument('--ylim', type=float, required=False, default=None)
198 + parser.add_argument('--log', action='store_true')
199 + parser.add_argument('--format', choices=['png', 'pdf'], default='png')
200 + args = parser.parse_args()
201 +
202 + plot_normalized_latency(
203 + args.exp_dir, args.duration, args.seed, args.warmup, args.xlim, args.ylim, args.log, args.format)
plot/plot_stats.py new
+52
@@ -0,0 +1,52 @@
1 +import os
2 +import pickle
3 +
4 +import matplotlib.pyplot as plt
5 +
6 +STAT_NAMES = [
7 + 'input_lens',
8 + 'num_running',
9 + 'num_waiting',
10 + 'num_preemption',
11 + 'gpu_cache_usage',
12 + 'cpu_cache_usage',
13 + 'num_swapped',
14 + 'swap_in_lens',
15 + 'swap_out_lens',
16 +]
17 +
18 +
19 +def plot_stats(output_dir: str):
20 + # Get stats.
21 + with open(os.path.join(output_dir, 'stats.pkl'), 'rb') as f:
22 + stats = pickle.load(f)
23 + timestamps = stats['timestamps']
24 +
25 + # Draw one figure for each stat.
26 + num_stats = len(STAT_NAMES)
27 + COLORS = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'orange', 'purple', 'pink', 'brown', 'gray']
28 + fig, axs = plt.subplots(num_stats, 1, figsize=(10, 2 * num_stats))
29 + for i, stat in enumerate(STAT_NAMES):
30 + data = stats[stat]
31 + if stat in ['gpu_cache_usage', 'cpu_cache_usage']:
32 + data = [x * 100 for x in data]
33 + stat = stat + ' (%)'
34 + axs[i].plot(timestamps, data, color=COLORS[i % len(COLORS)])
35 + axs[i].set_ylabel(stat.replace('_', ' '), fontdict={'fontsize': 12})
36 + axs[i].set_ylim(bottom=0)
37 +
38 + plt.xlabel('Time (s)')
39 + plt.tight_layout()
40 + fig_path = os.path.join(output_dir, 'stats.png')
41 + plt.savefig(fig_path)
42 + print(f'Saved stats to {fig_path}')
43 +
44 +
45 +if __name__ == '__main__':
46 + import argparse
47 +
48 + parser = argparse.ArgumentParser()
49 + parser.add_argument('output_dir', type=str, help='Output directory.')
50 + args = parser.parse_args()
51 +
52 + plot_stats(args.output_dir)
simple_server.py
+1
@@ -30,6 +30,7 @@ def main(args: argparse.Namespace):
30 seed=args.seed,
31 swap_space=args.swap_space,
32 max_num_batched_tokens=args.max_num_batched_tokens,
33 + max_num_sequences=args.max_num_sequences,
34 num_nodes=num_nodes,
35 num_devices_per_node=num_devices_per_node,
36 distributed_init_method=distributed_init_method,