@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e331957784
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Log system stats (#90)

Woosuk Kwon committed May 10, 2023 at 01:06 UTC e331957784339ec1ca225b4752ea93fbdc34d851
3 files changed +53 -136
cacheflow/core/scheduler.py
+48 -129
@@ -5,6 +5,7 @@ import time
5 from typing import Any, Dict, List, Optional, Tuple
6
7 from cacheflow.core.block_manager import BlockSpaceManager
8 +from cacheflow.logger import init_logger
9 from cacheflow.core.policy import PolicyFactory
10 from cacheflow.sampling_params import SamplingParams
11 from cacheflow.sequence import Sequence
@@ -14,6 +15,10 @@ from cacheflow.sequence import SequenceOutputs
15 from cacheflow.sequence import SequenceStatus
16
17
18 +logger = init_logger(__name__)
19 +_LOGGING_INTERVAL_SEC = 10
20 +
21 +
22 class PreemptionMode(enum.Enum):
23 """Preemption modes.
24
@@ -37,8 +42,7 @@ class Scheduler:
42 num_cpu_blocks: int,
43 max_num_batched_tokens: int,
44 max_num_sequences: int,
40 - collect_stats: bool,
41 - do_memory_analysis: bool = False,
45 + log_stats: bool,
46 ) -> None:
47 self.controllers = controllers
48 self.block_size = block_size
@@ -46,8 +50,7 @@ class Scheduler:
50 self.num_cpu_blocks = num_cpu_blocks
51 self.max_num_batched_tokens = max_num_batched_tokens
52 self.max_num_sequences = max_num_sequences
49 - self.collect_stats = collect_stats
50 - self.do_memory_analysis = do_memory_analysis
53 + self.log_stats = log_stats
54
55 # Instantiate the scheduling policy.
56 self.policy = PolicyFactory.get_policy(policy_name='fcfs')
@@ -69,8 +72,9 @@ class Scheduler:
72 # Sequence groups in the SWAPPED state.
73 self.swapped: List[SequenceGroup] = []
74
72 - # Performance-related statistics.
73 - self.stats = Stats(num_gpu_blocks, num_cpu_blocks)
75 + self.last_logging_time: float = 0.0
76 + # List[timestamp, num_tokens]
77 + self.num_input_tokens: List[Tuple[float, int]] = []
78
79 def add_sequence_groups(
80 self,
@@ -186,59 +190,46 @@ class Scheduler:
190 num_batched_tokens += num_prompt_tokens
191 prompt_group_ids.append(seq_group.group_id)
192
189 - if self.collect_stats:
190 - if self.running or blocks_to_swap_in or blocks_to_swap_out:
191 - self.stats.timestamps.append(now - self.stats.start_time)
192 - self.stats.input_lens.append(num_batched_tokens)
193 - self.stats.swap_out_lens.append(len(blocks_to_swap_out) * self.block_size)
194 - self.stats.swap_in_lens.append(len(blocks_to_swap_in) * self.block_size)
195 - self.stats.num_preemption.append(len(preempted))
196 - self.stats.num_swapped.append(len(self.swapped))
197 - self.stats.num_running.append(len(self.running))
198 - self.stats.num_waiting.append(len(self.waiting))
199 -
200 - num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
201 - num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
202 - self.stats.gpu_cache_usage.append(num_used_gpu_blocks / self.num_gpu_blocks)
193 + if not self.log_stats:
194 + return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
195 + prompt_group_ids)
196 +
197 + now = time.time()
198 + if num_batched_tokens > 0:
199 + self.num_input_tokens.append((now, num_batched_tokens))
200 + elapsed_time = now - self.last_logging_time
201 + if elapsed_time > _LOGGING_INTERVAL_SEC:
202 + self.last_logging_time = now
203 + self.num_input_tokens = [
204 + (t, n) for t, n in self.num_input_tokens
205 + if now - t < _LOGGING_INTERVAL_SEC
206 + ]
207 + if len(self.num_input_tokens) > 1:
208 + total_num_tokens = sum(n for _, n in self.num_input_tokens[:-1])
209 + window = now - self.num_input_tokens[0][0]
210 + avg_throughput = total_num_tokens / window
211 + else:
212 + avg_throughput = 0.0
213 +
214 + num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
215 + num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
216 + gpu_cache_usage = num_used_gpu_blocks / self.num_gpu_blocks
217 + if self.num_cpu_blocks > 0:
218 num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
219 num_used_cpu_blocks = self.num_cpu_blocks - num_free_cpu_blocks
205 - self.stats.cpu_cache_usage.append(num_used_cpu_blocks / self.num_cpu_blocks)
206 -
207 - if self.do_memory_analysis:
208 - block_tables = self.block_manager.block_tables
209 - num_logical_blocks = 0
210 - num_logical_tokens = 0
211 - num_physical_blocks = 0
212 - num_physical_tokens = 0
213 - physical_block_numbers = set()
214 - num_reserved_tokens = 0
215 - for seq_group in self.running:
216 - group_id = seq_group.group_id
217 - sampling_params = self.sampling_params[group_id]
218 - max_num_steps = sampling_params.max_num_steps
219 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
220 - num_logical_blocks += len(seq.logical_token_blocks)
221 - num_logical_tokens += seq.get_len()
222 -
223 - seq_id = seq.seq_id
224 - block_table = block_tables[seq_id]
225 - for i, block in enumerate(block_table):
226 - if block.block_number in physical_block_numbers:
227 - continue
228 - physical_block_numbers.add(block.block_number)
229 - num_physical_blocks += 1
230 - num_physical_tokens += seq.logical_token_blocks[i].num_tokens
231 -
232 - assert num_physical_blocks == num_used_gpu_blocks
233 - self.stats.num_logical_blocks.append(num_logical_blocks)
234 - self.stats.num_logical_tokens.append(num_logical_tokens)
235 - self.stats.num_physical_blocks.append(num_physical_blocks)
236 - self.stats.num_physical_tokens.append(num_physical_tokens)
237 - self.stats.num_reserved_tokens.append(num_reserved_tokens)
238 -
239 - return (blocks_to_swap_in,
240 - blocks_to_swap_out,
241 - blocks_to_copy,
220 + cpu_cache_usage = num_used_cpu_blocks / self.num_cpu_blocks
221 + else:
222 + cpu_cache_usage = 0.0
223 +
224 + logger.info(
225 + f"Throughput: {avg_throughput:.1f} tokens/s, "
226 + f"Running: {len(self.running)} reqs, "
227 + f"Swapped: {len(self.swapped)} reqs, "
228 + f"Pending: {len(self.waiting)} reqs, "
229 + f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
230 + f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
231 +
232 + return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
233 prompt_group_ids)
234
235 def step(self) -> List[SequenceGroup]:
@@ -455,75 +446,3 @@ class Scheduler:
446 blocks_to_swap_out.update(mapping)
447 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
448 seq.status = SequenceStatus.SWAPPED
458 -
459 - def reset_stats(self) -> None:
460 - self.stats.reset(self.num_gpu_blocks, self.num_cpu_blocks)
461 -
462 - def save_stats(
463 - self,
464 - output_dir: str,
465 - ) -> None:
466 - assert self.collect_stats, 'Statistics collection is disabled.'
467 - self.stats.save(output_dir)
468 -
469 -
470 -class Stats:
471 -
472 - def __init__(
473 - self,
474 - num_gpu_blocks: int,
475 - num_cpu_blocks: int,
476 - ) -> None:
477 - self.start_time: float = time.time()
478 - self.num_gpu_blocks = num_gpu_blocks
479 - self.num_cpu_blocks = num_cpu_blocks
480 -
481 - self.timestamps: List[float] = []
482 - self.input_lens: List[int] = []
483 - self.swap_out_lens: List[int] = []
484 - self.swap_in_lens: List[int] = []
485 - self.num_preemption: List[int] = []
486 - self.num_waiting: List[int] = []
487 - self.num_running: List[int] = []
488 - self.num_swapped: List[int] = []
489 - self.gpu_cache_usage: List[float] = []
490 - self.cpu_cache_usage: List[float] = []
491 -
492 - self.num_logical_blocks: List[int] = []
493 - self.num_logical_tokens: List[int] = []
494 - self.num_physical_blocks: List[int] = []
495 - self.num_physical_tokens: List[int] = []
496 - self.num_reserved_tokens: List[int] = []
497 -
498 - def reset(
499 - self,
500 - num_gpu_blocks: int,
501 - num_cpu_blocks: int,
502 - ) -> None:
503 - self.__init__(num_gpu_blocks, num_cpu_blocks)
504 -
505 - def to_dict(self) -> Dict[str, Any]:
506 - return {
507 - 'start_time': self.start_time,
508 - 'num_gpu_blocks': self.num_gpu_blocks,
509 - 'num_cpu_blocks': self.num_cpu_blocks,
510 - 'timestamps': self.timestamps,
511 - 'input_lens': self.input_lens,
512 - 'swap_out_lens': self.swap_out_lens,
513 - 'swap_in_lens': self.swap_in_lens,
514 - 'num_preemption': self.num_preemption,
515 - 'num_waiting': self.num_waiting,
516 - 'num_running': self.num_running,
517 - 'num_swapped': self.num_swapped,
518 - 'gpu_cache_usage': self.gpu_cache_usage,
519 - 'cpu_cache_usage': self.cpu_cache_usage,
520 - 'num_logical_blocks': self.num_logical_blocks,
521 - 'num_logical_tokens': self.num_logical_tokens,
522 - 'num_physical_blocks': self.num_physical_blocks,
523 - 'num_physical_tokens': self.num_physical_tokens,
524 - 'num_reserved_tokens': self.num_reserved_tokens,
525 - }
526 -
527 - def save(self, output_dir: str) -> None:
528 - with open(os.path.join(output_dir, 'stats.pkl'), 'wb') as f:
529 - pickle.dump(self.to_dict(), f)
cacheflow/core/server.py
+5 -6
@@ -44,18 +44,16 @@ class Server:
44 gpu_memory: int,
45 cpu_memory: int,
46 use_ray: bool,
47 - collect_stats: bool = False,
48 - do_memory_analysis: bool = False,
47 + log_stats: bool,
48 ):
49 logger.info(
50 "Initializing a server with config: "
51 f"model={model!r}, "
52 f"dtype={dtype}, "
53 f"use_dummy_weights={use_dummy_weights}, "
55 - f"cache_dir={cache_dir}, "
54 + f"cache_dir={cache_dir!r}, "
55 f"use_np_cache={use_np_cache}, "
56 f"tensor_parallel_size={tensor_parallel_size}, "
58 - f"block_size={block_size}, "
57 f"seed={seed})"
58 )
59 self.num_nodes = num_nodes
@@ -111,8 +109,7 @@ class Server:
109 num_cpu_blocks=self.num_cpu_blocks,
110 max_num_batched_tokens=max_num_batched_tokens,
111 max_num_sequences=max_num_sequences,
114 - collect_stats=collect_stats,
115 - do_memory_analysis=do_memory_analysis,
112 + log_stats=log_stats,
113 )
114 # Connect the controllers.
115 for i in range(len(self.controllers) - 1):
@@ -244,6 +241,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
241 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
242 parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
243 parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
244 + parser.add_argument('--log-stats', action='store_true', help='log system statistics')
245 return parser
246
247
@@ -286,6 +284,7 @@ def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
284 gpu_memory=get_gpu_memory(),
285 cpu_memory=get_cpu_memory(),
286 use_ray=args.use_ray,
287 + log_stats=args.log_stats,
288 )
289
290 # Create a frontend.
cacheflow/worker/worker.py
-1
@@ -91,7 +91,6 @@ class Worker:
91 initialize_model_parallel(tensor_parallel_size,
92 pipeline_parallel_size)
93
94 -
94 def prepare_inputs(
95 self,
96 seq_group_metadata_list: List[SequenceGroupMetadata],