A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Log system stats (#90)
Woosuk Kwon committed
May 10, 2023 at 01:06 UTC
e331957784339ec1ca225b4752ea93fbdc34d851
3 files changed
+53
-136
cacheflow/core/scheduler.py
+48
-129
@@ -5,6 +5,7 @@ import time
5
from typing import Any, Dict, List, Optional, Tuple
6
7
from cacheflow.core.block_manager import BlockSpaceManager
8
+from cacheflow.logger import init_logger
9
from cacheflow.core.policy import PolicyFactory
10
from cacheflow.sampling_params import SamplingParams
11
from cacheflow.sequence import Sequence
@@ -14,6 +15,10 @@ from cacheflow.sequence import SequenceOutputs
15
from cacheflow.sequence import SequenceStatus
16
17
18
+logger = init_logger(__name__)
19
+_LOGGING_INTERVAL_SEC = 10
20
+
21
+
22
class PreemptionMode(enum.Enum):
23
"""Preemption modes.
24
@@ -37,8 +42,7 @@ class Scheduler:
42
num_cpu_blocks: int,
43
max_num_batched_tokens: int,
44
max_num_sequences: int,
40
- collect_stats: bool,
41
- do_memory_analysis: bool = False,
45
+ log_stats: bool,
46
) -> None:
47
self.controllers = controllers
48
self.block_size = block_size
@@ -46,8 +50,7 @@ class Scheduler:
50
self.num_cpu_blocks = num_cpu_blocks
51
self.max_num_batched_tokens = max_num_batched_tokens
52
self.max_num_sequences = max_num_sequences
49
- self.collect_stats = collect_stats
50
- self.do_memory_analysis = do_memory_analysis
53
+ self.log_stats = log_stats
54
55
# Instantiate the scheduling policy.
56
self.policy = PolicyFactory.get_policy(policy_name='fcfs')
@@ -69,8 +72,9 @@ class Scheduler:
72
# Sequence groups in the SWAPPED state.
73
self.swapped: List[SequenceGroup] = []
74
72
- # Performance-related statistics.
73
- self.stats = Stats(num_gpu_blocks, num_cpu_blocks)
75
+ self.last_logging_time: float = 0.0
76
+ # List[timestamp, num_tokens]
77
+ self.num_input_tokens: List[Tuple[float, int]] = []
78
79
def add_sequence_groups(
80
self,
@@ -186,59 +190,46 @@ class Scheduler:
190
num_batched_tokens += num_prompt_tokens
191
prompt_group_ids.append(seq_group.group_id)
192
189
- if self.collect_stats:
190
- if self.running or blocks_to_swap_in or blocks_to_swap_out:
191
- self.stats.timestamps.append(now - self.stats.start_time)
192
- self.stats.input_lens.append(num_batched_tokens)
193
- self.stats.swap_out_lens.append(len(blocks_to_swap_out) * self.block_size)
194
- self.stats.swap_in_lens.append(len(blocks_to_swap_in) * self.block_size)
195
- self.stats.num_preemption.append(len(preempted))
196
- self.stats.num_swapped.append(len(self.swapped))
197
- self.stats.num_running.append(len(self.running))
198
- self.stats.num_waiting.append(len(self.waiting))
199
-
200
- num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
201
- num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
202
- self.stats.gpu_cache_usage.append(num_used_gpu_blocks / self.num_gpu_blocks)
193
+ if not self.log_stats:
194
+ return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
195
+ prompt_group_ids)
196
+
197
+ now = time.time()
198
+ if num_batched_tokens > 0:
199
+ self.num_input_tokens.append((now, num_batched_tokens))
200
+ elapsed_time = now - self.last_logging_time
201
+ if elapsed_time > _LOGGING_INTERVAL_SEC:
202
+ self.last_logging_time = now
203
+ self.num_input_tokens = [
204
+ (t, n) for t, n in self.num_input_tokens
205
+ if now - t < _LOGGING_INTERVAL_SEC
206
+ ]
207
+ if len(self.num_input_tokens) > 1:
208
+ total_num_tokens = sum(n for _, n in self.num_input_tokens[:-1])
209
+ window = now - self.num_input_tokens[0][0]
210
+ avg_throughput = total_num_tokens / window
211
+ else:
212
+ avg_throughput = 0.0
213
+
214
+ num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
215
+ num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
216
+ gpu_cache_usage = num_used_gpu_blocks / self.num_gpu_blocks
217
+ if self.num_cpu_blocks > 0:
218
num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
219
num_used_cpu_blocks = self.num_cpu_blocks - num_free_cpu_blocks
205
- self.stats.cpu_cache_usage.append(num_used_cpu_blocks / self.num_cpu_blocks)
206
-
207
- if self.do_memory_analysis:
208
- block_tables = self.block_manager.block_tables
209
- num_logical_blocks = 0
210
- num_logical_tokens = 0
211
- num_physical_blocks = 0
212
- num_physical_tokens = 0
213
- physical_block_numbers = set()
214
- num_reserved_tokens = 0
215
- for seq_group in self.running:
216
- group_id = seq_group.group_id
217
- sampling_params = self.sampling_params[group_id]
218
- max_num_steps = sampling_params.max_num_steps
219
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
220
- num_logical_blocks += len(seq.logical_token_blocks)
221
- num_logical_tokens += seq.get_len()
222
-
223
- seq_id = seq.seq_id
224
- block_table = block_tables[seq_id]
225
- for i, block in enumerate(block_table):
226
- if block.block_number in physical_block_numbers:
227
- continue
228
- physical_block_numbers.add(block.block_number)
229
- num_physical_blocks += 1
230
- num_physical_tokens += seq.logical_token_blocks[i].num_tokens
231
-
232
- assert num_physical_blocks == num_used_gpu_blocks
233
- self.stats.num_logical_blocks.append(num_logical_blocks)
234
- self.stats.num_logical_tokens.append(num_logical_tokens)
235
- self.stats.num_physical_blocks.append(num_physical_blocks)
236
- self.stats.num_physical_tokens.append(num_physical_tokens)
237
- self.stats.num_reserved_tokens.append(num_reserved_tokens)
238
-
239
- return (blocks_to_swap_in,
240
- blocks_to_swap_out,
241
- blocks_to_copy,
220
+ cpu_cache_usage = num_used_cpu_blocks / self.num_cpu_blocks
221
+ else:
222
+ cpu_cache_usage = 0.0
223
+
224
+ logger.info(
225
+ f"Throughput: {avg_throughput:.1f} tokens/s, "
226
+ f"Running: {len(self.running)} reqs, "
227
+ f"Swapped: {len(self.swapped)} reqs, "
228
+ f"Pending: {len(self.waiting)} reqs, "
229
+ f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
230
+ f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
231
+
232
+ return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
233
prompt_group_ids)
234
235
def step(self) -> List[SequenceGroup]:
@@ -455,75 +446,3 @@ class Scheduler:
446
blocks_to_swap_out.update(mapping)
447
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
448
seq.status = SequenceStatus.SWAPPED
458
-
459
- def reset_stats(self) -> None:
460
- self.stats.reset(self.num_gpu_blocks, self.num_cpu_blocks)
461
-
462
- def save_stats(
463
- self,
464
- output_dir: str,
465
- ) -> None:
466
- assert self.collect_stats, 'Statistics collection is disabled.'
467
- self.stats.save(output_dir)
468
-
469
-
470
-class Stats:
471
-
472
- def __init__(
473
- self,
474
- num_gpu_blocks: int,
475
- num_cpu_blocks: int,
476
- ) -> None:
477
- self.start_time: float = time.time()
478
- self.num_gpu_blocks = num_gpu_blocks
479
- self.num_cpu_blocks = num_cpu_blocks
480
-
481
- self.timestamps: List[float] = []
482
- self.input_lens: List[int] = []
483
- self.swap_out_lens: List[int] = []
484
- self.swap_in_lens: List[int] = []
485
- self.num_preemption: List[int] = []
486
- self.num_waiting: List[int] = []
487
- self.num_running: List[int] = []
488
- self.num_swapped: List[int] = []
489
- self.gpu_cache_usage: List[float] = []
490
- self.cpu_cache_usage: List[float] = []
491
-
492
- self.num_logical_blocks: List[int] = []
493
- self.num_logical_tokens: List[int] = []
494
- self.num_physical_blocks: List[int] = []
495
- self.num_physical_tokens: List[int] = []
496
- self.num_reserved_tokens: List[int] = []
497
-
498
- def reset(
499
- self,
500
- num_gpu_blocks: int,
501
- num_cpu_blocks: int,
502
- ) -> None:
503
- self.__init__(num_gpu_blocks, num_cpu_blocks)
504
-
505
- def to_dict(self) -> Dict[str, Any]:
506
- return {
507
- 'start_time': self.start_time,
508
- 'num_gpu_blocks': self.num_gpu_blocks,
509
- 'num_cpu_blocks': self.num_cpu_blocks,
510
- 'timestamps': self.timestamps,
511
- 'input_lens': self.input_lens,
512
- 'swap_out_lens': self.swap_out_lens,
513
- 'swap_in_lens': self.swap_in_lens,
514
- 'num_preemption': self.num_preemption,
515
- 'num_waiting': self.num_waiting,
516
- 'num_running': self.num_running,
517
- 'num_swapped': self.num_swapped,
518
- 'gpu_cache_usage': self.gpu_cache_usage,
519
- 'cpu_cache_usage': self.cpu_cache_usage,
520
- 'num_logical_blocks': self.num_logical_blocks,
521
- 'num_logical_tokens': self.num_logical_tokens,
522
- 'num_physical_blocks': self.num_physical_blocks,
523
- 'num_physical_tokens': self.num_physical_tokens,
524
- 'num_reserved_tokens': self.num_reserved_tokens,
525
- }
526
-
527
- def save(self, output_dir: str) -> None:
528
- with open(os.path.join(output_dir, 'stats.pkl'), 'wb') as f:
529
- pickle.dump(self.to_dict(), f)
cacheflow/core/server.py
+5
-6
@@ -44,18 +44,16 @@ class Server:
44
gpu_memory: int,
45
cpu_memory: int,
46
use_ray: bool,
47
- collect_stats: bool = False,
48
- do_memory_analysis: bool = False,
47
+ log_stats: bool,
48
):
49
logger.info(
50
"Initializing a server with config: "
51
f"model={model!r}, "
52
f"dtype={dtype}, "
53
f"use_dummy_weights={use_dummy_weights}, "
55
- f"cache_dir={cache_dir}, "
54
+ f"cache_dir={cache_dir!r}, "
55
f"use_np_cache={use_np_cache}, "
56
f"tensor_parallel_size={tensor_parallel_size}, "
58
- f"block_size={block_size}, "
57
f"seed={seed})"
58
)
59
self.num_nodes = num_nodes
@@ -111,8 +109,7 @@ class Server:
109
num_cpu_blocks=self.num_cpu_blocks,
110
max_num_batched_tokens=max_num_batched_tokens,
111
max_num_sequences=max_num_sequences,
114
- collect_stats=collect_stats,
115
- do_memory_analysis=do_memory_analysis,
112
+ log_stats=log_stats,
113
)
114
# Connect the controllers.
115
for i in range(len(self.controllers) - 1):
@@ -244,6 +241,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
241
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
242
parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
243
parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
244
+ parser.add_argument('--log-stats', action='store_true', help='log system statistics')
245
return parser
246
247
@@ -286,6 +284,7 @@ def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
284
gpu_memory=get_gpu_memory(),
285
cpu_memory=get_cpu_memory(),
286
use_ray=args.use_ray,
287
+ log_stats=args.log_stats,
288
)
289
290
# Create a frontend.
cacheflow/worker/worker.py
-1
@@ -91,7 +91,6 @@ class Worker:
91
initialize_model_parallel(tensor_parallel_size,
92
pipeline_parallel_size)
93
94
-
94
def prepare_inputs(
95
self,
96
seq_group_metadata_list: List[SequenceGroupMetadata],