@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8917782af6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add a system logger (#85)

Woosuk Kwon committed May 8, 2023 at 23:03 UTC 8917782af6a5892a0afb697badbe761dc8558619
5 files changed +85 -13
cacheflow/logger.py new
+51
@@ -0,0 +1,51 @@
1 +# Adapted from https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
2 +
3 +import logging
4 +import sys
5 +
6 +
7 +_FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s"
8 +_DATE_FORMAT = "%m-%d %H:%M:%S"
9 +
10 +
11 +class NewLineFormatter(logging.Formatter):
12 + """Adds logging prefix to newlines to align multi-line messages."""
13 +
14 + def __init__(self, fmt, datefmt=None):
15 + logging.Formatter.__init__(self, fmt, datefmt)
16 +
17 + def format(self, record):
18 + msg = logging.Formatter.format(self, record)
19 + if record.message != "":
20 + parts = msg.split(record.message)
21 + msg = msg.replace("\n", "\r\n" + parts[0])
22 + return msg
23 +
24 +
25 +_root_logger = logging.getLogger("cacheflow")
26 +_default_handler = None
27 +
28 +
29 +def _setup_logger():
30 + _root_logger.setLevel(logging.DEBUG)
31 + global _default_handler
32 + if _default_handler is None:
33 + _default_handler = logging.StreamHandler(sys.stdout)
34 + _default_handler.flush = sys.stdout.flush # type: ignore
35 + _default_handler.setLevel(logging.INFO)
36 + _root_logger.addHandler(_default_handler)
37 + fmt = NewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT)
38 + _default_handler.setFormatter(fmt)
39 + # Setting this will avoid the message
40 + # being propagated to the parent logger.
41 + _root_logger.propagate = False
42 +
43 +
44 +# The logger is initialized when the module is imported.
45 +# This is thread-safe as the module is only imported once,
46 +# guaranteed by the Python GIL.
47 +_setup_logger()
48 +
49 +
50 +def init_logger(name: str):
51 + return logging.getLogger(name)
cacheflow/master/server.py
+16 -3
@@ -8,6 +8,7 @@ try:
8 except ImportError:
9 ray = None
10
11 +from cacheflow.logger import init_logger
12 from cacheflow.master.scheduler import Scheduler
13 from cacheflow.master.simple_frontend import SimpleFrontend
14 from cacheflow.models import get_memory_analyzer
@@ -17,6 +18,9 @@ from cacheflow.sampling_params import SamplingParams
18 from cacheflow.utils import get_gpu_memory, get_cpu_memory
19
20
21 +logger = init_logger(__name__)
22 +
23 +
24 class Server:
25 def __init__(
26 self,
@@ -42,6 +46,17 @@ class Server:
46 collect_stats: bool = False,
47 do_memory_analysis: bool = False,
48 ):
49 + logger.info(
50 + "Initializing a server with config: "
51 + f"model={model!r}, "
52 + f"dtype={dtype}, "
53 + f"use_dummy_weights={use_dummy_weights}, "
54 + f"cache_dir={cache_dir}, "
55 + f"use_np_cache={use_np_cache}, "
56 + f"tensor_parallel_size={tensor_parallel_size}, "
57 + f"block_size={block_size}, "
58 + f"seed={seed})"
59 + )
60 self.num_nodes = num_nodes
61 self.num_devices_per_node = num_devices_per_node
62 self.world_size = pipeline_parallel_size * tensor_parallel_size
@@ -61,9 +76,7 @@ class Server:
76 self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
77 max_num_batched_tokens=max_num_batched_tokens)
78 self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
64 - swap_space=swap_space)
65 - print(f'# GPU blocks: {self.num_gpu_blocks}, '
66 - f'# CPU blocks: {self.num_cpu_blocks}')
79 + swap_space_gib=swap_space)
80
81 # Create a controller for each pipeline stage.
82 self.controllers: List[Controller] = []
cacheflow/master/simple_frontend.py
+6 -2
@@ -1,13 +1,17 @@
1 import time
2 -from typing import List, Optional, Set, Tuple
2 +from typing import List, Optional, Tuple
3
4 from transformers import AutoTokenizer
5
6 +from cacheflow.logger import init_logger
7 from cacheflow.sampling_params import SamplingParams
8 from cacheflow.sequence import Sequence, SequenceGroup
9 from cacheflow.utils import Counter
10
11
12 +logger = init_logger(__name__)
13 +
14 +
15 class SimpleFrontend:
16
17 def __init__(
@@ -66,4 +70,4 @@ class SimpleFrontend:
70 token_ids = seq.get_token_ids()
71 output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
72 output = output.strip()
69 - print(f'Seq {seq.seq_id}: {output!r}')
73 + logger.info(f"Seq {seq.seq_id}: {output!r}")
cacheflow/models/memory_analyzer.py
+11 -7
@@ -1,8 +1,12 @@
1 import torch
2 from transformers import AutoConfig
3
4 +from cacheflow.logger import init_logger
5 from cacheflow.models.utils import get_dtype_size
6
7 +
8 +logger = init_logger(__name__)
9 +
10 _GiB = 1 << 30
11
12
@@ -23,20 +27,20 @@ class CacheFlowMemoryAnalyzer:
27
28 def get_max_num_cpu_blocks(
29 self,
26 - swap_space: int,
30 + swap_space_gib: int,
31 ) -> int:
28 - swap_space = swap_space * _GiB
32 + swap_space = swap_space_gib * _GiB
33 cpu_memory = self.cpu_memory
34 if swap_space > 0.8 * cpu_memory:
31 - raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
35 + raise ValueError(f'The swap space ({swap_space_gib:.2f} GiB) '
36 'takes more than 80% of the available memory '
37 f'({cpu_memory / _GiB:.2f} GiB).'
38 'Please check the swap space size.')
39 if swap_space > 0.5 * cpu_memory:
36 - print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
37 - 'takes more than 50% of the available memory '
38 - f'({cpu_memory / _GiB:.2f} GiB).'
39 - 'This may slow the system performance.')
40 + logger.info(f'WARNING: The swap space ({swap_space_gib:.2f} GiB) '
41 + 'takes more than 50% of the available memory '
42 + f'({cpu_memory / _GiB:.2f} GiB).'
43 + 'This may slow the system performance.')
44 max_num_blocks = swap_space // self.get_cache_block_size()
45 return max_num_blocks
46
simple_server.py
+1 -1
@@ -1,11 +1,11 @@
1 import argparse
2 -from typing import List
2
3 from cacheflow.master.server import (
4 add_server_arguments, process_server_arguments,
5 init_local_server_and_frontend_with_arguments)
6 from cacheflow.sampling_params import SamplingParams
7
8 +
9 def main(args: argparse.Namespace):
10 server, frontend = init_local_server_and_frontend_with_arguments(args)
11 # Test the following inputs.