A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add a system logger (#85)
Woosuk Kwon committed
May 8, 2023 at 23:03 UTC
8917782af6a5892a0afb697badbe761dc8558619
5 files changed
+85
-13
cacheflow/logger.py
new
+51
@@ -0,0 +1,51 @@
1
+# Adapted from https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
2
+
3
+import logging
4
+import sys
5
+
6
+
7
+_FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s"
8
+_DATE_FORMAT = "%m-%d %H:%M:%S"
9
+
10
+
11
+class NewLineFormatter(logging.Formatter):
12
+ """Adds logging prefix to newlines to align multi-line messages."""
13
+
14
+ def __init__(self, fmt, datefmt=None):
15
+ logging.Formatter.__init__(self, fmt, datefmt)
16
+
17
+ def format(self, record):
18
+ msg = logging.Formatter.format(self, record)
19
+ if record.message != "":
20
+ parts = msg.split(record.message)
21
+ msg = msg.replace("\n", "\r\n" + parts[0])
22
+ return msg
23
+
24
+
25
+_root_logger = logging.getLogger("cacheflow")
26
+_default_handler = None
27
+
28
+
29
+def _setup_logger():
30
+ _root_logger.setLevel(logging.DEBUG)
31
+ global _default_handler
32
+ if _default_handler is None:
33
+ _default_handler = logging.StreamHandler(sys.stdout)
34
+ _default_handler.flush = sys.stdout.flush # type: ignore
35
+ _default_handler.setLevel(logging.INFO)
36
+ _root_logger.addHandler(_default_handler)
37
+ fmt = NewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT)
38
+ _default_handler.setFormatter(fmt)
39
+ # Setting this will avoid the message
40
+ # being propagated to the parent logger.
41
+ _root_logger.propagate = False
42
+
43
+
44
+# The logger is initialized when the module is imported.
45
+# This is thread-safe as the module is only imported once,
46
+# guaranteed by the Python GIL.
47
+_setup_logger()
48
+
49
+
50
+def init_logger(name: str):
51
+ return logging.getLogger(name)
cacheflow/master/server.py
+16
-3
@@ -8,6 +8,7 @@ try:
8
except ImportError:
9
ray = None
10
11
+from cacheflow.logger import init_logger
12
from cacheflow.master.scheduler import Scheduler
13
from cacheflow.master.simple_frontend import SimpleFrontend
14
from cacheflow.models import get_memory_analyzer
@@ -17,6 +18,9 @@ from cacheflow.sampling_params import SamplingParams
18
from cacheflow.utils import get_gpu_memory, get_cpu_memory
19
20
21
+logger = init_logger(__name__)
22
+
23
+
24
class Server:
25
def __init__(
26
self,
@@ -42,6 +46,17 @@ class Server:
46
collect_stats: bool = False,
47
do_memory_analysis: bool = False,
48
):
49
+ logger.info(
50
+ "Initializing a server with config: "
51
+ f"model={model!r}, "
52
+ f"dtype={dtype}, "
53
+ f"use_dummy_weights={use_dummy_weights}, "
54
+ f"cache_dir={cache_dir}, "
55
+ f"use_np_cache={use_np_cache}, "
56
+ f"tensor_parallel_size={tensor_parallel_size}, "
57
+ f"block_size={block_size}, "
58
+ f"seed={seed})"
59
+ )
60
self.num_nodes = num_nodes
61
self.num_devices_per_node = num_devices_per_node
62
self.world_size = pipeline_parallel_size * tensor_parallel_size
@@ -61,9 +76,7 @@ class Server:
76
self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
77
max_num_batched_tokens=max_num_batched_tokens)
78
self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
64
- swap_space=swap_space)
65
- print(f'# GPU blocks: {self.num_gpu_blocks}, '
66
- f'# CPU blocks: {self.num_cpu_blocks}')
79
+ swap_space_gib=swap_space)
80
81
# Create a controller for each pipeline stage.
82
self.controllers: List[Controller] = []
cacheflow/master/simple_frontend.py
+6
-2
@@ -1,13 +1,17 @@
1
import time
2
-from typing import List, Optional, Set, Tuple
2
+from typing import List, Optional, Tuple
3
4
from transformers import AutoTokenizer
5
6
+from cacheflow.logger import init_logger
7
from cacheflow.sampling_params import SamplingParams
8
from cacheflow.sequence import Sequence, SequenceGroup
9
from cacheflow.utils import Counter
10
11
12
+logger = init_logger(__name__)
13
+
14
+
15
class SimpleFrontend:
16
17
def __init__(
@@ -66,4 +70,4 @@ class SimpleFrontend:
70
token_ids = seq.get_token_ids()
71
output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
72
output = output.strip()
69
- print(f'Seq {seq.seq_id}: {output!r}')
73
+ logger.info(f"Seq {seq.seq_id}: {output!r}")
cacheflow/models/memory_analyzer.py
+11
-7
@@ -1,8 +1,12 @@
1
import torch
2
from transformers import AutoConfig
3
4
+from cacheflow.logger import init_logger
5
from cacheflow.models.utils import get_dtype_size
6
7
+
8
+logger = init_logger(__name__)
9
+
10
_GiB = 1 << 30
11
12
@@ -23,20 +27,20 @@ class CacheFlowMemoryAnalyzer:
27
28
def get_max_num_cpu_blocks(
29
self,
26
- swap_space: int,
30
+ swap_space_gib: int,
31
) -> int:
28
- swap_space = swap_space * _GiB
32
+ swap_space = swap_space_gib * _GiB
33
cpu_memory = self.cpu_memory
34
if swap_space > 0.8 * cpu_memory:
31
- raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
35
+ raise ValueError(f'The swap space ({swap_space_gib:.2f} GiB) '
36
'takes more than 80% of the available memory '
37
f'({cpu_memory / _GiB:.2f} GiB).'
38
'Please check the swap space size.')
39
if swap_space > 0.5 * cpu_memory:
36
- print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
37
- 'takes more than 50% of the available memory '
38
- f'({cpu_memory / _GiB:.2f} GiB).'
39
- 'This may slow the system performance.')
40
+ logger.info(f'WARNING: The swap space ({swap_space_gib:.2f} GiB) '
41
+ 'takes more than 50% of the available memory '
42
+ f'({cpu_memory / _GiB:.2f} GiB).'
43
+ 'This may slow the system performance.')
44
max_num_blocks = swap_space // self.get_cache_block_size()
45
return max_num_blocks
46
simple_server.py
+1
-1
@@ -1,11 +1,11 @@
1
import argparse
2
-from typing import List
2
3
from cacheflow.master.server import (
4
add_server_arguments, process_server_arguments,
5
init_local_server_and_frontend_with_arguments)
6
from cacheflow.sampling_params import SamplingParams
7
8
+
9
def main(args: argparse.Namespace):
10
server, frontend = init_local_server_and_frontend_with_arguments(args)
11
# Test the following inputs.