A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Print warnings/errors for large swap space (#123)
Woosuk Kwon committed
May 23, 2023 at 18:22 UTC
aedba6d5ec4e1eaad10745f71970c10b601f9dc1
3 files changed
+34
cacheflow/config.py
+31
@@ -3,6 +3,11 @@ from typing import Optional
3
import torch
4
from transformers import AutoConfig, PretrainedConfig
5
6
+from cacheflow.logger import init_logger
7
+from cacheflow.utils import get_cpu_memory
8
+
9
+logger = init_logger(__name__)
10
+
11
_GiB = 1 << 30
12
13
@@ -73,11 +78,37 @@ class CacheConfig:
78
self.block_size = block_size
79
self.gpu_memory_utilization = gpu_memory_utilization
80
self.swap_space_bytes = swap_space * _GiB
81
+ self._verify_args()
82
83
# Will be set after profiling.
84
self.num_gpu_blocks = None
85
self.num_cpu_blocks = None
86
87
+ def _verify_args(self) -> None:
88
+ if self.gpu_memory_utilization > 1.0:
89
+ raise ValueError(
90
+ "GPU memory utilization must be less than 1.0. Got "
91
+ f"{self.gpu_memory_utilization}.")
92
+
93
+ def verify_with_parallel_config(
94
+ self,
95
+ parallel_config: "ParallelConfig",
96
+ ) -> None:
97
+ total_cpu_memory = get_cpu_memory()
98
+ # FIXME(woosuk): Here, it is assumed that the GPUs in a tensor parallel
99
+ # group are in the same node. However, the GPUs may span multiple nodes.
100
+ num_gpus_per_node = parallel_config.tensor_parallel_size
101
+ cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node
102
+
103
+ msg = (
104
+ f"{cpu_memory_usage / _GiB:.2f} GiB out of "
105
+ f"the {total_cpu_memory / _GiB:.2f} GiB total CPU memory is "
106
+ "allocated for the swap space.")
107
+ if cpu_memory_usage > 0.7 * total_cpu_memory:
108
+ raise ValueError("Too large swap space. " + msg)
109
+ elif cpu_memory_usage > 0.4 * total_cpu_memory:
110
+ logger.warn("Possibly too large swap space. " + msg)
111
+
112
113
class ParallelConfig:
114
cacheflow/server/llm_server.py
+1
@@ -84,6 +84,7 @@ class LLMServer:
84
85
def _verify_args(self) -> None:
86
self.model_config.verify_with_parallel_config(self.parallel_config)
87
+ self.cache_config.verify_with_parallel_config(self.parallel_config)
88
89
def _init_cache(self) -> None:
90
# Get the maximum number of blocks that can be allocated on GPU and CPU.
cacheflow/utils.py
+2
@@ -24,8 +24,10 @@ class Counter:
24
25
26
def get_gpu_memory(gpu: int = 0) -> int:
27
+ """Returns the total memory of the GPU in bytes."""
28
return torch.cuda.get_device_properties(gpu).total_memory
29
30
31
def get_cpu_memory() -> int:
32
+ """Returns the total CPU memory of the node in bytes."""
33
return psutil.virtual_memory().total