@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aedba6d5ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Print warnings/errors for large swap space (#123)

Woosuk Kwon committed May 23, 2023 at 18:22 UTC aedba6d5ec4e1eaad10745f71970c10b601f9dc1
3 files changed +34
cacheflow/config.py
+31
@@ -3,6 +3,11 @@ from typing import Optional
3 import torch
4 from transformers import AutoConfig, PretrainedConfig
5
6 +from cacheflow.logger import init_logger
7 +from cacheflow.utils import get_cpu_memory
8 +
9 +logger = init_logger(__name__)
10 +
11 _GiB = 1 << 30
12
13
@@ -73,11 +78,37 @@ class CacheConfig:
78 self.block_size = block_size
79 self.gpu_memory_utilization = gpu_memory_utilization
80 self.swap_space_bytes = swap_space * _GiB
81 + self._verify_args()
82
83 # Will be set after profiling.
84 self.num_gpu_blocks = None
85 self.num_cpu_blocks = None
86
87 + def _verify_args(self) -> None:
88 + if self.gpu_memory_utilization > 1.0:
89 + raise ValueError(
90 + "GPU memory utilization must be less than 1.0. Got "
91 + f"{self.gpu_memory_utilization}.")
92 +
93 + def verify_with_parallel_config(
94 + self,
95 + parallel_config: "ParallelConfig",
96 + ) -> None:
97 + total_cpu_memory = get_cpu_memory()
98 + # FIXME(woosuk): Here, it is assumed that the GPUs in a tensor parallel
99 + # group are in the same node. However, the GPUs may span multiple nodes.
100 + num_gpus_per_node = parallel_config.tensor_parallel_size
101 + cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node
102 +
103 + msg = (
104 + f"{cpu_memory_usage / _GiB:.2f} GiB out of "
105 + f"the {total_cpu_memory / _GiB:.2f} GiB total CPU memory is "
106 + "allocated for the swap space.")
107 + if cpu_memory_usage > 0.7 * total_cpu_memory:
108 + raise ValueError("Too large swap space. " + msg)
109 + elif cpu_memory_usage > 0.4 * total_cpu_memory:
110 + logger.warn("Possibly too large swap space. " + msg)
111 +
112
113 class ParallelConfig:
114
cacheflow/server/llm_server.py
+1
@@ -84,6 +84,7 @@ class LLMServer:
84
85 def _verify_args(self) -> None:
86 self.model_config.verify_with_parallel_config(self.parallel_config)
87 + self.cache_config.verify_with_parallel_config(self.parallel_config)
88
89 def _init_cache(self) -> None:
90 # Get the maximum number of blocks that can be allocated on GPU and CPU.
cacheflow/utils.py
+2
@@ -24,8 +24,10 @@ class Counter:
24
25
26 def get_gpu_memory(gpu: int = 0) -> int:
27 + """Returns the total memory of the GPU in bytes."""
28 return torch.cuda.get_device_properties(gpu).total_memory
29
30
31 def get_cpu_memory() -> int:
32 + """Returns the total CPU memory of the node in bytes."""
33 return psutil.virtual_memory().total