A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Consolidate `GB` constant and enable float GB arguments (#7416)
Cyrus Leung committed
Aug 13, 2024 at 05:14 UTC
4ddc4743d7ae93fca090d8485b31321abe3743e7
6 files changed
+21
-21
vllm/config.py
+6
-7
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
13
from vllm.tracing import is_otel_installed
14
from vllm.transformers_utils.config import get_config, get_hf_text_config
15
-from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
15
+from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
16
cuda_device_count_stateless, get_cpu_memory, is_cpu,
17
is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
18
print_warning_once)
@@ -27,7 +27,6 @@ if TYPE_CHECKING:
27
28
logger = init_logger(__name__)
29
30
-_GB = 1 << 30
30
_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
31
32
_PP_SUPPORTED_MODELS = [
@@ -492,7 +491,7 @@ class CacheConfig:
491
self,
492
block_size: int,
493
gpu_memory_utilization: float,
495
- swap_space: int,
494
+ swap_space: float,
495
cache_dtype: str,
496
num_gpu_blocks_override: Optional[int] = None,
497
sliding_window: Optional[int] = None,
@@ -501,7 +500,7 @@ class CacheConfig:
500
) -> None:
501
self.block_size = block_size
502
self.gpu_memory_utilization = gpu_memory_utilization
504
- self.swap_space_bytes = swap_space * _GB
503
+ self.swap_space_bytes = swap_space * GiB_bytes
504
self.num_gpu_blocks_override = num_gpu_blocks_override
505
self.cache_dtype = cache_dtype
506
self.sliding_window = sliding_window
@@ -561,9 +560,9 @@ class CacheConfig:
560
num_gpus_per_node = parallel_config.tensor_parallel_size
561
cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node
562
564
- msg = (f"{cpu_memory_usage / _GB:.2f} GiB out of "
565
- f"the {total_cpu_memory / _GB:.2f} GiB total CPU memory is "
566
- "allocated for the swap space.")
563
+ msg = (f"{cpu_memory_usage / GiB_bytes:.2f} GiB out of the "
564
+ f"{total_cpu_memory / GiB_bytes:.2f} GiB total CPU memory "
565
+ "is allocated for the swap space.")
566
if cpu_memory_usage > 0.7 * total_cpu_memory:
567
raise ValueError("Too large swap space. " + msg)
568
elif cpu_memory_usage > 0.4 * total_cpu_memory:
vllm/engine/arg_utils.py
+3
-3
@@ -58,8 +58,8 @@ class EngineArgs:
58
enable_prefix_caching: bool = False
59
disable_sliding_window: bool = False
60
use_v2_block_manager: bool = False
61
- swap_space: int = 4 # GiB
62
- cpu_offload_gb: int = 0 # GiB
61
+ swap_space: float = 4 # GiB
62
+ cpu_offload_gb: float = 0 # GiB
63
gpu_memory_utilization: float = 0.90
64
max_num_batched_tokens: Optional[int] = None
65
max_num_seqs: int = 256
@@ -321,7 +321,7 @@ class EngineArgs:
321
default=EngineArgs.seed,
322
help='Random seed for operations.')
323
parser.add_argument('--swap-space',
324
- type=int,
324
+ type=float,
325
default=EngineArgs.swap_space,
326
help='CPU swap space size (GiB) per GPU.')
327
parser.add_argument(
vllm/executor/cpu_executor.py
+3
-4
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sequence import ExecuteModelRequest, SamplerOutput
16
-from vllm.utils import (get_distributed_init_method, get_open_port,
16
+from vllm.utils import (GiB_bytes, get_distributed_init_method, get_open_port,
17
get_vllm_instance_id, make_async)
18
from vllm.worker.worker_base import WorkerWrapperBase
19
@@ -332,7 +332,6 @@ def _verify_and_get_scheduler_config(
332
333
334
def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
335
- _GB = 1 << 30
335
if config.enable_prefix_caching:
336
logger.warning("Prefix caching is not supported on CPU, disable it.")
337
config.enable_prefix_caching = False
@@ -341,11 +340,11 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
340
341
if kv_cache_space >= 0:
342
if kv_cache_space == 0:
344
- config.cpu_kvcache_space_bytes = 4 * _GB # type: ignore
343
+ config.cpu_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
344
logger.warning("Environment variable VLLM_CPU_KVCACHE_SPACE (GB) "
345
"for CPU backend is not set, using 4 by default.")
346
else:
348
- config.cpu_kvcache_space_bytes = kv_cache_space * _GB # type: ignore
347
+ config.cpu_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
348
else:
349
raise RuntimeError(
350
"Invalid environment variable VLLM_CPU_KVCACHE_SPACE"
vllm/executor/openvino_executor.py
+4
-5
@@ -10,8 +10,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.sequence import ExecuteModelRequest, SamplerOutput
13
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
14
- make_async)
13
+from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
14
+ get_open_port, make_async)
15
16
logger = init_logger(__name__)
17
@@ -165,14 +165,13 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
165
166
kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
167
if kv_cache_space >= 0:
168
- _GB = 1 << 30
168
if kv_cache_space == 0:
170
- config.openvino_kvcache_space_bytes = 4 * _GB # type: ignore
169
+ config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
170
logger.warning(
171
"Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
172
"for OpenVINO backend is not set, using 4 by default.")
173
else:
175
- config.openvino_kvcache_space_bytes = kv_cache_space * _GB # type: ignore
174
+ config.openvino_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
175
else:
176
raise RuntimeError(
177
"Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
vllm/utils.py
+3
@@ -115,6 +115,9 @@ STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
115
STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
116
STR_INVALID_VAL: str = "INVALID"
117
118
+GiB_bytes = 1 << 30
119
+"""The number of bytes in one gibibyte (GiB)."""
120
+
121
STR_DTYPE_TO_TORCH_DTYPE = {
122
"half": torch.half,
123
"bfloat16": torch.bfloat16,
vllm/worker/tpu_worker.py
+2
-2
@@ -143,8 +143,8 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
143
num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
144
145
# Calculate the CPU KV cache size based on the config.
146
- num_cpu_blocks = (self.cache_config.swap_space_bytes //
147
- block_size_bytes)
146
+ num_cpu_blocks = int(self.cache_config.swap_space_bytes //
147
+ block_size_bytes)
148
num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
149
return num_tpu_blocks, num_cpu_blocks
150