@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ddc4743d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Consolidate `GB` constant and enable float GB arguments (#7416)

Cyrus Leung committed Aug 13, 2024 at 05:14 UTC 4ddc4743d7ae93fca090d8485b31321abe3743e7
6 files changed +21 -21
vllm/config.py
+6 -7
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
13 from vllm.tracing import is_otel_installed
14 from vllm.transformers_utils.config import get_config, get_hf_text_config
15 -from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
15 +from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
16 cuda_device_count_stateless, get_cpu_memory, is_cpu,
17 is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
18 print_warning_once)
@@ -27,7 +27,6 @@ if TYPE_CHECKING:
27
28 logger = init_logger(__name__)
29
30 -_GB = 1 << 30
30 _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
31
32 _PP_SUPPORTED_MODELS = [
@@ -492,7 +491,7 @@ class CacheConfig:
491 self,
492 block_size: int,
493 gpu_memory_utilization: float,
495 - swap_space: int,
494 + swap_space: float,
495 cache_dtype: str,
496 num_gpu_blocks_override: Optional[int] = None,
497 sliding_window: Optional[int] = None,
@@ -501,7 +500,7 @@ class CacheConfig:
500 ) -> None:
501 self.block_size = block_size
502 self.gpu_memory_utilization = gpu_memory_utilization
504 - self.swap_space_bytes = swap_space * _GB
503 + self.swap_space_bytes = swap_space * GiB_bytes
504 self.num_gpu_blocks_override = num_gpu_blocks_override
505 self.cache_dtype = cache_dtype
506 self.sliding_window = sliding_window
@@ -561,9 +560,9 @@ class CacheConfig:
560 num_gpus_per_node = parallel_config.tensor_parallel_size
561 cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node
562
564 - msg = (f"{cpu_memory_usage / _GB:.2f} GiB out of "
565 - f"the {total_cpu_memory / _GB:.2f} GiB total CPU memory is "
566 - "allocated for the swap space.")
563 + msg = (f"{cpu_memory_usage / GiB_bytes:.2f} GiB out of the "
564 + f"{total_cpu_memory / GiB_bytes:.2f} GiB total CPU memory "
565 + "is allocated for the swap space.")
566 if cpu_memory_usage > 0.7 * total_cpu_memory:
567 raise ValueError("Too large swap space. " + msg)
568 elif cpu_memory_usage > 0.4 * total_cpu_memory:
vllm/engine/arg_utils.py
+3 -3
@@ -58,8 +58,8 @@ class EngineArgs:
58 enable_prefix_caching: bool = False
59 disable_sliding_window: bool = False
60 use_v2_block_manager: bool = False
61 - swap_space: int = 4 # GiB
62 - cpu_offload_gb: int = 0 # GiB
61 + swap_space: float = 4 # GiB
62 + cpu_offload_gb: float = 0 # GiB
63 gpu_memory_utilization: float = 0.90
64 max_num_batched_tokens: Optional[int] = None
65 max_num_seqs: int = 256
@@ -321,7 +321,7 @@ class EngineArgs:
321 default=EngineArgs.seed,
322 help='Random seed for operations.')
323 parser.add_argument('--swap-space',
324 - type=int,
324 + type=float,
325 default=EngineArgs.swap_space,
326 help='CPU swap space size (GiB) per GPU.')
327 parser.add_argument(
vllm/executor/cpu_executor.py
+3 -4
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sequence import ExecuteModelRequest, SamplerOutput
16 -from vllm.utils import (get_distributed_init_method, get_open_port,
16 +from vllm.utils import (GiB_bytes, get_distributed_init_method, get_open_port,
17 get_vllm_instance_id, make_async)
18 from vllm.worker.worker_base import WorkerWrapperBase
19
@@ -332,7 +332,6 @@ def _verify_and_get_scheduler_config(
332
333
334 def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
335 - _GB = 1 << 30
335 if config.enable_prefix_caching:
336 logger.warning("Prefix caching is not supported on CPU, disable it.")
337 config.enable_prefix_caching = False
@@ -341,11 +340,11 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
340
341 if kv_cache_space >= 0:
342 if kv_cache_space == 0:
344 - config.cpu_kvcache_space_bytes = 4 * _GB # type: ignore
343 + config.cpu_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
344 logger.warning("Environment variable VLLM_CPU_KVCACHE_SPACE (GB) "
345 "for CPU backend is not set, using 4 by default.")
346 else:
348 - config.cpu_kvcache_space_bytes = kv_cache_space * _GB # type: ignore
347 + config.cpu_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
348 else:
349 raise RuntimeError(
350 "Invalid environment variable VLLM_CPU_KVCACHE_SPACE"
vllm/executor/openvino_executor.py
+4 -5
@@ -10,8 +10,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.sequence import ExecuteModelRequest, SamplerOutput
13 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
14 - make_async)
13 +from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
14 + get_open_port, make_async)
15
16 logger = init_logger(__name__)
17
@@ -165,14 +165,13 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
165
166 kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE
167 if kv_cache_space >= 0:
168 - _GB = 1 << 30
168 if kv_cache_space == 0:
170 - config.openvino_kvcache_space_bytes = 4 * _GB # type: ignore
169 + config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
170 logger.warning(
171 "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) "
172 "for OpenVINO backend is not set, using 4 by default.")
173 else:
175 - config.openvino_kvcache_space_bytes = kv_cache_space * _GB # type: ignore
174 + config.openvino_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
175 else:
176 raise RuntimeError(
177 "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
vllm/utils.py
+3
@@ -115,6 +115,9 @@ STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
115 STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
116 STR_INVALID_VAL: str = "INVALID"
117
118 +GiB_bytes = 1 << 30
119 +"""The number of bytes in one gibibyte (GiB)."""
120 +
121 STR_DTYPE_TO_TORCH_DTYPE = {
122 "half": torch.half,
123 "bfloat16": torch.bfloat16,
vllm/worker/tpu_worker.py
+2 -2
@@ -143,8 +143,8 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
143 num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
144
145 # Calculate the CPU KV cache size based on the config.
146 - num_cpu_blocks = (self.cache_config.swap_space_bytes //
147 - block_size_bytes)
146 + num_cpu_blocks = int(self.cache_config.swap_space_bytes //
147 + block_size_bytes)
148 num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
149 return num_tpu_blocks, num_cpu_blocks
150