A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Platform] platform agnostic for EngineArgs initialization (#11225)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Dec 17, 2024 at 14:11 UTC
e88db68cf5712956f36e77c288699592327b15bd
9 files changed
+37
-6
vllm/engine/arg_utils.py
+2
-6
@@ -112,9 +112,7 @@ class EngineArgs:
112
pipeline_parallel_size: int = 1
113
tensor_parallel_size: int = 1
114
max_parallel_loading_workers: Optional[int] = None
115
- # NOTE(kzawora): default block size for Gaudi should be 128
116
- # smaller sizes still work, but very inefficiently
117
- block_size: int = 16 if not current_platform.is_hpu() else 128
115
+ block_size: Optional[int] = None
116
enable_prefix_caching: Optional[bool] = None
117
disable_sliding_window: bool = False
118
use_v2_block_manager: bool = True
@@ -1036,9 +1034,7 @@ class EngineArgs:
1034
self.enable_prefix_caching = False
1035
1036
cache_config = CacheConfig(
1039
- # neuron needs block_size = max_model_len
1040
- block_size=self.block_size if self.device != "neuron" else
1041
- (self.max_model_len if self.max_model_len is not None else 0),
1037
+ block_size=self.block_size,
1038
gpu_memory_utilization=self.gpu_memory_utilization,
1039
swap_space=self.swap_space,
1040
cache_dtype=self.kv_cache_dtype,
vllm/platforms/cpu.py
+3
@@ -60,6 +60,9 @@ class CpuPlatform(Platform):
60
61
cache_config = vllm_config.cache_config
62
63
+ if cache_config and cache_config.block_size is None:
64
+ cache_config.block_size = 16
65
+
66
kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
67
68
if kv_cache_space >= 0:
vllm/platforms/cuda.py
+4
@@ -137,6 +137,10 @@ class CudaPlatformBase(Platform):
137
else:
138
parallel_config.worker_cls = "vllm.worker.worker.Worker"
139
140
+ cache_config = vllm_config.cache_config
141
+ if cache_config and cache_config.block_size is None:
142
+ cache_config.block_size = 16
143
+
144
145
# NVML utils
146
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/hpu.py
+6
@@ -48,6 +48,12 @@ class HpuPlatform(Platform):
48
if parallel_config.worker_cls == "auto":
49
parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"
50
51
+ # NOTE(kzawora): default block size for Gaudi should be 128
52
+ # smaller sizes still work, but very inefficiently
53
+ cache_config = vllm_config.cache_config
54
+ if cache_config and cache_config.block_size is None:
55
+ cache_config.block_size = 128
56
+
57
@classmethod
58
def is_pin_memory_available(cls):
59
logger.warning("Pin memory is not supported on HPU.")
vllm/platforms/neuron.py
+6
@@ -33,6 +33,12 @@ class NeuronPlatform(Platform):
33
parallel_config.worker_cls = \
34
"vllm.worker.neuron_worker.NeuronWorker"
35
36
+ cache_config = vllm_config.cache_config
37
+ if cache_config:
38
+ # neuron needs block_size = max_model_len
39
+ vllm_config.cache_config.block_size = \
40
+ vllm_config.model_config.max_model_len
41
+
42
@classmethod
43
def is_pin_memory_available(cls) -> bool:
44
logger.warning("Pin memory is not supported on Neuron.")
vllm/platforms/openvino.py
+3
@@ -87,6 +87,9 @@ class OpenVinoPlatform(Platform):
87
# check and update cache config
88
ov_core = ov.Core()
89
cache_config = vllm_config.cache_config
90
+ if cache_config and cache_config.block_size is None:
91
+ cache_config.block_size = 16
92
+
93
if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
94
if not OpenVinoPlatform.is_openvino_cpu():
95
logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
vllm/platforms/rocm.py
+4
@@ -84,6 +84,10 @@ class RocmPlatform(Platform):
84
85
@classmethod
86
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
87
+ cache_config = vllm_config.cache_config
88
+ if cache_config and cache_config.block_size is None:
89
+ cache_config.block_size = 16
90
+
91
parallel_config = vllm_config.parallel_config
92
scheduler_config = vllm_config.scheduler_config
93
if parallel_config.worker_cls == "auto":
vllm/platforms/tpu.py
+5
@@ -46,6 +46,11 @@ class TpuPlatform(Platform):
46
@classmethod
47
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
48
from vllm.config import CompilationLevel
49
+
50
+ cache_config = vllm_config.cache_config
51
+ if cache_config and cache_config.block_size is None:
52
+ cache_config.block_size = 16
53
+
54
compilation_config = vllm_config.compilation_config
55
if compilation_config.level == CompilationLevel.NO_COMPILATION:
56
# TPU does not support NO_COMPILATION
vllm/platforms/xpu.py
+4
@@ -51,6 +51,10 @@ class XPUPlatform(Platform):
51
52
@classmethod
53
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
54
+ cache_config = vllm_config.cache_config
55
+ if cache_config and cache_config.block_size is None:
56
+ cache_config.block_size = 16
57
+
58
# check and update model config
59
model_config = vllm_config.model_config
60
if model_config.dtype == torch.bfloat16: