@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e88db68cf5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Platform] platform agnostic for EngineArgs initialization (#11225)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Dec 17, 2024 at 14:11 UTC e88db68cf5712956f36e77c288699592327b15bd
9 files changed +37 -6
vllm/engine/arg_utils.py
+2 -6
@@ -112,9 +112,7 @@ class EngineArgs:
112 pipeline_parallel_size: int = 1
113 tensor_parallel_size: int = 1
114 max_parallel_loading_workers: Optional[int] = None
115 - # NOTE(kzawora): default block size for Gaudi should be 128
116 - # smaller sizes still work, but very inefficiently
117 - block_size: int = 16 if not current_platform.is_hpu() else 128
115 + block_size: Optional[int] = None
116 enable_prefix_caching: Optional[bool] = None
117 disable_sliding_window: bool = False
118 use_v2_block_manager: bool = True
@@ -1036,9 +1034,7 @@ class EngineArgs:
1034 self.enable_prefix_caching = False
1035
1036 cache_config = CacheConfig(
1039 - # neuron needs block_size = max_model_len
1040 - block_size=self.block_size if self.device != "neuron" else
1041 - (self.max_model_len if self.max_model_len is not None else 0),
1037 + block_size=self.block_size,
1038 gpu_memory_utilization=self.gpu_memory_utilization,
1039 swap_space=self.swap_space,
1040 cache_dtype=self.kv_cache_dtype,
vllm/platforms/cpu.py
+3
@@ -60,6 +60,9 @@ class CpuPlatform(Platform):
60
61 cache_config = vllm_config.cache_config
62
63 + if cache_config and cache_config.block_size is None:
64 + cache_config.block_size = 16
65 +
66 kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
67
68 if kv_cache_space >= 0:
vllm/platforms/cuda.py
+4
@@ -137,6 +137,10 @@ class CudaPlatformBase(Platform):
137 else:
138 parallel_config.worker_cls = "vllm.worker.worker.Worker"
139
140 + cache_config = vllm_config.cache_config
141 + if cache_config and cache_config.block_size is None:
142 + cache_config.block_size = 16
143 +
144
145 # NVML utils
146 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/hpu.py
+6
@@ -48,6 +48,12 @@ class HpuPlatform(Platform):
48 if parallel_config.worker_cls == "auto":
49 parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"
50
51 + # NOTE(kzawora): default block size for Gaudi should be 128
52 + # smaller sizes still work, but very inefficiently
53 + cache_config = vllm_config.cache_config
54 + if cache_config and cache_config.block_size is None:
55 + cache_config.block_size = 128
56 +
57 @classmethod
58 def is_pin_memory_available(cls):
59 logger.warning("Pin memory is not supported on HPU.")
vllm/platforms/neuron.py
+6
@@ -33,6 +33,12 @@ class NeuronPlatform(Platform):
33 parallel_config.worker_cls = \
34 "vllm.worker.neuron_worker.NeuronWorker"
35
36 + cache_config = vllm_config.cache_config
37 + if cache_config:
38 + # neuron needs block_size = max_model_len
39 + vllm_config.cache_config.block_size = \
40 + vllm_config.model_config.max_model_len
41 +
42 @classmethod
43 def is_pin_memory_available(cls) -> bool:
44 logger.warning("Pin memory is not supported on Neuron.")
vllm/platforms/openvino.py
+3
@@ -87,6 +87,9 @@ class OpenVinoPlatform(Platform):
87 # check and update cache config
88 ov_core = ov.Core()
89 cache_config = vllm_config.cache_config
90 + if cache_config and cache_config.block_size is None:
91 + cache_config.block_size = 16
92 +
93 if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8":
94 if not OpenVinoPlatform.is_openvino_cpu():
95 logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is"
vllm/platforms/rocm.py
+4
@@ -84,6 +84,10 @@ class RocmPlatform(Platform):
84
85 @classmethod
86 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
87 + cache_config = vllm_config.cache_config
88 + if cache_config and cache_config.block_size is None:
89 + cache_config.block_size = 16
90 +
91 parallel_config = vllm_config.parallel_config
92 scheduler_config = vllm_config.scheduler_config
93 if parallel_config.worker_cls == "auto":
vllm/platforms/tpu.py
+5
@@ -46,6 +46,11 @@ class TpuPlatform(Platform):
46 @classmethod
47 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
48 from vllm.config import CompilationLevel
49 +
50 + cache_config = vllm_config.cache_config
51 + if cache_config and cache_config.block_size is None:
52 + cache_config.block_size = 16
53 +
54 compilation_config = vllm_config.compilation_config
55 if compilation_config.level == CompilationLevel.NO_COMPILATION:
56 # TPU does not support NO_COMPILATION
vllm/platforms/xpu.py
+4
@@ -51,6 +51,10 @@ class XPUPlatform(Platform):
51
52 @classmethod
53 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
54 + cache_config = vllm_config.cache_config
55 + if cache_config and cache_config.block_size is None:
56 + cache_config.block_size = 16
57 +
58 # check and update model config
59 model_config = vllm_config.model_config
60 if model_config.dtype == torch.bfloat16: