A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[CPU] Change default block_size for CPU backend (#16002)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Apr 5, 2025 at 00:39 UTC
2386803f2a2e3df1f29ea05212eaf68590b85805
1 file changed
+9
-1
vllm/platforms/cpu.py
+9
-1
@@ -2,6 +2,7 @@
2
3
import os
4
import sys
5
+from importlib.util import find_spec
6
from typing import TYPE_CHECKING, Optional
7
8
import psutil
@@ -68,8 +69,15 @@ class CpuPlatform(Platform):
69
70
cache_config = vllm_config.cache_config
71
72
+ ipex_avaliable = find_spec("intel_extension_for_pytorch") is not None
73
+
74
if cache_config and cache_config.block_size is None:
72
- cache_config.block_size = 16
75
+ cache_config.block_size = 128 if ipex_avaliable else 16
76
+
77
+ if not ipex_avaliable and cache_config.block_size != 16:
78
+ raise RuntimeError(
79
+ f"--block-size={cache_config.block_size} requires"
80
+ " intel_extension_for_pytorch")
81
82
scheduler_config = vllm_config.scheduler_config
83
if ((scheduler_config.chunked_prefill_enabled