A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix][Example] make lmcache v0 work. (#18051)
Signed-off-by: Ma, Jianpeng <jianpeng.ma@intel.com>
majianpeng committed
May 14, 2025 at 14:43 UTC
e7ef61c1f039a8eac98602a9e5ab7517027e7278
1 file changed
+4
-2
examples/lmcache/cpu_offload_lmcache.py
+4
-2
@@ -34,7 +34,7 @@ from vllm.config import KVTransferConfig
34
from vllm.engine.arg_utils import EngineArgs
35
36
37
-def setup_environment_variables():
37
+def setup_environment_variables(vllm_version: str):
38
# LMCache-related environment variables
39
# Use experimental features in LMCache
40
os.environ["LMCACHE_USE_EXPERIMENTAL"] = "True"
@@ -44,6 +44,8 @@ def setup_environment_variables():
44
os.environ["LMCACHE_LOCAL_CPU"] = "True"
45
# Set local CPU memory limit to 5.0 GB
46
os.environ["LMCACHE_MAX_LOCAL_CPU_SIZE"] = "5.0"
47
+ if vllm_version == "v0":
48
+ os.environ["VLLM_USE_V1"] = "0"
49
50
51
@contextlib.contextmanager
@@ -120,7 +122,7 @@ def main():
122
lmcache_connector = "LMCacheConnectorV1"
123
model = "meta-llama/Meta-Llama-3.1-8B-Instruct"
124
123
- setup_environment_variables()
125
+ setup_environment_variables(args.version)
126
127
with build_llm_with_lmcache(lmcache_connector, model, args.version) as llm:
128