A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix] Update the example code, make it work with the latest lmcache (#19453)
Signed-off-by: Runzhen Wang <wangrunzhen@gmail.com>
runzhen committed
Jun 11, 2025 at 05:42 UTC
943ffa57032d62c21610e9cebffbdbe6c5c886ca
2 files changed
+3
-3
examples/others/lmcache/cpu_offload_lmcache.py
+1
-1
@@ -28,8 +28,8 @@ import os
28
import time
29
from dataclasses import asdict
30
31
-from lmcache.experimental.cache_engine import LMCacheEngineBuilder
31
from lmcache.integration.vllm.utils import ENGINE_NAME
32
+from lmcache.v1.cache_engine import LMCacheEngineBuilder
33
34
from vllm import LLM, SamplingParams
35
from vllm.config import KVTransferConfig
examples/others/lmcache/kv_cache_sharing_lmcache_v1.py
+2
-2
@@ -17,8 +17,8 @@ import subprocess
17
import time
18
from multiprocessing import Event, Process
19
20
-from lmcache.experimental.cache_engine import LMCacheEngineBuilder
20
from lmcache.integration.vllm.utils import ENGINE_NAME
21
+from lmcache.v1.cache_engine import LMCacheEngineBuilder
22
23
from vllm import LLM, SamplingParams
24
from vllm.config import KVTransferConfig
@@ -105,7 +105,7 @@ def run_retrieve(store_done, prompts, timeout=1):
105
106
def run_lmcache_server(port):
107
server_proc = subprocess.Popen(
108
- ["python", "-m", "lmcache.experimental.server", "localhost", str(port)]
108
+ ["python", "-m", "lmcache.v1.server", "localhost", str(port)]
109
)
110
return server_proc
111