A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Misc] Update outdated note: LMCache now supports chunked prefill (#16697)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Apr 18, 2025 at 13:12 UTC
7a4a5de729466de4142e70b845e78a0171296468
1 file changed
+2
-2
examples/offline_inference/cpu_offload_lmcache.py
+2
-2
@@ -37,11 +37,11 @@ def build_llm_with_lmcache():
37
'{"kv_connector":"LMCacheConnector", "kv_role":"kv_both"}')
38
# Set GPU memory utilization to 0.8 for an A40 GPU with 40GB
39
# memory. Reduce the value if your GPU has less memory.
40
- # Note that LMCache is not compatible with chunked prefill for now.
40
+ # Note: LMCache supports chunked prefill (see vLLM#14505, LMCache#392).
41
llm = LLM(model="mistralai/Mistral-7B-Instruct-v0.2",
42
kv_transfer_config=ktc,
43
max_model_len=8000,
44
- enable_chunked_prefill=False,
44
+ enable_chunked_prefill=True,
45
gpu_memory_utilization=0.8)
46
47
try: