A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Bugfix] fix automatic prefix args and add log info (#3608)
TianYu GUO committed
Mar 25, 2024 at 20:35 UTC
e67c295b0c812a4969d5bbc70818f6d87c87f9e4
2 files changed
+7
-1
vllm/core/block_manager.py
+5
@@ -9,6 +9,9 @@ from vllm.block import BlockTable, PhysicalTokenBlock
9
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
10
from vllm.utils import Device
11
from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
12
+from vllm.logger import init_logger
13
+
14
+logger = init_logger(__name__)
15
16
17
class BlockAllocatorBase(ABC):
@@ -241,11 +244,13 @@ class BlockSpaceManager:
244
self.watermark_blocks = int(watermark * num_gpu_blocks)
245
246
if self.enable_caching:
247
+ logger.info("enable automatic prefix caching")
248
self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
249
num_gpu_blocks)
250
self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
251
num_cpu_blocks)
252
else:
253
+ logger.info("disable automatic prefix caching")
254
self.gpu_allocator = UncachedBlockAllocator(
255
Device.GPU, block_size, num_gpu_blocks)
256
self.cpu_allocator = UncachedBlockAllocator(
vllm/engine/arg_utils.py
+2
-1
@@ -337,7 +337,8 @@ class EngineArgs:
337
cache_config = CacheConfig(self.block_size,
338
self.gpu_memory_utilization,
339
self.swap_space, self.kv_cache_dtype,
340
- model_config.get_sliding_window())
340
+ model_config.get_sliding_window(),
341
+ self.enable_prefix_caching)
342
parallel_config = ParallelConfig(
343
self.pipeline_parallel_size, self.tensor_parallel_size,
344
self.worker_use_ray, self.max_parallel_loading_workers,