@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e67c295b0c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Bugfix] fix automatic prefix args and add log info (#3608)

TianYu GUO committed Mar 25, 2024 at 20:35 UTC e67c295b0c812a4969d5bbc70818f6d87c87f9e4
2 files changed +7 -1
vllm/core/block_manager.py
+5
@@ -9,6 +9,9 @@ from vllm.block import BlockTable, PhysicalTokenBlock
9 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
10 from vllm.utils import Device
11 from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
12 +from vllm.logger import init_logger
13 +
14 +logger = init_logger(__name__)
15
16
17 class BlockAllocatorBase(ABC):
@@ -241,11 +244,13 @@ class BlockSpaceManager:
244 self.watermark_blocks = int(watermark * num_gpu_blocks)
245
246 if self.enable_caching:
247 + logger.info("enable automatic prefix caching")
248 self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
249 num_gpu_blocks)
250 self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
251 num_cpu_blocks)
252 else:
253 + logger.info("disable automatic prefix caching")
254 self.gpu_allocator = UncachedBlockAllocator(
255 Device.GPU, block_size, num_gpu_blocks)
256 self.cpu_allocator = UncachedBlockAllocator(
vllm/engine/arg_utils.py
+2 -1
@@ -337,7 +337,8 @@ class EngineArgs:
337 cache_config = CacheConfig(self.block_size,
338 self.gpu_memory_utilization,
339 self.swap_space, self.kv_cache_dtype,
340 - model_config.get_sliding_window())
340 + model_config.get_sliding_window(),
341 + self.enable_prefix_caching)
342 parallel_config = ParallelConfig(
343 self.pipeline_parallel_size, self.tensor_parallel_size,
344 self.worker_use_ray, self.max_parallel_loading_workers,