A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc][Minor] Fix CPU block num log in CPUExecutor. (#4088)
Li, Jiang committed
Apr 15, 2024 at 23:35 UTC
0003e9154bf1091d0de7ca7a6c7f1253df1eca5b
1 file changed
+4
-1
vllm/executor/cpu_executor.py
+4
-1
@@ -74,7 +74,10 @@ class CPUExecutor(ExecutorBase):
74
# NOTE: We log here to avoid multiple logs when number of workers is
75
# greater than one. We could log in the engine, but not all executors
76
# have GPUs.
77
- logger.info(f"# CPU blocks: {num_cpu_blocks}")
77
+ # NOTE: `cpu block` for CPU backend is located on CPU memory but is
78
+ # referred as `gpu block`. Because we want to reuse the existing block
79
+ # management procedure.
80
+ logger.info(f"# CPU blocks: {num_gpu_blocks}")
81
self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
82
83
def execute_model(self,