@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0003e9154b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc][Minor] Fix CPU block num log in CPUExecutor. (#4088)

Li, Jiang committed Apr 15, 2024 at 23:35 UTC 0003e9154bf1091d0de7ca7a6c7f1253df1eca5b
1 file changed +4 -1
vllm/executor/cpu_executor.py
+4 -1
@@ -74,7 +74,10 @@ class CPUExecutor(ExecutorBase):
74 # NOTE: We log here to avoid multiple logs when number of workers is
75 # greater than one. We could log in the engine, but not all executors
76 # have GPUs.
77 - logger.info(f"# CPU blocks: {num_cpu_blocks}")
77 + # NOTE: `cpu block` for CPU backend is located on CPU memory but is
78 + # referred as `gpu block`. Because we want to reuse the existing block
79 + # management procedure.
80 + logger.info(f"# CPU blocks: {num_gpu_blocks}")
81 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
82
83 def execute_model(self,