A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix] Fix cpu usage and cache hit stats reporting on cpu environment (#18674)
Signed-off-by: zzzyq <zhangyuqi94@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Yuqi Zhang committed
May 25, 2025 at 02:36 UTC
f2faac745dc9b9d7d2fa92a9a4cfba6b230db2d4
1 file changed
+14
vllm/engine/llm_engine.py
+14
@@ -1650,6 +1650,20 @@ class LLMEngine:
1650
gpu_prefix_cache_hit_rate = self.scheduler[
1651
0].get_prefix_cache_hit_rate(Device.GPU)
1652
1653
+ # Exchange the uasge and cache hit stats between gpu and cpu when
1654
+ # running on cpu because the cpu_worker.py intentionally reports the
1655
+ # number of cpu blocks as gpu blocks in favor of cache management.
1656
+ if self.device_config.device_type == "cpu":
1657
+ num_total_gpu, num_total_cpu = num_total_cpu, num_total_gpu
1658
+ gpu_cache_usage_sys, cpu_cache_usage_sys = (
1659
+ cpu_cache_usage_sys,
1660
+ gpu_cache_usage_sys,
1661
+ )
1662
+ gpu_prefix_cache_hit_rate, cpu_prefix_cache_hit_rate = (
1663
+ cpu_prefix_cache_hit_rate,
1664
+ gpu_prefix_cache_hit_rate,
1665
+ )
1666
+
1667
# Iteration stats
1668
num_prompt_tokens_iter = 0
1669
num_generation_tokens_iter = 0