A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix divide by zero when serving Mamba models (#9617)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Oct 23, 2024 at 12:40 UTC
e5ac6a4199fd967d2655310712cee6e642e91bd7
1 file changed
+2
-2
vllm/engine/llm_engine.py
+2
-2
@@ -1612,7 +1612,7 @@ class LLMEngine:
1612
# KV Cache Usage in %
1613
num_total_gpu = self.cache_config.num_gpu_blocks
1614
gpu_cache_usage_sys = 0.
1615
- if num_total_gpu is not None:
1615
+ if num_total_gpu: # Guard against both None and 0
1616
num_free_gpu = sum(
1617
scheduler.block_manager.get_num_free_gpu_blocks()
1618
for scheduler in self.scheduler)
@@ -1620,7 +1620,7 @@ class LLMEngine:
1620
1621
num_total_cpu = self.cache_config.num_cpu_blocks
1622
cpu_cache_usage_sys = 0.
1623
- if num_total_cpu is not None and num_total_cpu > 0:
1623
+ if num_total_cpu: # Guard against both None and 0
1624
num_free_cpu = sum(
1625
scheduler.block_manager.get_num_free_cpu_blocks()
1626
for scheduler in self.scheduler)