@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e5ac6a4199
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix divide by zero when serving Mamba models (#9617)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Oct 23, 2024 at 12:40 UTC e5ac6a4199fd967d2655310712cee6e642e91bd7
1 file changed +2 -2
vllm/engine/llm_engine.py
+2 -2
@@ -1612,7 +1612,7 @@ class LLMEngine:
1612 # KV Cache Usage in %
1613 num_total_gpu = self.cache_config.num_gpu_blocks
1614 gpu_cache_usage_sys = 0.
1615 - if num_total_gpu is not None:
1615 + if num_total_gpu: # Guard against both None and 0
1616 num_free_gpu = sum(
1617 scheduler.block_manager.get_num_free_gpu_blocks()
1618 for scheduler in self.scheduler)
@@ -1620,7 +1620,7 @@ class LLMEngine:
1620
1621 num_total_cpu = self.cache_config.num_cpu_blocks
1622 cpu_cache_usage_sys = 0.
1623 - if num_total_cpu is not None and num_total_cpu > 0:
1623 + if num_total_cpu: # Guard against both None and 0
1624 num_free_cpu = sum(
1625 scheduler.block_manager.get_num_free_cpu_blocks()
1626 for scheduler in self.scheduler)