A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Misc] Fix minor typo in scheduler (#8765)
Woo-Yeon Lee committed
Sep 25, 2024 at 16:53 UTC
8fae5ed7f6bfd63b81310fcb24b310d9205c9687
1 file changed
+3
-3
vllm/core/scheduler.py
+3
-3
@@ -1554,14 +1554,14 @@ class Scheduler:
1554
# the number of new tokens that is dividable by the block size
1555
# to avoid partial block matching.
1556
block_size = self.cache_config.block_size
1557
- reminder = budget.token_budget % block_size
1558
- if reminder != 0:
1557
+ remainder = budget.token_budget % block_size
1558
+ if remainder != 0:
1559
raise ValueError("When enabling chunked prefill and "
1560
"prefix caching, max_num_batched_tokens "
1561
"(chunk size) must be dividable by "
1562
"block size, but got chunk_size "
1563
f"({budget.token_budget}) % block_size "
1564
- f"({block_size}) = {reminder}")
1564
+ f"({block_size}) = {remainder}")
1565
if remaining_token_budget < num_new_tokens:
1566
num_new_tokens = (remaining_token_budget //
1567
block_size) * block_size