@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8fae5ed7f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Misc] Fix minor typo in scheduler (#8765)

Woo-Yeon Lee committed Sep 25, 2024 at 16:53 UTC 8fae5ed7f6bfd63b81310fcb24b310d9205c9687
1 file changed +3 -3
vllm/core/scheduler.py
+3 -3
@@ -1554,14 +1554,14 @@ class Scheduler:
1554 # the number of new tokens that is dividable by the block size
1555 # to avoid partial block matching.
1556 block_size = self.cache_config.block_size
1557 - reminder = budget.token_budget % block_size
1558 - if reminder != 0:
1557 + remainder = budget.token_budget % block_size
1558 + if remainder != 0:
1559 raise ValueError("When enabling chunked prefill and "
1560 "prefix caching, max_num_batched_tokens "
1561 "(chunk size) must be dividable by "
1562 "block size, but got chunk_size "
1563 f"({budget.token_budget}) % block_size "
1564 - f"({block_size}) = {reminder}")
1564 + f"({block_size}) = {remainder}")
1565 if remaining_token_budget < num_new_tokens:
1566 num_new_tokens = (remaining_token_budget //
1567 block_size) * block_size