A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Model] Phi-3 4k sliding window temp. fix (#4380)
Caio Mendes committed
Apr 27, 2024 at 07:08 UTC
3da24c2df735354ccb463650c29cca8ce506fa07
1 file changed
+4
-3
vllm/core/block_manager_v1.py
+4
-3
@@ -1,4 +1,5 @@
1
"""A block manager that manages token blocks."""
2
+import math
3
from abc import ABC, abstractmethod
4
from itertools import count, takewhile
5
from os.path import commonprefix
@@ -220,9 +221,9 @@ class BlockSpaceManagerV1(BlockSpaceManager):
221
222
self.block_sliding_window = None
223
if sliding_window is not None:
223
- assert sliding_window % block_size == 0, (sliding_window,
224
- block_size)
225
- self.block_sliding_window = sliding_window // block_size
224
+ # Round up to nearest block size to regularize sliding window
225
+ # allocation sizes.
226
+ self.block_sliding_window = math.ceil(sliding_window / block_size)
227
228
self.watermark = watermark
229
assert watermark >= 0.0