@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3da24c2df7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Model] Phi-3 4k sliding window temp. fix (#4380)

Caio Mendes committed Apr 27, 2024 at 07:08 UTC 3da24c2df735354ccb463650c29cca8ce506fa07
1 file changed +4 -3
vllm/core/block_manager_v1.py
+4 -3
@@ -1,4 +1,5 @@
1 """A block manager that manages token blocks."""
2 +import math
3 from abc import ABC, abstractmethod
4 from itertools import count, takewhile
5 from os.path import commonprefix
@@ -220,9 +221,9 @@ class BlockSpaceManagerV1(BlockSpaceManager):
221
222 self.block_sliding_window = None
223 if sliding_window is not None:
223 - assert sliding_window % block_size == 0, (sliding_window,
224 - block_size)
225 - self.block_sliding_window = sliding_window // block_size
224 + # Round up to nearest block size to regularize sliding window
225 + # allocation sizes.
226 + self.block_sliding_window = math.ceil(sliding_window / block_size)
227
228 self.watermark = watermark
229 assert watermark >= 0.0