@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f19283549
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] latency optimization (#3890)

youkaichao committed Apr 6, 2024 at 19:14 UTC 2f1928354903ae0c6edfe76cc90081eb513ead2c
1 file changed +1 -1
vllm/core/block_manager_v1.py
+1 -1
@@ -328,7 +328,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
328 self,
329 seq: Sequence,
330 ) -> bool:
331 - token_ids_len = len(seq.data.get_token_ids())
331 + token_ids_len = seq.data.get_len()
332 return token_ids_len > 0 and token_ids_len % seq.block_size == 0
333
334 def _maybe_promote_last_block(