A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] latency optimization (#3890)
youkaichao committed
Apr 6, 2024 at 19:14 UTC
2f1928354903ae0c6edfe76cc90081eb513ead2c
1 file changed
+1
-1
vllm/core/block_manager_v1.py
+1
-1
@@ -328,7 +328,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
328
self,
329
seq: Sequence,
330
) -> bool:
331
- token_ids_len = len(seq.data.get_token_ids())
331
+ token_ids_len = seq.data.get_len()
332
return token_ids_len > 0 and token_ids_len % seq.block_size == 0
333
334
def _maybe_promote_last_block(