A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Fix edge case in chunked prefill + block manager v2 (#7380)
Cade Daniel committed
Aug 9, 2024 at 16:48 UTC
baa240252ed8d44ae51455e24ebe644220d52116
2 files changed
+21
-3
tests/core/block/e2e/test_correctness.py
+15
-3
@@ -261,11 +261,22 @@ def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
261
# skip cuda graph creation for fast test.
262
"enforce_eager": True,
263
"enable_chunked_prefill": True,
264
- "max_num_batched_tokens": 2,
265
- "max_num_seqs": 2,
264
},
265
])
268
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
266
+@pytest.mark.parametrize("per_test_common_llm_kwargs",
267
+ [{
268
+ "block_size": 8,
269
+ "max_num_batched_tokens": 2,
270
+ "max_num_seqs": 2,
271
+ }, {
272
+ "block_size": 8,
273
+ "max_num_batched_tokens": 3,
274
+ "max_num_seqs": 2,
275
+ }, {
276
+ "block_size": 8,
277
+ "max_num_batched_tokens": 256,
278
+ "max_num_seqs": 10,
279
+ }])
280
@pytest.mark.parametrize("baseline_llm_kwargs", [
281
{
282
"use_v2_block_manager": False,
@@ -294,6 +305,7 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
305
prompts = [
306
"Hello, my name is",
307
"The president of the United States is",
308
+ ("1 + " * 50) + " 1 = ", # Longer prompt.
309
"The capital of France is",
310
"The future of AI is",
311
]
vllm/core/block/block_table.py
+6
@@ -356,7 +356,13 @@ class BlockTable:
356
appended to blocks. The first such "token block" may have less token ids
357
than the block size, since the last allocated block may be partially
358
full.
359
+
360
+ If no token ids are provided, then no chunks are returned.
361
"""
362
+
363
+ if not token_ids:
364
+ return []
365
+
366
first_chunk_size = self._block_size - (self._num_full_slots %
367
self._block_size)
368
token_blocks = [token_ids[:first_chunk_size]]