@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baa240252e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Fix edge case in chunked prefill + block manager v2 (#7380)

Cade Daniel committed Aug 9, 2024 at 16:48 UTC baa240252ed8d44ae51455e24ebe644220d52116
2 files changed +21 -3
tests/core/block/e2e/test_correctness.py
+15 -3
@@ -261,11 +261,22 @@ def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
261 # skip cuda graph creation for fast test.
262 "enforce_eager": True,
263 "enable_chunked_prefill": True,
264 - "max_num_batched_tokens": 2,
265 - "max_num_seqs": 2,
264 },
265 ])
268 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
266 +@pytest.mark.parametrize("per_test_common_llm_kwargs",
267 + [{
268 + "block_size": 8,
269 + "max_num_batched_tokens": 2,
270 + "max_num_seqs": 2,
271 + }, {
272 + "block_size": 8,
273 + "max_num_batched_tokens": 3,
274 + "max_num_seqs": 2,
275 + }, {
276 + "block_size": 8,
277 + "max_num_batched_tokens": 256,
278 + "max_num_seqs": 10,
279 + }])
280 @pytest.mark.parametrize("baseline_llm_kwargs", [
281 {
282 "use_v2_block_manager": False,
@@ -294,6 +305,7 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
305 prompts = [
306 "Hello, my name is",
307 "The president of the United States is",
308 + ("1 + " * 50) + " 1 = ", # Longer prompt.
309 "The capital of France is",
310 "The future of AI is",
311 ]
vllm/core/block/block_table.py
+6
@@ -356,7 +356,13 @@ class BlockTable:
356 appended to blocks. The first such "token block" may have less token ids
357 than the block size, since the last allocated block may be partially
358 full.
359 +
360 + If no token ids are provided, then no chunks are returned.
361 """
362 +
363 + if not token_ids:
364 + return []
365 +
366 first_chunk_size = self._block_size - (self._num_full_slots %
367 self._block_size)
368 token_blocks = [token_ids[:first_chunk_size]]