A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core][Bugfix]: fix prefix caching for blockv2 (#5364)
Signed-off-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Lei Wen <wenlei03@qiyi.com>
leiwen83 committed
Jun 15, 2024 at 08:23 UTC
1b8a0d71cf5aa1a43c14478ec90538c3fbe1b315
2 files changed
+72
-2
tests/core/block/e2e/test_correctness.py
+67
@@ -477,3 +477,70 @@ def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
477
assert expected_token_ids == actual_token_ids
478
479
assert baseline_token_ids == test_token_ids
480
+
481
+
482
+@pytest.mark.parametrize(
483
+ "common_llm_kwargs",
484
+ [{
485
+ # Use a small model for a fast test.
486
+ "model": "facebook/opt-125m",
487
+
488
+ # skip cuda graph creation for fast test.
489
+ "enforce_eager": True,
490
+
491
+ # we keep the blocks small, so that hit eviction quickly
492
+ "max_model_len": 48,
493
+ "block_size": 16,
494
+ "num_gpu_blocks_override": 3,
495
+
496
+ # Test APC in v2 block
497
+ "use_v2_block_manager": True,
498
+ }])
499
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
500
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
501
+ "enable_prefix_caching": False
502
+}])
503
+@pytest.mark.parametrize("test_llm_kwargs", [{
504
+ "enable_prefix_caching": True,
505
+}])
506
+@pytest.mark.parametrize("seed", [1])
507
+def test_auto_prefix_caching_after_evition_start(baseline_llm_generator,
508
+ test_llm_generator):
509
+ """Verify block manager v2 with auto prefix caching could works normal
510
+ even when eviction started.
511
+ With APC enabled, all blocks are held by native block at the beginning.
512
+ Then blocks are managed by evictor instead. If cache hit at the evitor's
513
+ block, then it could be reused, or we need to recompute its kv cache.
514
+ """
515
+ output_len = 10
516
+ temperature = 0.0
517
+
518
+ prompts = [
519
+ "You are a helpful assistant. Please answer truthfully and write "
520
+ "out your thinking step by step to be sure you get the right answer. "
521
+ "If you make a mistake, attempt to correct it. who are you?",
522
+ "You are a helpful assistant. Please answer truthfully and write out "
523
+ "your thinking step by step to be sure you get the right answer. You "
524
+ "are helpful and harmless and you follow ethical guidelines. "
525
+ "who are you?"
526
+ ]
527
+
528
+ sampling_params = SamplingParams(
529
+ max_tokens=output_len,
530
+ ignore_eos=True,
531
+ temperature=temperature,
532
+ )
533
+
534
+ print('Getting token ids with APC disabled')
535
+ baseline_token_ids = get_token_ids_from_llm_generator(
536
+ baseline_llm_generator, prompts, sampling_params)
537
+
538
+ print('Getting token ids with APC enabled')
539
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
540
+ prompts, sampling_params)
541
+
542
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
543
+ test_token_ids):
544
+ assert expected_token_ids == actual_token_ids
545
+
546
+ assert baseline_token_ids == test_token_ids
vllm/core/block/prefix_caching_block.py
+5
-2
@@ -176,14 +176,17 @@ class PrefixCachingBlockAllocator(BlockAllocator):
176
177
self._refcounter.incr(block_id)
178
179
- # the block comes from evictor already contain computed result
179
+ # Now this block is pop from evictor and ready to write
180
+ # with new content which most probably different with
181
+ # original content. So need to tell worker to recompute
182
+ # its kvcache
183
block = self._create_block(
184
prev_block=prev_block,
185
token_ids=[],
186
block_size=self._block_size,
187
allocator=self,
188
block_id=block_id,
186
- computed=True,
189
+ computed=False,
190
)
191
assert block.content_hash is None
192