A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix auto prefix bug (#3239)
ElizaWszola committed
Mar 8, 2024 at 01:37 UTC
b35cc93420e37b72dc1c4bbedb06012fd294b743
3 files changed
+51
-12
tests/engine/test_computed_prefix_blocks.py
new
+34
@@ -0,0 +1,34 @@
1
+import pytest
2
+
3
+from vllm.engine.arg_utils import EngineArgs
4
+from vllm.engine.llm_engine import LLMEngine
5
+from vllm.sampling_params import SamplingParams
6
+
7
+
8
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
9
+@pytest.mark.parametrize("block_size", [16])
10
+def test_computed_prefix_blocks(model: str, block_size: int):
11
+ # This test checks if we are able to run the engine to completion
12
+ # without triggering asserts.
13
+ # We are in a scenario where all blocks from the second request's prompt
14
+ # are full and already computed when the second request arrives.
15
+ prompt = (
16
+ "You are a helpful assistant. How do I build a car from cardboard and "
17
+ "paper clips? Is there an easy to follow video tutorial available "
18
+ "online for free?")
19
+ prompt2 = (
20
+ " Please recommend to me some resources where I can learn not only to "
21
+ "handle technical difficulties of building a car, but also "
22
+ "decoration.")
23
+
24
+ engine_args = EngineArgs(model=model,
25
+ block_size=block_size,
26
+ enable_prefix_caching=True)
27
+
28
+ engine = LLMEngine.from_engine_args(engine_args)
29
+ sampling_params = SamplingParams()
30
+
31
+ engine.add_request("0", prompt + prompt2, sampling_params)
32
+ engine.step()
33
+ engine.add_request("1", prompt, sampling_params)
34
+ engine.step()
vllm/core/block_manager.py
+16
-12
@@ -1,6 +1,6 @@
1
"""A block manager that manages token blocks."""
2
import enum
3
-from itertools import count
3
+from itertools import count, takewhile
4
from os.path import commonprefix
5
from typing import Dict, List, Optional, Set, Tuple
6
@@ -426,23 +426,29 @@ class BlockSpaceManager:
426
for block in block_table:
427
block.last_accessed = access_time
428
429
- def compute_last_full_block_in_seq(self, seq: Sequence):
429
+ def compute_full_blocks_in_seq(self, seq: Sequence):
430
if seq.seq_id not in self.block_tables:
431
return
432
max_full_block = seq.get_len() // self.block_size - 1
433
block_table = self.block_tables[seq.seq_id]
434
if max_full_block == -1:
435
return
436
- block_table[max_full_block].computed = True
436
+ for i in reversed(range(max_full_block)):
437
+ if block_table[i].computed:
438
+ break
439
+ block_table[i].computed = True
440
438
- def get_all_block_ids_till_computed(self, seq: Sequence) -> List[int]:
441
+ def get_all_computed_blocks(self, seq: Sequence) -> List[int]:
442
if seq.seq_id not in self.block_tables:
443
return []
444
block_table = self.block_tables[seq.seq_id]
442
- for block_idx in reversed(range(len(block_table))):
443
- if block_table[block_idx].computed:
444
- return [b.block_number for b in block_table[:block_idx + 1]]
445
- return []
445
+ # NOTE We exclude the last block to avoid the case where the entire
446
+ # prompt is cached. This would cause erroneous behavior in model
447
+ # runner.
448
+ return [
449
+ b.block_number
450
+ for b in takewhile(lambda b: b.computed, block_table[:-1])
451
+ ]
452
453
def get_common_computed_block_ids(self,
454
seq_group: SequenceGroup) -> List[int]:
@@ -451,14 +457,12 @@ class BlockSpaceManager:
457
return []
458
459
ids_list = [
454
- self.get_all_block_ids_till_computed(seq)
460
+ self.get_all_computed_blocks(seq)
461
for seq in iter(seq_group.seqs_dict.values())
462
]
463
return commonprefix([ids for ids in ids_list if ids != []])
464
465
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
460
- # NOTE: We only mark the last full block because with prefix caching,
461
- # all blocks until the marked one are guaranteed to be computed.
466
if self.enable_caching:
467
for seq in seq_group.seqs_dict.values():
464
- self.compute_last_full_block_in_seq(seq)
468
+ self.compute_full_blocks_in_seq(seq)
vllm/worker/model_runner.py
+1
@@ -215,6 +215,7 @@ class ModelRunner:
215
slot_mapping[-1].append(slot)
216
217
max_prompt_len = max(subquery_lens)
218
+ assert max_prompt_len > 0
219
input_tokens = _make_tensor_with_pad(input_tokens,
220
max_prompt_len,
221
pad=0,