@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b35cc93420
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix auto prefix bug (#3239)

ElizaWszola committed Mar 8, 2024 at 01:37 UTC b35cc93420e37b72dc1c4bbedb06012fd294b743
3 files changed +51 -12
tests/engine/test_computed_prefix_blocks.py new
+34
@@ -0,0 +1,34 @@
1 +import pytest
2 +
3 +from vllm.engine.arg_utils import EngineArgs
4 +from vllm.engine.llm_engine import LLMEngine
5 +from vllm.sampling_params import SamplingParams
6 +
7 +
8 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
9 +@pytest.mark.parametrize("block_size", [16])
10 +def test_computed_prefix_blocks(model: str, block_size: int):
11 + # This test checks if we are able to run the engine to completion
12 + # without triggering asserts.
13 + # We are in a scenario where all blocks from the second request's prompt
14 + # are full and already computed when the second request arrives.
15 + prompt = (
16 + "You are a helpful assistant. How do I build a car from cardboard and "
17 + "paper clips? Is there an easy to follow video tutorial available "
18 + "online for free?")
19 + prompt2 = (
20 + " Please recommend to me some resources where I can learn not only to "
21 + "handle technical difficulties of building a car, but also "
22 + "decoration.")
23 +
24 + engine_args = EngineArgs(model=model,
25 + block_size=block_size,
26 + enable_prefix_caching=True)
27 +
28 + engine = LLMEngine.from_engine_args(engine_args)
29 + sampling_params = SamplingParams()
30 +
31 + engine.add_request("0", prompt + prompt2, sampling_params)
32 + engine.step()
33 + engine.add_request("1", prompt, sampling_params)
34 + engine.step()
vllm/core/block_manager.py
+16 -12
@@ -1,6 +1,6 @@
1 """A block manager that manages token blocks."""
2 import enum
3 -from itertools import count
3 +from itertools import count, takewhile
4 from os.path import commonprefix
5 from typing import Dict, List, Optional, Set, Tuple
6
@@ -426,23 +426,29 @@ class BlockSpaceManager:
426 for block in block_table:
427 block.last_accessed = access_time
428
429 - def compute_last_full_block_in_seq(self, seq: Sequence):
429 + def compute_full_blocks_in_seq(self, seq: Sequence):
430 if seq.seq_id not in self.block_tables:
431 return
432 max_full_block = seq.get_len() // self.block_size - 1
433 block_table = self.block_tables[seq.seq_id]
434 if max_full_block == -1:
435 return
436 - block_table[max_full_block].computed = True
436 + for i in reversed(range(max_full_block)):
437 + if block_table[i].computed:
438 + break
439 + block_table[i].computed = True
440
438 - def get_all_block_ids_till_computed(self, seq: Sequence) -> List[int]:
441 + def get_all_computed_blocks(self, seq: Sequence) -> List[int]:
442 if seq.seq_id not in self.block_tables:
443 return []
444 block_table = self.block_tables[seq.seq_id]
442 - for block_idx in reversed(range(len(block_table))):
443 - if block_table[block_idx].computed:
444 - return [b.block_number for b in block_table[:block_idx + 1]]
445 - return []
445 + # NOTE We exclude the last block to avoid the case where the entire
446 + # prompt is cached. This would cause erroneous behavior in model
447 + # runner.
448 + return [
449 + b.block_number
450 + for b in takewhile(lambda b: b.computed, block_table[:-1])
451 + ]
452
453 def get_common_computed_block_ids(self,
454 seq_group: SequenceGroup) -> List[int]:
@@ -451,14 +457,12 @@ class BlockSpaceManager:
457 return []
458
459 ids_list = [
454 - self.get_all_block_ids_till_computed(seq)
460 + self.get_all_computed_blocks(seq)
461 for seq in iter(seq_group.seqs_dict.values())
462 ]
463 return commonprefix([ids for ids in ids_list if ids != []])
464
465 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
460 - # NOTE: We only mark the last full block because with prefix caching,
461 - # all blocks until the marked one are guaranteed to be computed.
466 if self.enable_caching:
467 for seq in seq_group.seqs_dict.values():
464 - self.compute_last_full_block_in_seq(seq)
468 + self.compute_full_blocks_in_seq(seq)
vllm/worker/model_runner.py
+1
@@ -215,6 +215,7 @@ class ModelRunner:
215 slot_mapping[-1].append(slot)
216
217 max_prompt_len = max(subquery_lens)
218 + assert max_prompt_len > 0
219 input_tokens = _make_tensor_with_pad(input_tokens,
220 max_prompt_len,
221 pad=0,