@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8cbba4622c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Possible fix for conflict between Automated Prefix Caching (#2762) and multi-LoRA support (#1804) (#3263)

jacobthebanana committed Mar 7, 2024 at 18:03 UTC 8cbba4622c8c526b207b17e3ba51e18e2c766419
2 files changed +33 -16
tests/test_cache_block_hashing.py
+31 -15
@@ -2,8 +2,11 @@
2
3 Run `pytest tests/test_cache_block_hashing.py`.
4 """
5 +from typing import List, Optional
6 +
7 import pytest
8
9 +from vllm.lora.request import LoRARequest
10 from vllm.transformers_utils.tokenizer import TokenizerGroup
11 from vllm.sequence import Sequence
12
@@ -36,7 +39,10 @@ def flatten_2d(li):
39 @pytest.mark.parametrize("model", ["facebook/opt-125m"])
40 @pytest.mark.parametrize("block_size", [16])
41 @pytest.mark.parametrize("max_num_seqs", [256])
39 -def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int):
42 +@pytest.mark.parametrize("concurrent_lora_int_ids",
43 + [[None], [1], [None, 1], [None, 1, 2], [1, 2]])
44 +def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
45 + concurrent_lora_int_ids: List[Optional[int]]):
46
47 tokenizer = TokenizerGroup(
48 tokenizer_id="facebook/opt-125m",
@@ -48,20 +54,30 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int):
54 hashes = []
55
56 for prefix in prefixes:
51 - hashes.append([])
52 - prompts = [prefix + prompt for prompt in sample_prompts]
53 - seq_id = 0
54 - for prompt in prompts:
55 - hashes[-1].append([])
56 - prompt_token_ids = tokenizer.encode(prompt)
57 - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
58 - tokenizer.tokenizer.eos_token_id)
59 -
60 - num_blocks = len(prompt_token_ids) // block_size
61 - for idx in range(num_blocks):
62 - hashes[-1][-1].append(seq.hash_of_block(idx))
63 -
64 - seq_id += 1
57 + for lora_int_id in concurrent_lora_int_ids:
58 + lora_request = None
59 +
60 + if lora_int_id is not None:
61 + lora_request = LoRARequest(
62 + f"example_lora_{lora_int_id}",
63 + lora_int_id,
64 + f"example/path/to/lora_{lora_int_id}",
65 + )
66 +
67 + hashes.append([])
68 + prompts = [prefix + prompt for prompt in sample_prompts]
69 + seq_id = 0
70 + for prompt in prompts:
71 + hashes[-1].append([])
72 + prompt_token_ids = tokenizer.encode(prompt)
73 + seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
74 + tokenizer.tokenizer.eos_token_id, lora_request)
75 +
76 + num_blocks = len(prompt_token_ids) // block_size
77 + for idx in range(num_blocks):
78 + hashes[-1][-1].append(seq.hash_of_block(idx))
79 +
80 + seq_id += 1
81
82 # Check that hashes made with two prefixes with different first blocks are
83 # different everywhere.
vllm/sequence.py
+2 -1
@@ -175,7 +175,8 @@ class Sequence:
175 # TODO: The current hashing function is O(L^2). We should optimize
176 # this in the future.
177 num_tokens = self.num_hashed_tokens_of_block(logical_idx)
178 - return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
178 + return hash(
179 + (tuple(self.data.get_token_ids()[0:num_tokens]), self.lora_int_id))
180
181 def num_hashed_tokens_of_block(self, logical_idx: int):
182 return logical_idx * self.block_size + self.block_size