@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ccb63a8245
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Hash][Automatic Prefix caching] Accelerating the hashing function by avoiding deep copies (#4696)

Kuntai Du committed May 14, 2024 at 05:34 UTC ccb63a8245bceb9e6ba260eeef41b54ca8bdb370
2 files changed +77 -2
benchmarks/overheads/benchmark_hashing.py new
+63
@@ -0,0 +1,63 @@
1 +import argparse
2 +import cProfile
3 +import pstats
4 +
5 +from vllm import LLM, SamplingParams
6 +
7 +# A very long prompt, total number of tokens is about 15k.
8 +LONG_PROMPT = ["You are an expert in large language models, aren't you?"
9 + ] * 1000
10 +LONG_PROMPT = ' '.join(LONG_PROMPT)
11 +
12 +
13 +def main(args):
14 + llm = LLM(
15 + model=args.model,
16 + enforce_eager=True,
17 + enable_prefix_caching=True,
18 + tensor_parallel_size=args.tensor_parallel_size,
19 + use_v2_block_manager=args.use_v2_block_manager,
20 + )
21 +
22 + sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
23 + profiler = cProfile.Profile()
24 +
25 + print("------warm up------")
26 + for i in range(3):
27 + output = llm.generate(LONG_PROMPT, sampling_params)
28 + print(output[0].outputs[0].text)
29 +
30 + print("------start generating------")
31 + for i in range(3):
32 + profiler.runctx('llm.generate(LONG_PROMPT, sampling_params)',
33 + globals(), locals())
34 +
35 + # analyze the runtime of hashing function
36 + stats = pstats.Stats(profiler)
37 + stats.sort_stats('cumulative')
38 + total_time = 0
39 + total_calls = 0
40 + for func in stats.stats:
41 + if 'hash_of_block' in func[2]:
42 + total_time = stats.stats[func][3]
43 + total_calls = stats.stats[func][0]
44 + percentage = (total_time / stats.total_tt) * 100
45 + print(f"Hashing took {total_time:.2f} seconds,"
46 + f"{percentage:.2f}% of the total runtime.")
47 +
48 +
49 +if __name__ == "__main__":
50 + parser = argparse.ArgumentParser(
51 + description='Benchmark the performance of hashing function in'
52 + 'automatic prefix caching.')
53 + parser.add_argument('--model', type=str, default='lmsys/longchat-7b-16k')
54 + parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
55 + parser.add_argument('--output-len', type=int, default=10)
56 + parser.add_argument('--enable-prefix-caching',
57 + action='store_true',
58 + help='enable prefix caching')
59 + parser.add_argument('--use-v2-block-manager',
60 + action='store_true',
61 + help='Use BlockSpaceMangerV2')
62 + args = parser.parse_args()
63 + main(args)
vllm/sequence.py
+14 -2
@@ -121,6 +121,7 @@ class SequenceData:
121 output_token_ids = []
122
123 self.prompt_token_ids = prompt_token_ids
124 + self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
125 self.output_token_ids = output_token_ids
126 self.cumulative_logprob = 0.0
127 # The number of tokens that are computed (that run against the model).
@@ -143,6 +144,17 @@ class SequenceData:
144 def get_token_ids(self) -> List[int]:
145 return self.prompt_token_ids + self.output_token_ids
146
147 + def get_prefix_token_ids(
148 + self, num_tokens: int
149 + ) -> Tuple[Tuple[int, ...], Optional[Tuple[int, ...]]]:
150 + """Get prefix tokens, and make the return value hashable"""
151 + prompt_length = len(self.prompt_token_ids)
152 + if num_tokens > prompt_length:
153 + return (self._prompt_token_ids_tuple,
154 + tuple(self.output_token_ids[:num_tokens - prompt_length]))
155 + else:
156 + return (self._prompt_token_ids_tuple[:num_tokens], None)
157 +
158 def get_num_computed_tokens(self) -> int:
159 """Return the number of prefill tokens that are already computed."""
160 return self._num_computed_tokens
@@ -253,8 +265,8 @@ class Sequence:
265 # TODO: The current hashing function is O(L^2). We should optimize
266 # this in the future.
267 num_tokens = self.num_hashed_tokens_of_block(logical_idx)
256 - return hash(
257 - (tuple(self.data.get_token_ids()[0:num_tokens]), self.lora_int_id))
268 + hashed_tokens = self.data.get_prefix_token_ids(num_tokens)
269 + return hash((hashed_tokens, self.lora_int_id))
270
271 def num_hashed_tokens_of_block(self, logical_idx: int):
272 return logical_idx * self.block_size + self.block_size