@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 996d095c54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[FIX] Fix styles in automatic prefix caching & add a automatic prefix caching benchmark (#3158)

Zhuohan Li committed Mar 3, 2024 at 14:37 UTC 996d095c541e1cd67f0a7ec2579bc3bb0a435494
4 files changed +69 -18
benchmarks/benchmark_prefix_caching.py new
+59
@@ -0,0 +1,59 @@
1 +import argparse
2 +import time
3 +
4 +from vllm import LLM
5 +from vllm import SamplingParams
6 +
7 +PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n"
8 +
9 +
10 +def test_prefix(llm=None, sampling_params=None, prompts=None, prefix_len=None):
11 + start_time = time.time()
12 + # whether use Prefix
13 + if prefix_len != None:
14 + # start inference
15 + llm.generate(prompts,
16 + sampling_params=sampling_params,
17 + prefix_pos=prefix_len)
18 + else:
19 + llm.generate(prompts, sampling_params=sampling_params)
20 +
21 + end_time = time.time()
22 + print(f"cost time {end_time - start_time}")
23 +
24 +
25 +def main(args):
26 + llm = LLM(model="baichuan-inc/Baichuan2-13B-Chat",
27 + tokenizer_mode='auto',
28 + trust_remote_code=True,
29 + enforce_eager=True,
30 + enable_prefix_caching=args.enable_prefix_caching)
31 +
32 + num_prompts = 100
33 + prompts = [PROMPT] * num_prompts
34 + sampling_params = SamplingParams(temperature=0, max_tokens=100)
35 +
36 + print("------warm up------")
37 + test_prefix(
38 + llm=llm,
39 + prompts=prompts[:1],
40 + sampling_params=sampling_params,
41 + )
42 +
43 + print("------start generating------")
44 + test_prefix(
45 + llm=llm,
46 + prompts=prompts,
47 + sampling_params=sampling_params,
48 + )
49 +
50 +
51 +if __name__ == "__main__":
52 + parser = argparse.ArgumentParser(
53 + description='Benchmark the performance with or without automatic '
54 + 'prefix caching.')
55 + parser.add_argument('--enable-prefix-caching',
56 + action='store_true',
57 + help='enable prefix caching')
58 + args = parser.parse_args()
59 + main(args)
benchmarks/benchmark_throughput.py
+4 -1
@@ -303,7 +303,10 @@ if __name__ == "__main__":
303 default="cuda",
304 choices=["cuda"],
305 help='device type for vLLM execution, supporting CUDA only currently.')
306 - parser.add_argument("--enable_prefix_caching", action='store_true')
306 + parser.add_argument(
307 + "--enable-prefix-caching",
308 + action='store_true',
309 + help="enable automatic prefix caching for vLLM backend.")
310 args = parser.parse_args()
311 if args.tokenizer is None:
312 args.tokenizer = args.model
vllm/core/block_manager.py
+4 -11
@@ -236,13 +236,6 @@ class BlockSpaceManager:
236 token_ids_len = len(seq.data.get_token_ids())
237 return token_ids_len > 0 and token_ids_len % seq.block_size == 0
238
239 - def _is_last_block(
240 - self,
241 - seq: Sequence,
242 - index: int,
243 - ) -> bool:
244 - return index == len(seq.logical_token_blocks) - 1
245 -
239 def _maybe_promote_last_block(
240 self,
241 seq: Sequence,
@@ -436,7 +429,7 @@ class BlockSpaceManager:
429 def compute_last_full_block_in_seq(self, seq: Sequence):
430 if seq.seq_id not in self.block_tables:
431 return
439 - max_full_block = seq.get_len() // seq.block_size - 1
432 + max_full_block = seq.get_len() // self.block_size - 1
433 block_table = self.block_tables[seq.seq_id]
434 if max_full_block == -1:
435 return
@@ -451,9 +444,9 @@ class BlockSpaceManager:
444 return [b.block_number for b in block_table[:block_idx + 1]]
445 return []
446
454 - # Can return non-empty result only with prefix caching enabled.
447 def get_common_computed_block_ids(self,
448 seq_group: SequenceGroup) -> List[int]:
449 + # Can return non-empty result only with prefix caching enabled.
450 if not self.enable_caching:
451 return []
452
@@ -463,9 +456,9 @@ class BlockSpaceManager:
456 ]
457 return commonprefix([ids for ids in ids_list if ids != []])
458
466 - # We only mark the last full block because with prefix caching,
467 - # all blocks until the marked one are guaranteed to be computed.
459 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
460 + # NOTE: We only mark the last full block because with prefix caching,
461 + # all blocks until the marked one are guaranteed to be computed.
462 if self.enable_caching:
463 for seq in seq_group.seqs_dict.values():
464 self.compute_last_full_block_in_seq(seq)
vllm/sequence.py
+2 -6
@@ -160,10 +160,10 @@ class Sequence:
160 def lora_int_id(self) -> int:
161 return self.lora_request.lora_int_id if self.lora_request else 0
162
163 - # TODO The current hashing function is O(L^2). We should optimize this in
164 - # the future.
163 def hash_of_block(self, logical_idx: int) -> int:
164 # Compute the number of tokens in the sequence
165 + # TODO: The current hashing function is O(L^2). We should optimize
166 + # this in the future.
167 num_tokens = self.num_hashed_tokens_of_block(logical_idx)
168 return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
169
@@ -308,10 +308,6 @@ class SequenceGroup:
308 # We use the prompt of an arbitrary sequence.
309 return next(iter(self.seqs_dict.values())).data.prompt_token_ids
310
311 - @property
312 - def block_size(self) -> int:
313 - return next(iter(self.seqs_dict.values())).block_size
314 -
311 @property
312 def lora_int_id(self) -> int:
313 return self.lora_request.lora_int_id if self.lora_request else 0