A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[FIX] Fix styles in automatic prefix caching & add a automatic prefix caching benchmark (#3158)
Zhuohan Li committed
Mar 3, 2024 at 14:37 UTC
996d095c541e1cd67f0a7ec2579bc3bb0a435494
4 files changed
+69
-18
benchmarks/benchmark_prefix_caching.py
new
+59
@@ -0,0 +1,59 @@
1
+import argparse
2
+import time
3
+
4
+from vllm import LLM
5
+from vllm import SamplingParams
6
+
7
+PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n"
8
+
9
+
10
+def test_prefix(llm=None, sampling_params=None, prompts=None, prefix_len=None):
11
+ start_time = time.time()
12
+ # whether use Prefix
13
+ if prefix_len != None:
14
+ # start inference
15
+ llm.generate(prompts,
16
+ sampling_params=sampling_params,
17
+ prefix_pos=prefix_len)
18
+ else:
19
+ llm.generate(prompts, sampling_params=sampling_params)
20
+
21
+ end_time = time.time()
22
+ print(f"cost time {end_time - start_time}")
23
+
24
+
25
+def main(args):
26
+ llm = LLM(model="baichuan-inc/Baichuan2-13B-Chat",
27
+ tokenizer_mode='auto',
28
+ trust_remote_code=True,
29
+ enforce_eager=True,
30
+ enable_prefix_caching=args.enable_prefix_caching)
31
+
32
+ num_prompts = 100
33
+ prompts = [PROMPT] * num_prompts
34
+ sampling_params = SamplingParams(temperature=0, max_tokens=100)
35
+
36
+ print("------warm up------")
37
+ test_prefix(
38
+ llm=llm,
39
+ prompts=prompts[:1],
40
+ sampling_params=sampling_params,
41
+ )
42
+
43
+ print("------start generating------")
44
+ test_prefix(
45
+ llm=llm,
46
+ prompts=prompts,
47
+ sampling_params=sampling_params,
48
+ )
49
+
50
+
51
+if __name__ == "__main__":
52
+ parser = argparse.ArgumentParser(
53
+ description='Benchmark the performance with or without automatic '
54
+ 'prefix caching.')
55
+ parser.add_argument('--enable-prefix-caching',
56
+ action='store_true',
57
+ help='enable prefix caching')
58
+ args = parser.parse_args()
59
+ main(args)
benchmarks/benchmark_throughput.py
+4
-1
@@ -303,7 +303,10 @@ if __name__ == "__main__":
303
default="cuda",
304
choices=["cuda"],
305
help='device type for vLLM execution, supporting CUDA only currently.')
306
- parser.add_argument("--enable_prefix_caching", action='store_true')
306
+ parser.add_argument(
307
+ "--enable-prefix-caching",
308
+ action='store_true',
309
+ help="enable automatic prefix caching for vLLM backend.")
310
args = parser.parse_args()
311
if args.tokenizer is None:
312
args.tokenizer = args.model
vllm/core/block_manager.py
+4
-11
@@ -236,13 +236,6 @@ class BlockSpaceManager:
236
token_ids_len = len(seq.data.get_token_ids())
237
return token_ids_len > 0 and token_ids_len % seq.block_size == 0
238
239
- def _is_last_block(
240
- self,
241
- seq: Sequence,
242
- index: int,
243
- ) -> bool:
244
- return index == len(seq.logical_token_blocks) - 1
245
-
239
def _maybe_promote_last_block(
240
self,
241
seq: Sequence,
@@ -436,7 +429,7 @@ class BlockSpaceManager:
429
def compute_last_full_block_in_seq(self, seq: Sequence):
430
if seq.seq_id not in self.block_tables:
431
return
439
- max_full_block = seq.get_len() // seq.block_size - 1
432
+ max_full_block = seq.get_len() // self.block_size - 1
433
block_table = self.block_tables[seq.seq_id]
434
if max_full_block == -1:
435
return
@@ -451,9 +444,9 @@ class BlockSpaceManager:
444
return [b.block_number for b in block_table[:block_idx + 1]]
445
return []
446
454
- # Can return non-empty result only with prefix caching enabled.
447
def get_common_computed_block_ids(self,
448
seq_group: SequenceGroup) -> List[int]:
449
+ # Can return non-empty result only with prefix caching enabled.
450
if not self.enable_caching:
451
return []
452
@@ -463,9 +456,9 @@ class BlockSpaceManager:
456
]
457
return commonprefix([ids for ids in ids_list if ids != []])
458
466
- # We only mark the last full block because with prefix caching,
467
- # all blocks until the marked one are guaranteed to be computed.
459
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
460
+ # NOTE: We only mark the last full block because with prefix caching,
461
+ # all blocks until the marked one are guaranteed to be computed.
462
if self.enable_caching:
463
for seq in seq_group.seqs_dict.values():
464
self.compute_last_full_block_in_seq(seq)
vllm/sequence.py
+2
-6
@@ -160,10 +160,10 @@ class Sequence:
160
def lora_int_id(self) -> int:
161
return self.lora_request.lora_int_id if self.lora_request else 0
162
163
- # TODO The current hashing function is O(L^2). We should optimize this in
164
- # the future.
163
def hash_of_block(self, logical_idx: int) -> int:
164
# Compute the number of tokens in the sequence
165
+ # TODO: The current hashing function is O(L^2). We should optimize
166
+ # this in the future.
167
num_tokens = self.num_hashed_tokens_of_block(logical_idx)
168
return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
169
@@ -308,10 +308,6 @@ class SequenceGroup:
308
# We use the prompt of an arbitrary sequence.
309
return next(iter(self.seqs_dict.values())).data.prompt_token_ids
310
311
- @property
312
- def block_size(self) -> int:
313
- return next(iter(self.seqs_dict.values())).block_size
314
-
311
@property
312
def lora_int_id(self) -> int:
313
return self.lora_request.lora_int_id if self.lora_request else 0