A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add throughput benchmarking script (#133)
Woosuk Kwon committed
May 28, 2023 at 03:20 UTC
211318d44ad3225eaba488150f797efceab6cc32
12 files changed
+145
-257
benchmark/benchmark_attention.py
deleted
-165
@@ -1,165 +0,0 @@
1
-import functools
2
-import random
3
-import time
4
-from typing import List
5
-
6
-from flash_attn.flash_attn_interface import _flash_attn_forward
7
-import torch
8
-
9
-from cacheflow import attention_ops
10
-
11
-
12
-def benchmark(name, f, num_warmup = 10, num_iters = 100):
13
- for _ in range(num_warmup):
14
- f()
15
- torch.cuda.synchronize()
16
-
17
- start = time.time()
18
- for _ in range(num_iters):
19
- f()
20
- torch.cuda.synchronize()
21
- end = time.time()
22
- print(f'{name}: {(end - start) / num_iters * 1000:.3f} ms')
23
-
24
-
25
-@torch.inference_mode()
26
-def benchmark_multi_query_cached_kv_attention(
27
- query_lens: List[int],
28
- context_lens: List[int],
29
- num_heads: int,
30
- head_size: int,
31
- block_size: int,
32
- num_blocks: int,
33
- dtype: torch.dtype,
34
-) -> None:
35
- print(f'query_lens: {query_lens}, context_lens: {context_lens}, '
36
- f'num_heads: {num_heads}, head_size: {head_size}, block_size: '
37
- f'{block_size}, num_blocks: {num_blocks}, dtype: {dtype}')
38
- # Create query tensor.
39
- num_queries = len(query_lens)
40
- cu_query_lens = [0]
41
- for query_len in query_lens:
42
- cu_query_lens.append(cu_query_lens[-1] + query_len)
43
- num_total_tokens = cu_query_lens[-1]
44
- qkv = torch.randn(
45
- num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
46
- query, _, _ = qkv.unbind(dim=1)
47
-
48
- # Create key and value cache.
49
- x = 16 // torch.tensor([], dtype=dtype).element_size()
50
- key_block_shape = (num_heads, head_size // x, block_size, x)
51
- key_cache = torch.randn(
52
- size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
53
- value_block_shape = (num_heads, head_size, block_size)
54
- value_cache = torch.randn(
55
- size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
56
-
57
- # Create block tables.
58
- max_context_len = max(context_lens)
59
- max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
60
- block_tables = []
61
- for _ in range(num_queries):
62
- block_table = [
63
- random.randint(0, num_blocks - 1)
64
- for _ in range(max_num_blocks_per_seq)
65
- ]
66
- block_tables.append(block_table)
67
- block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
68
-
69
- # Create input and output data structures.
70
- cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
71
- context_len_tensor = torch.tensor(context_lens, dtype=torch.int, device='cuda')
72
- scale = float(1.0 / (head_size ** 0.5))
73
- output = torch.empty(
74
- num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
75
-
76
- # Run our implementation.
77
- def run_ours():
78
- attention_ops.multi_query_cached_kv_attention(
79
- cu_query_lens,
80
- output,
81
- query,
82
- key_cache,
83
- value_cache,
84
- scale,
85
- block_tables,
86
- context_len_tensor,
87
- block_size,
88
- max_context_len,
89
- )
90
- benchmark('Ours', run_ours)
91
-
92
- # Upper bound: Flash attention.
93
- # Becuase Flash attention cannot read our own cache,
94
- # we make key and value tensors contiguous.
95
- num_kv_tokens = sum(context_lens)
96
- cu_context_lens = [0]
97
- for context_len in context_lens:
98
- cu_context_lens.append(cu_context_lens[-1] + context_len)
99
- cu_context_lens = torch.tensor(cu_context_lens, dtype=torch.int, device='cuda')
100
- qkv = torch.randn(
101
- num_kv_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
102
- _, key, value = qkv.unbind(dim=1)
103
- ref_output = torch.empty_like(output)
104
-
105
- # Run Flash attention.
106
- def run_flash_attn():
107
- _flash_attn_forward(
108
- query,
109
- key,
110
- value,
111
- ref_output,
112
- cu_query_lens,
113
- cu_context_lens,
114
- max(query_lens),
115
- max_context_len,
116
- dropout_p=0.0,
117
- softmax_scale=scale,
118
- causal=True,
119
- return_softmax=False,
120
- )
121
- benchmark('Flash attention', run_flash_attn)
122
-
123
-
124
-if __name__ == '__main__':
125
- BLOCK_SIZE = 8
126
- NUM_BLOCKS = 1024
127
- DTYPE = torch.half
128
-
129
- # LLaMA-13B and OPT-13B
130
- NUM_HEADS = 40
131
- HEAD_SIZE = 128
132
-
133
- run_benchmark = functools.partial(
134
- benchmark_multi_query_cached_kv_attention,
135
- num_heads=NUM_HEADS,
136
- head_size=HEAD_SIZE,
137
- block_size=BLOCK_SIZE,
138
- num_blocks=NUM_BLOCKS,
139
- dtype=DTYPE,
140
- )
141
-
142
- run_benchmark(
143
- query_lens=[64] * 1,
144
- context_lens=[64] * 1,
145
- )
146
- run_benchmark(
147
- query_lens=[128] * 1,
148
- context_lens=[128] * 1,
149
- )
150
- run_benchmark(
151
- query_lens=[64] * 8,
152
- context_lens=[64] * 8,
153
- )
154
- run_benchmark(
155
- query_lens=[128] * 8,
156
- context_lens=[128] * 8,
157
- )
158
- run_benchmark(
159
- query_lens=[64, 32, 16],
160
- context_lens=[128, 256, 64],
161
- )
162
- run_benchmark(
163
- query_lens=[1024],
164
- context_lens=[1024],
165
- )
benchmark/benchmark_cache.py
deleted
-81
@@ -1,81 +0,0 @@
1
-import functools
2
-import random
3
-import time
4
-
5
-import torch
6
-
7
-from cacheflow import cache_ops
8
-
9
-
10
-def benchmark(name, f, size: int, num_warmup = 10, num_iters = 100):
11
- for _ in range(num_warmup):
12
- f()
13
- torch.cuda.synchronize()
14
-
15
- start = time.time()
16
- for _ in range(num_iters):
17
- f()
18
- torch.cuda.synchronize()
19
- end = time.time()
20
- avg_time = (end - start) / num_iters
21
- print(f'[Latency] {name}: {avg_time * 1000:.3f} ms')
22
- print(f'[Throughput] {name}: {size / avg_time / 2 ** 30:.3f} GB/s')
23
-
24
-
25
-@torch.inference_mode()
26
-def test_gather_cached_kv(
27
- num_tokens: int,
28
- num_heads: int,
29
- head_size: int,
30
- block_size: int,
31
- num_blocks: int,
32
- dtype: torch.dtype,
33
-) -> None:
34
- print(f'num_tokens: {num_tokens}, num_heads: {num_heads}, '
35
- f'head_size: {head_size}, block_size: {block_size}, '
36
- f'num_blocks: {num_blocks}, dtype: {dtype}')
37
-
38
- num_slots = block_size * num_blocks
39
- slot_mapping = random.sample(range(num_slots), num_tokens)
40
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
41
-
42
- qkv = torch.randn(
43
- num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
44
- _, key, value = qkv.unbind(dim=1)
45
-
46
- x = 16 // torch.tensor([], dtype=dtype).element_size()
47
- key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
48
- key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
49
-
50
- value_cache_shape = (num_blocks, num_heads, head_size, block_size)
51
- value_cache = torch.randn(
52
- size=value_cache_shape, dtype=dtype, device='cuda')
53
-
54
- # Run Flash attention.
55
- def run():
56
- cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
57
-
58
- benchmark('gather_cached_kv', run,
59
- size=num_tokens * num_heads * head_size * 2 * qkv.element_size())
60
-
61
-
62
-if __name__ == '__main__':
63
- BLOCK_SIZE = 8
64
- NUM_BLOCKS = 1024
65
- DTYPE = torch.half
66
-
67
- # LLaMA-13B and OPT-13B
68
- NUM_HEADS = 40
69
- HEAD_SIZE = 128
70
-
71
- run_benchmark = functools.partial(
72
- test_gather_cached_kv,
73
- num_heads=NUM_HEADS,
74
- head_size=HEAD_SIZE,
75
- block_size=BLOCK_SIZE,
76
- num_blocks=NUM_BLOCKS,
77
- dtype=DTYPE,
78
- )
79
-
80
- for i in range(6, 12):
81
- run_benchmark(num_tokens=2 ** i)
benchmarks/README.md
new
+8
@@ -0,0 +1,8 @@
1
+# Benchmarking CacheFlow
2
+
3
+## Downloading the ShareGPT dataset
4
+
5
+You can download the dataset by running:
6
+```bash
7
+wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
8
+```
benchmarks/benchmark_latency.py
renamed
benchmarks/benchmark_text_completion.py
renamed
benchmarks/benchmark_throughput.py
new
+104
@@ -0,0 +1,104 @@
1
+import argparse
2
+import json
3
+import random
4
+import time
5
+from typing import List, Tuple
6
+
7
+from cacheflow import LLM, SamplingParams
8
+from transformers import PreTrainedTokenizerBase
9
+
10
+
11
+def sample_requests(
12
+ dataset_path: str,
13
+ num_requests: int,
14
+ tokenizer: PreTrainedTokenizerBase,
15
+) -> List[Tuple[List[int], int]]:
16
+ # Load the dataset.
17
+ with open(dataset_path) as f:
18
+ dataset = json.load(f)
19
+ # Filter out the conversations with less than 2 turns.
20
+ dataset = [
21
+ data for data in dataset
22
+ if len(data["conversations"]) >= 2
23
+ ]
24
+ # Only keep the first two turns of each conversation.
25
+ dataset = [
26
+ (data["conversations"][0]["value"], data["conversations"][1]["value"])
27
+ for data in dataset
28
+ ]
29
+
30
+ # Tokenize the prompts and completions.
31
+ prompts = [prompt for prompt, _ in dataset]
32
+ prompt_token_ids = tokenizer(prompts).input_ids
33
+ completions = [completion for _, completion in dataset]
34
+ completion_token_ids = tokenizer(completions).input_ids
35
+ tokenized_dataset = []
36
+ for i in range(len(dataset)):
37
+ output_len = len(completion_token_ids[i])
38
+ tokenized_dataset.append((prompt_token_ids[i], output_len))
39
+ # Filter out if the prompt length + output length is greater than 2048.
40
+ tokenized_dataset = [
41
+ (prompt_token_ids, output_len)
42
+ for prompt_token_ids, output_len in tokenized_dataset
43
+ if len(prompt_token_ids) + output_len <= 2048
44
+ ]
45
+
46
+ # Sample the requests.
47
+ sampled_requests = random.sample(tokenized_dataset, num_requests)
48
+ return sampled_requests
49
+
50
+
51
+def main(args: argparse.Namespace):
52
+ print(args)
53
+ random.seed(args.seed)
54
+
55
+ llm = LLM(
56
+ model=args.model,
57
+ tensor_parallel_size=args.tensor_parallel_size,
58
+ seed=args.seed,
59
+ )
60
+ tokenizer = llm.get_tokenizer()
61
+ requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
62
+
63
+ # Add the requests to the server.
64
+ for prompt_token_ids, output_len in requests:
65
+ sampling_params = SamplingParams(
66
+ n=args.n,
67
+ temperature=0.0 if args.use_beam_search else 1.0,
68
+ top_p=1.0,
69
+ use_beam_search=args.use_beam_search,
70
+ ignore_eos=True,
71
+ max_tokens=output_len,
72
+ )
73
+ # FIXME(woosuk): Do not use internal method.
74
+ llm._add_request(
75
+ prompt="",
76
+ sampling_params=sampling_params,
77
+ prompt_token_ids=prompt_token_ids,
78
+ )
79
+
80
+ start = time.time()
81
+ # FIXME(woosuk): Do use internal method.
82
+ llm._run_server(use_tqdm=True)
83
+ end = time.time()
84
+ total_num_tokens = sum(
85
+ len(prompt_token_ids) + output_len
86
+ for prompt_token_ids, output_len in requests
87
+ )
88
+ print(f"Throughput: {total_num_tokens / (end - start):.2f} tokens/s")
89
+
90
+
91
+if __name__ == "__main__":
92
+ parser = argparse.ArgumentParser(description="Benchmark the throughput.")
93
+ parser.add_argument("--dataset", type=str, required=True,
94
+ help="Path to the dataset.")
95
+ parser.add_argument("--model", type=str, default="facebook/opt-125m")
96
+ parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
97
+ parser.add_argument("--n", type=int, default=1,
98
+ help="Number of generated sequences per prompt.")
99
+ parser.add_argument("--use-beam-search", action="store_true")
100
+ parser.add_argument("--num-prompts", type=int, default=1000,
101
+ help="Number of prompts to process.")
102
+ parser.add_argument("--seed", type=int, default=0)
103
+ args = parser.parse_args()
104
+ main(args)
benchmarks/trace.py
renamed
cacheflow/__init__.py
+2
-1
@@ -1,5 +1,5 @@
1
from cacheflow.entrypoints.llm import LLM
2
-from cacheflow.outputs import RequestOutput
2
+from cacheflow.outputs import RequestOutput, CompletionOutput
3
from cacheflow.sampling_params import SamplingParams
4
from cacheflow.server.arg_utils import ServerArgs
5
from cacheflow.server.llm_server import LLMServer
@@ -9,6 +9,7 @@ __all__ = [
9
"LLM",
10
"SamplingParams",
11
"RequestOutput",
12
+ "CompletionOutput",
13
"LLMServer",
14
"ServerArgs",
15
"initialize_cluster",
cacheflow/core/scheduler.py
+3
@@ -87,6 +87,9 @@ class Scheduler:
87
def has_unfinished_seqs(self) -> bool:
88
return self.waiting or self.running or self.swapped
89
90
+ def get_num_unfinished_seq_groups(self) -> int:
91
+ return len(self.waiting) + len(self.running) + len(self.swapped)
92
+
93
def _schedule(self) -> Tuple[SchedulerOutputs, List[str]]:
94
# Blocks that need to be swaped or copied before model execution.
95
blocks_to_swap_in: Dict[int, int] = {}
cacheflow/entrypoints/llm.py
+24
-8
@@ -1,5 +1,6 @@
1
-from typing import List, Optional
1
+from typing import List, Optional, Union
2
3
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4
from tqdm import tqdm
5
6
from cacheflow.outputs import RequestOutput
@@ -31,6 +32,11 @@ class LLM:
32
self.llm_server = LLMServer.from_server_args(server_args)
33
self.request_counter = Counter()
34
35
+ def get_tokenizer(
36
+ self,
37
+ ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
38
+ return self.llm_server.tokenizer
39
+
40
def generate(
41
self,
42
prompts: List[str],
@@ -41,10 +47,6 @@ class LLM:
47
if sampling_params is None:
48
# Use default sampling params.
49
sampling_params = SamplingParams()
44
- # Initialize tqdm.
45
- if use_tqdm:
46
- pbar = tqdm(total=len(prompts), desc="Processed prompts")
47
-
50
# Add requests to the server.
51
for i in range(len(prompts)):
52
prompt = prompts[i]
@@ -52,10 +54,24 @@ class LLM:
54
token_ids = None
55
else:
56
token_ids = prompt_token_ids[i]
55
- request_id = str(next(self.request_counter))
56
- self.llm_server.add_request(request_id, prompt, sampling_params,
57
- token_ids)
57
+ self._add_request(prompt, sampling_params, token_ids)
58
+ return self._run_server(use_tqdm)
59
60
+ def _add_request(
61
+ self,
62
+ prompt: str,
63
+ sampling_params: SamplingParams,
64
+ prompt_token_ids: Optional[List[int]],
65
+ ) -> None:
66
+ request_id = str(next(self.request_counter))
67
+ self.llm_server.add_request(request_id, prompt, sampling_params,
68
+ prompt_token_ids)
69
+
70
+ def _run_server(self, use_tqdm: bool) -> List[RequestOutput]:
71
+ # Initialize tqdm.
72
+ if use_tqdm:
73
+ num_requests = self.llm_server.get_num_unfinished_requests()
74
+ pbar = tqdm(total=num_requests, desc="Processed prompts")
75
# Run the server.
76
outputs: List[RequestOutput] = []
77
while self.llm_server.has_unfinished_requests():
cacheflow/server/llm_server.py
+3
@@ -151,6 +151,9 @@ class LLMServer:
151
# Add the sequence group to the scheduler.
152
self.scheduler.add_seq_group(seq_group)
153
154
+ def get_num_unfinished_requests(self) -> int:
155
+ return self.scheduler.get_num_unfinished_seq_groups()
156
+
157
def has_unfinished_requests(self) -> bool:
158
return self.scheduler.has_unfinished_seqs()
159
examples/simple_server.py
+1
-2
@@ -19,9 +19,8 @@ def main(args: argparse.Namespace):
19
SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20
]
21
22
- request_id = 0
23
-
22
# Run the server.
23
+ request_id = 0
24
while True:
25
# To test iteration-level scheduling, we add one request at each step.
26
if test_prompts: