@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 211318d44a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add throughput benchmarking script (#133)

Woosuk Kwon committed May 28, 2023 at 03:20 UTC 211318d44ad3225eaba488150f797efceab6cc32
12 files changed +145 -257
benchmark/benchmark_attention.py deleted
-165
@@ -1,165 +0,0 @@
1 -import functools
2 -import random
3 -import time
4 -from typing import List
5 -
6 -from flash_attn.flash_attn_interface import _flash_attn_forward
7 -import torch
8 -
9 -from cacheflow import attention_ops
10 -
11 -
12 -def benchmark(name, f, num_warmup = 10, num_iters = 100):
13 - for _ in range(num_warmup):
14 - f()
15 - torch.cuda.synchronize()
16 -
17 - start = time.time()
18 - for _ in range(num_iters):
19 - f()
20 - torch.cuda.synchronize()
21 - end = time.time()
22 - print(f'{name}: {(end - start) / num_iters * 1000:.3f} ms')
23 -
24 -
25 -@torch.inference_mode()
26 -def benchmark_multi_query_cached_kv_attention(
27 - query_lens: List[int],
28 - context_lens: List[int],
29 - num_heads: int,
30 - head_size: int,
31 - block_size: int,
32 - num_blocks: int,
33 - dtype: torch.dtype,
34 -) -> None:
35 - print(f'query_lens: {query_lens}, context_lens: {context_lens}, '
36 - f'num_heads: {num_heads}, head_size: {head_size}, block_size: '
37 - f'{block_size}, num_blocks: {num_blocks}, dtype: {dtype}')
38 - # Create query tensor.
39 - num_queries = len(query_lens)
40 - cu_query_lens = [0]
41 - for query_len in query_lens:
42 - cu_query_lens.append(cu_query_lens[-1] + query_len)
43 - num_total_tokens = cu_query_lens[-1]
44 - qkv = torch.randn(
45 - num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
46 - query, _, _ = qkv.unbind(dim=1)
47 -
48 - # Create key and value cache.
49 - x = 16 // torch.tensor([], dtype=dtype).element_size()
50 - key_block_shape = (num_heads, head_size // x, block_size, x)
51 - key_cache = torch.randn(
52 - size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
53 - value_block_shape = (num_heads, head_size, block_size)
54 - value_cache = torch.randn(
55 - size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
56 -
57 - # Create block tables.
58 - max_context_len = max(context_lens)
59 - max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
60 - block_tables = []
61 - for _ in range(num_queries):
62 - block_table = [
63 - random.randint(0, num_blocks - 1)
64 - for _ in range(max_num_blocks_per_seq)
65 - ]
66 - block_tables.append(block_table)
67 - block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
68 -
69 - # Create input and output data structures.
70 - cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
71 - context_len_tensor = torch.tensor(context_lens, dtype=torch.int, device='cuda')
72 - scale = float(1.0 / (head_size ** 0.5))
73 - output = torch.empty(
74 - num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
75 -
76 - # Run our implementation.
77 - def run_ours():
78 - attention_ops.multi_query_cached_kv_attention(
79 - cu_query_lens,
80 - output,
81 - query,
82 - key_cache,
83 - value_cache,
84 - scale,
85 - block_tables,
86 - context_len_tensor,
87 - block_size,
88 - max_context_len,
89 - )
90 - benchmark('Ours', run_ours)
91 -
92 - # Upper bound: Flash attention.
93 - # Becuase Flash attention cannot read our own cache,
94 - # we make key and value tensors contiguous.
95 - num_kv_tokens = sum(context_lens)
96 - cu_context_lens = [0]
97 - for context_len in context_lens:
98 - cu_context_lens.append(cu_context_lens[-1] + context_len)
99 - cu_context_lens = torch.tensor(cu_context_lens, dtype=torch.int, device='cuda')
100 - qkv = torch.randn(
101 - num_kv_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
102 - _, key, value = qkv.unbind(dim=1)
103 - ref_output = torch.empty_like(output)
104 -
105 - # Run Flash attention.
106 - def run_flash_attn():
107 - _flash_attn_forward(
108 - query,
109 - key,
110 - value,
111 - ref_output,
112 - cu_query_lens,
113 - cu_context_lens,
114 - max(query_lens),
115 - max_context_len,
116 - dropout_p=0.0,
117 - softmax_scale=scale,
118 - causal=True,
119 - return_softmax=False,
120 - )
121 - benchmark('Flash attention', run_flash_attn)
122 -
123 -
124 -if __name__ == '__main__':
125 - BLOCK_SIZE = 8
126 - NUM_BLOCKS = 1024
127 - DTYPE = torch.half
128 -
129 - # LLaMA-13B and OPT-13B
130 - NUM_HEADS = 40
131 - HEAD_SIZE = 128
132 -
133 - run_benchmark = functools.partial(
134 - benchmark_multi_query_cached_kv_attention,
135 - num_heads=NUM_HEADS,
136 - head_size=HEAD_SIZE,
137 - block_size=BLOCK_SIZE,
138 - num_blocks=NUM_BLOCKS,
139 - dtype=DTYPE,
140 - )
141 -
142 - run_benchmark(
143 - query_lens=[64] * 1,
144 - context_lens=[64] * 1,
145 - )
146 - run_benchmark(
147 - query_lens=[128] * 1,
148 - context_lens=[128] * 1,
149 - )
150 - run_benchmark(
151 - query_lens=[64] * 8,
152 - context_lens=[64] * 8,
153 - )
154 - run_benchmark(
155 - query_lens=[128] * 8,
156 - context_lens=[128] * 8,
157 - )
158 - run_benchmark(
159 - query_lens=[64, 32, 16],
160 - context_lens=[128, 256, 64],
161 - )
162 - run_benchmark(
163 - query_lens=[1024],
164 - context_lens=[1024],
165 - )
benchmark/benchmark_cache.py deleted
-81
@@ -1,81 +0,0 @@
1 -import functools
2 -import random
3 -import time
4 -
5 -import torch
6 -
7 -from cacheflow import cache_ops
8 -
9 -
10 -def benchmark(name, f, size: int, num_warmup = 10, num_iters = 100):
11 - for _ in range(num_warmup):
12 - f()
13 - torch.cuda.synchronize()
14 -
15 - start = time.time()
16 - for _ in range(num_iters):
17 - f()
18 - torch.cuda.synchronize()
19 - end = time.time()
20 - avg_time = (end - start) / num_iters
21 - print(f'[Latency] {name}: {avg_time * 1000:.3f} ms')
22 - print(f'[Throughput] {name}: {size / avg_time / 2 ** 30:.3f} GB/s')
23 -
24 -
25 -@torch.inference_mode()
26 -def test_gather_cached_kv(
27 - num_tokens: int,
28 - num_heads: int,
29 - head_size: int,
30 - block_size: int,
31 - num_blocks: int,
32 - dtype: torch.dtype,
33 -) -> None:
34 - print(f'num_tokens: {num_tokens}, num_heads: {num_heads}, '
35 - f'head_size: {head_size}, block_size: {block_size}, '
36 - f'num_blocks: {num_blocks}, dtype: {dtype}')
37 -
38 - num_slots = block_size * num_blocks
39 - slot_mapping = random.sample(range(num_slots), num_tokens)
40 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
41 -
42 - qkv = torch.randn(
43 - num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
44 - _, key, value = qkv.unbind(dim=1)
45 -
46 - x = 16 // torch.tensor([], dtype=dtype).element_size()
47 - key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
48 - key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
49 -
50 - value_cache_shape = (num_blocks, num_heads, head_size, block_size)
51 - value_cache = torch.randn(
52 - size=value_cache_shape, dtype=dtype, device='cuda')
53 -
54 - # Run Flash attention.
55 - def run():
56 - cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
57 -
58 - benchmark('gather_cached_kv', run,
59 - size=num_tokens * num_heads * head_size * 2 * qkv.element_size())
60 -
61 -
62 -if __name__ == '__main__':
63 - BLOCK_SIZE = 8
64 - NUM_BLOCKS = 1024
65 - DTYPE = torch.half
66 -
67 - # LLaMA-13B and OPT-13B
68 - NUM_HEADS = 40
69 - HEAD_SIZE = 128
70 -
71 - run_benchmark = functools.partial(
72 - test_gather_cached_kv,
73 - num_heads=NUM_HEADS,
74 - head_size=HEAD_SIZE,
75 - block_size=BLOCK_SIZE,
76 - num_blocks=NUM_BLOCKS,
77 - dtype=DTYPE,
78 - )
79 -
80 - for i in range(6, 12):
81 - run_benchmark(num_tokens=2 ** i)
benchmarks/README.md new
+8
@@ -0,0 +1,8 @@
1 +# Benchmarking CacheFlow
2 +
3 +## Downloading the ShareGPT dataset
4 +
5 +You can download the dataset by running:
6 +```bash
7 +wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
8 +```
benchmarks/benchmark_latency.py renamed
benchmarks/benchmark_text_completion.py renamed
benchmarks/benchmark_throughput.py new
+104
@@ -0,0 +1,104 @@
1 +import argparse
2 +import json
3 +import random
4 +import time
5 +from typing import List, Tuple
6 +
7 +from cacheflow import LLM, SamplingParams
8 +from transformers import PreTrainedTokenizerBase
9 +
10 +
11 +def sample_requests(
12 + dataset_path: str,
13 + num_requests: int,
14 + tokenizer: PreTrainedTokenizerBase,
15 +) -> List[Tuple[List[int], int]]:
16 + # Load the dataset.
17 + with open(dataset_path) as f:
18 + dataset = json.load(f)
19 + # Filter out the conversations with less than 2 turns.
20 + dataset = [
21 + data for data in dataset
22 + if len(data["conversations"]) >= 2
23 + ]
24 + # Only keep the first two turns of each conversation.
25 + dataset = [
26 + (data["conversations"][0]["value"], data["conversations"][1]["value"])
27 + for data in dataset
28 + ]
29 +
30 + # Tokenize the prompts and completions.
31 + prompts = [prompt for prompt, _ in dataset]
32 + prompt_token_ids = tokenizer(prompts).input_ids
33 + completions = [completion for _, completion in dataset]
34 + completion_token_ids = tokenizer(completions).input_ids
35 + tokenized_dataset = []
36 + for i in range(len(dataset)):
37 + output_len = len(completion_token_ids[i])
38 + tokenized_dataset.append((prompt_token_ids[i], output_len))
39 + # Filter out if the prompt length + output length is greater than 2048.
40 + tokenized_dataset = [
41 + (prompt_token_ids, output_len)
42 + for prompt_token_ids, output_len in tokenized_dataset
43 + if len(prompt_token_ids) + output_len <= 2048
44 + ]
45 +
46 + # Sample the requests.
47 + sampled_requests = random.sample(tokenized_dataset, num_requests)
48 + return sampled_requests
49 +
50 +
51 +def main(args: argparse.Namespace):
52 + print(args)
53 + random.seed(args.seed)
54 +
55 + llm = LLM(
56 + model=args.model,
57 + tensor_parallel_size=args.tensor_parallel_size,
58 + seed=args.seed,
59 + )
60 + tokenizer = llm.get_tokenizer()
61 + requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
62 +
63 + # Add the requests to the server.
64 + for prompt_token_ids, output_len in requests:
65 + sampling_params = SamplingParams(
66 + n=args.n,
67 + temperature=0.0 if args.use_beam_search else 1.0,
68 + top_p=1.0,
69 + use_beam_search=args.use_beam_search,
70 + ignore_eos=True,
71 + max_tokens=output_len,
72 + )
73 + # FIXME(woosuk): Do not use internal method.
74 + llm._add_request(
75 + prompt="",
76 + sampling_params=sampling_params,
77 + prompt_token_ids=prompt_token_ids,
78 + )
79 +
80 + start = time.time()
81 + # FIXME(woosuk): Do use internal method.
82 + llm._run_server(use_tqdm=True)
83 + end = time.time()
84 + total_num_tokens = sum(
85 + len(prompt_token_ids) + output_len
86 + for prompt_token_ids, output_len in requests
87 + )
88 + print(f"Throughput: {total_num_tokens / (end - start):.2f} tokens/s")
89 +
90 +
91 +if __name__ == "__main__":
92 + parser = argparse.ArgumentParser(description="Benchmark the throughput.")
93 + parser.add_argument("--dataset", type=str, required=True,
94 + help="Path to the dataset.")
95 + parser.add_argument("--model", type=str, default="facebook/opt-125m")
96 + parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
97 + parser.add_argument("--n", type=int, default=1,
98 + help="Number of generated sequences per prompt.")
99 + parser.add_argument("--use-beam-search", action="store_true")
100 + parser.add_argument("--num-prompts", type=int, default=1000,
101 + help="Number of prompts to process.")
102 + parser.add_argument("--seed", type=int, default=0)
103 + args = parser.parse_args()
104 + main(args)
benchmarks/trace.py renamed
cacheflow/__init__.py
+2 -1
@@ -1,5 +1,5 @@
1 from cacheflow.entrypoints.llm import LLM
2 -from cacheflow.outputs import RequestOutput
2 +from cacheflow.outputs import RequestOutput, CompletionOutput
3 from cacheflow.sampling_params import SamplingParams
4 from cacheflow.server.arg_utils import ServerArgs
5 from cacheflow.server.llm_server import LLMServer
@@ -9,6 +9,7 @@ __all__ = [
9 "LLM",
10 "SamplingParams",
11 "RequestOutput",
12 + "CompletionOutput",
13 "LLMServer",
14 "ServerArgs",
15 "initialize_cluster",
cacheflow/core/scheduler.py
+3
@@ -87,6 +87,9 @@ class Scheduler:
87 def has_unfinished_seqs(self) -> bool:
88 return self.waiting or self.running or self.swapped
89
90 + def get_num_unfinished_seq_groups(self) -> int:
91 + return len(self.waiting) + len(self.running) + len(self.swapped)
92 +
93 def _schedule(self) -> Tuple[SchedulerOutputs, List[str]]:
94 # Blocks that need to be swaped or copied before model execution.
95 blocks_to_swap_in: Dict[int, int] = {}
cacheflow/entrypoints/llm.py
+24 -8
@@ -1,5 +1,6 @@
1 -from typing import List, Optional
1 +from typing import List, Optional, Union
2
3 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4 from tqdm import tqdm
5
6 from cacheflow.outputs import RequestOutput
@@ -31,6 +32,11 @@ class LLM:
32 self.llm_server = LLMServer.from_server_args(server_args)
33 self.request_counter = Counter()
34
35 + def get_tokenizer(
36 + self,
37 + ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
38 + return self.llm_server.tokenizer
39 +
40 def generate(
41 self,
42 prompts: List[str],
@@ -41,10 +47,6 @@ class LLM:
47 if sampling_params is None:
48 # Use default sampling params.
49 sampling_params = SamplingParams()
44 - # Initialize tqdm.
45 - if use_tqdm:
46 - pbar = tqdm(total=len(prompts), desc="Processed prompts")
47 -
50 # Add requests to the server.
51 for i in range(len(prompts)):
52 prompt = prompts[i]
@@ -52,10 +54,24 @@ class LLM:
54 token_ids = None
55 else:
56 token_ids = prompt_token_ids[i]
55 - request_id = str(next(self.request_counter))
56 - self.llm_server.add_request(request_id, prompt, sampling_params,
57 - token_ids)
57 + self._add_request(prompt, sampling_params, token_ids)
58 + return self._run_server(use_tqdm)
59
60 + def _add_request(
61 + self,
62 + prompt: str,
63 + sampling_params: SamplingParams,
64 + prompt_token_ids: Optional[List[int]],
65 + ) -> None:
66 + request_id = str(next(self.request_counter))
67 + self.llm_server.add_request(request_id, prompt, sampling_params,
68 + prompt_token_ids)
69 +
70 + def _run_server(self, use_tqdm: bool) -> List[RequestOutput]:
71 + # Initialize tqdm.
72 + if use_tqdm:
73 + num_requests = self.llm_server.get_num_unfinished_requests()
74 + pbar = tqdm(total=num_requests, desc="Processed prompts")
75 # Run the server.
76 outputs: List[RequestOutput] = []
77 while self.llm_server.has_unfinished_requests():
cacheflow/server/llm_server.py
+3
@@ -151,6 +151,9 @@ class LLMServer:
151 # Add the sequence group to the scheduler.
152 self.scheduler.add_seq_group(seq_group)
153
154 + def get_num_unfinished_requests(self) -> int:
155 + return self.scheduler.get_num_unfinished_seq_groups()
156 +
157 def has_unfinished_requests(self) -> bool:
158 return self.scheduler.has_unfinished_seqs()
159
examples/simple_server.py
+1 -2
@@ -19,9 +19,8 @@ def main(args: argparse.Namespace):
19 SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20 ]
21
22 - request_id = 0
23 -
22 # Run the server.
23 + request_id = 0
24 while True:
25 # To test iteration-level scheduling, we add one request at each step.
26 if test_prompts: