A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Benchmark] Refactor sample_requests in benchmark_throughput (#3613)
Co-authored-by: Roger Wang <ywang@roblox.com>
TianYu GUO committed
Apr 4, 2024 at 17:56 UTC
b7782002e1da25de77e0b1890ff8b72dd4df917c
1 file changed
+15
-16
benchmarks/benchmark_throughput.py
+15
-16
@@ -29,22 +29,23 @@ def sample_requests(
29
dataset = [(data["conversations"][0]["value"],
30
data["conversations"][1]["value"]) for data in dataset]
31
32
- # Tokenize the prompts and completions.
33
- prompts = [prompt for prompt, _ in dataset]
34
- prompt_token_ids = tokenizer(prompts).input_ids
35
- completions = [completion for _, completion in dataset]
36
- completion_token_ids = tokenizer(completions).input_ids
37
- tokenized_dataset = []
38
- for i in range(len(dataset)):
39
- output_len = len(completion_token_ids[i])
40
- if fixed_output_len is not None:
41
- output_len = fixed_output_len
42
- tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
32
+ # Shuffle the dataset.
33
+ random.shuffle(dataset)
34
44
- # Filter out too long sequences.
35
+ # Filter out sequences that are too long or too short
36
filtered_dataset: List[Tuple[str, int, int]] = []
46
- for prompt, prompt_token_ids, output_len in tokenized_dataset:
37
+ for i in range(len(dataset)):
38
+ if len(filtered_dataset) == num_requests:
39
+ break
40
+
41
+ # Tokenize the prompts and completions.
42
+ prompt = dataset[i][0]
43
+ prompt_token_ids = tokenizer(prompt).input_ids
44
+ completion = dataset[i][1]
45
+ completion_token_ids = tokenizer(completion).input_ids
46
prompt_len = len(prompt_token_ids)
47
+ output_len = len(completion_token_ids
48
+ ) if fixed_output_len is None else fixed_output_len
49
if prompt_len < 4 or output_len < 4:
50
# Prune too short sequences.
51
continue
@@ -53,9 +54,7 @@ def sample_requests(
54
continue
55
filtered_dataset.append((prompt, prompt_len, output_len))
56
56
- # Sample the requests.
57
- sampled_requests = random.sample(filtered_dataset, num_requests)
58
- return sampled_requests
57
+ return filtered_dataset
58
59
60
def run_vllm(