A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Speculative decoding] Add serving benchmark for llama3 70b + speculative decoding (#6964)
Cade Daniel committed
Jul 30, 2024 at 17:53 UTC
c32ab8be1ada40552e8a7c104a5de4830da2c7df
1 file changed
+22
-1
.buildkite/nightly-benchmarks/tests/serving-tests.json
+22
-1
@@ -55,5 +55,26 @@
55
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
56
"num_prompts": 200
57
}
58
+ },
59
+ {
60
+ "test_name": "serving_llama70B_tp4_sharegpt_specdecode",
61
+ "qps_list": [2],
62
+ "server_parameters": {
63
+ "model": "meta-llama/Meta-Llama-3-70B-Instruct",
64
+ "disable_log_requests": "",
65
+ "tensor_parallel_size": 4,
66
+ "swap_space": 16,
67
+ "speculative_model": "turboderp/Qwama-0.5B-Instruct",
68
+ "num_speculative_tokens": 4,
69
+ "speculative_draft_tensor_parallel_size": 1,
70
+ "use_v2_block_manager": ""
71
+ },
72
+ "client_parameters": {
73
+ "model": "meta-llama/Meta-Llama-3-70B-Instruct",
74
+ "backend": "vllm",
75
+ "dataset_name": "sharegpt",
76
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
77
+ "num_prompts": 200
78
+ }
79
}
59
-]
\ No newline at end of file
80
+]