@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c32ab8be1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Speculative decoding] Add serving benchmark for llama3 70b + speculative decoding (#6964)

Cade Daniel committed Jul 30, 2024 at 17:53 UTC c32ab8be1ada40552e8a7c104a5de4830da2c7df
1 file changed +22 -1
.buildkite/nightly-benchmarks/tests/serving-tests.json
+22 -1
@@ -55,5 +55,26 @@
55 "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
56 "num_prompts": 200
57 }
58 + },
59 + {
60 + "test_name": "serving_llama70B_tp4_sharegpt_specdecode",
61 + "qps_list": [2],
62 + "server_parameters": {
63 + "model": "meta-llama/Meta-Llama-3-70B-Instruct",
64 + "disable_log_requests": "",
65 + "tensor_parallel_size": 4,
66 + "swap_space": 16,
67 + "speculative_model": "turboderp/Qwama-0.5B-Instruct",
68 + "num_speculative_tokens": 4,
69 + "speculative_draft_tensor_parallel_size": 1,
70 + "use_v2_block_manager": ""
71 + },
72 + "client_parameters": {
73 + "model": "meta-llama/Meta-Llama-3-70B-Instruct",
74 + "backend": "vllm",
75 + "dataset_name": "sharegpt",
76 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
77 + "num_prompts": 200
78 + }
79 }
59 -]
\ No newline at end of file
80 +]