A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 00:30 UTC
[Benchmark] Cleanup deprecated nightly benchmark and adjust the docstring for performance benchmark (#25786)
Signed-off-by: KuntaiDu <kuntai@uchicago.edu>
Kuntai Du committed
Oct 29, 2025 at 21:43 UTC
8bff831f0aa239006f34b721e63e1340e3472067
29 files changed
+10
-1289
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
deleted
-184
@@ -1,184 +0,0 @@
1
-steps:
2
- - label: "Wait for container to be ready"
3
- key: wait-for-container-image
4
- agents:
5
- queue: A100
6
- plugins:
7
- - kubernetes:
8
- podSpec:
9
- containers:
10
- - image: badouralix/curl-jq
11
- command:
12
- - sh .buildkite/nightly-benchmarks/scripts/wait-for-image.sh
13
- - label: "Cleanup H100"
14
- agents:
15
- queue: H100
16
- depends_on: ~
17
- command: docker system prune -a --volumes --force
18
-
19
- - label: "A100"
20
- # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
21
- agents:
22
- queue: A100
23
- depends_on: wait-for-container-image
24
- if: build.branch == "main"
25
- plugins:
26
- - kubernetes:
27
- podSpec:
28
- priorityClassName: perf-benchmark
29
- containers:
30
- - image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:$BUILDKITE_COMMIT
31
- command:
32
- - bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
33
- resources:
34
- limits:
35
- nvidia.com/gpu: 8
36
- volumeMounts:
37
- - name: devshm
38
- mountPath: /dev/shm
39
- env:
40
- - name: VLLM_USAGE_SOURCE
41
- value: ci-test
42
- - name: HF_TOKEN
43
- valueFrom:
44
- secretKeyRef:
45
- name: hf-token-secret
46
- key: token
47
- nodeSelector:
48
- nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB
49
- volumes:
50
- - name: devshm
51
- emptyDir:
52
- medium: Memory
53
-
54
- - label: "H200"
55
- # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
56
- agents:
57
- queue: H200
58
- depends_on: wait-for-container-image
59
- if: build.branch == "main"
60
- plugins:
61
- - docker#v5.12.0:
62
- image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:$BUILDKITE_COMMIT
63
- command:
64
- - bash
65
- - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
66
- mount-buildkite-agent: true
67
- propagate-environment: true
68
- ipc: host
69
- gpus: 4,5,6,7
70
- volumes:
71
- - /data/benchmark-hf-cache:/root/.cache/huggingface
72
- environment:
73
- - VLLM_USAGE_SOURCE
74
- - HF_TOKEN
75
-
76
- #- block: "Run H100 Benchmark"
77
- #key: block-h100
78
- #depends_on: ~
79
-
80
- - label: "H100"
81
- # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
82
- agents:
83
- queue: H100
84
- depends_on: wait-for-container-image
85
- if: build.branch == "main"
86
- plugins:
87
- - docker#v5.12.0:
88
- image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:$BUILDKITE_COMMIT
89
- command:
90
- - bash
91
- - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
92
- mount-buildkite-agent: true
93
- propagate-environment: true
94
- ipc: host
95
- gpus: all # see CUDA_VISIBLE_DEVICES for actual GPUs used
96
- volumes:
97
- - /data/benchmark-hf-cache:/root/.cache/huggingface
98
- environment:
99
- - VLLM_USAGE_SOURCE
100
- - HF_TOKEN
101
-
102
- # Premerge benchmark
103
- - label: "A100"
104
- # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
105
- agents:
106
- queue: A100
107
- depends_on: wait-for-container-image
108
- if: build.branch != "main"
109
- plugins:
110
- - kubernetes:
111
- podSpec:
112
- priorityClassName: perf-benchmark
113
- containers:
114
- - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
115
- command:
116
- - bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
117
- resources:
118
- limits:
119
- nvidia.com/gpu: 8
120
- volumeMounts:
121
- - name: devshm
122
- mountPath: /dev/shm
123
- env:
124
- - name: VLLM_USAGE_SOURCE
125
- value: ci-test
126
- - name: HF_TOKEN
127
- valueFrom:
128
- secretKeyRef:
129
- name: hf-token-secret
130
- key: token
131
- nodeSelector:
132
- nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB
133
- volumes:
134
- - name: devshm
135
- emptyDir:
136
- medium: Memory
137
-
138
- - label: "H200"
139
- # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
140
- agents:
141
- queue: H200
142
- depends_on: wait-for-container-image
143
- if: build.branch != "main"
144
- plugins:
145
- - docker#v5.12.0:
146
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
147
- command:
148
- - bash
149
- - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
150
- mount-buildkite-agent: true
151
- propagate-environment: true
152
- ipc: host
153
- gpus: 4,5,6,7
154
- volumes:
155
- - /data/benchmark-hf-cache:/root/.cache/huggingface
156
- environment:
157
- - VLLM_USAGE_SOURCE
158
- - HF_TOKEN
159
-
160
- #- block: "Run H100 Benchmark"
161
- #key: block-h100
162
- #depends_on: ~
163
-
164
- - label: "H100"
165
- # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
166
- agents:
167
- queue: H100
168
- depends_on: wait-for-container-image
169
- if: build.branch != "main"
170
- plugins:
171
- - docker#v5.12.0:
172
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
173
- command:
174
- - bash
175
- - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
176
- mount-buildkite-agent: true
177
- propagate-environment: true
178
- ipc: host
179
- gpus: all # see CUDA_VISIBLE_DEVICES for actual GPUs used
180
- volumes:
181
- - /data/benchmark-hf-cache:/root/.cache/huggingface
182
- environment:
183
- - VLLM_USAGE_SOURCE
184
- - HF_TOKEN
.buildkite/nightly-benchmarks/nightly-annotation.md
deleted
-28
@@ -1,28 +0,0 @@
1
-# Nightly benchmark annotation
2
-
3
-## Description
4
-
5
-This file contains the downloading link for benchmarking results.
6
-
7
-- [benchmarking pipeline](artifact://nightly-pipeline.yaml)
8
-- [benchmarking results](artifact://results.zip)
9
-- [benchmarking code](artifact://nightly-benchmarks.zip)
10
-
11
-Please download the visualization scripts in the post
12
-
13
-## Results reproduction
14
-
15
-- Find the docker we use in `benchmarking pipeline`
16
-- Deploy the docker, and inside the docker:
17
- - Download `nightly-benchmarks.zip`.
18
- - In the same folder, run the following code:
19
-
20
- ```bash
21
- export HF_TOKEN=<your HF token>
22
- apt update
23
- apt install -y git
24
- unzip nightly-benchmarks.zip
25
- VLLM_SOURCE_CODE_LOC=./ bash .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
26
- ```
27
-
28
-And the results will be inside `./benchmarks/results`.
.buildkite/nightly-benchmarks/nightly-descriptions.md
deleted
-39
@@ -1,39 +0,0 @@
1
-
2
-# Nightly benchmark
3
-
4
-This benchmark aims to:
5
-
6
-- Provide performance clarity: Provide clarity on which one (vllm, tensorrt-llm, lmdeploy and SGLang) leads in performance in what workload.
7
-- Be reproducible: one can run the exact same set of benchmarking commands inside the exact same docker by following reproducing instructions.
8
-
9
-Latest results: [results link](https://blog.vllm.ai/2024/09/05/perf-update.html), scroll to the end.
10
-
11
-Latest reproduction guide: [github issue link](https://github.com/vllm-project/vllm/issues/8176)
12
-
13
-## Setup
14
-
15
-- Docker images:
16
- - vLLM: `vllm/vllm-openai:v0.6.2`
17
- - SGLang: `lmsysorg/sglang:v0.3.2-cu121`
18
- - LMDeploy: `openmmlab/lmdeploy:v0.6.1-cu12`
19
- - TensorRT-LLM: `nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3`
20
- - *NOTE: we use r24.07 as the current implementation only works for this version. We are going to bump this up.*
21
- - Check [nightly-pipeline.yaml](nightly-pipeline.yaml) for the concrete docker images, specs and commands we use for the benchmark.
22
-- Hardware
23
- - 8x Nvidia A100 GPUs
24
-- Workload:
25
- - Dataset
26
- - ShareGPT dataset
27
- - Prefill-heavy dataset (in average 462 input tokens, 16 tokens as output)
28
- - Decode-heavy dataset (in average 462 input tokens, 256 output tokens)
29
- - Check [nightly-tests.json](tests/nightly-tests.json) for the concrete configuration of datasets we use.
30
- - Models: llama-3 8B, llama-3 70B.
31
- - We do not use llama 3.1 as it is incompatible with trt-llm r24.07. ([issue](https://github.com/NVIDIA/TensorRT-LLM/issues/2105)).
32
- - Average QPS (query per second): 2, 4, 8, 16, 32 and inf.
33
- - Queries are randomly sampled, and arrival patterns are determined via Poisson process, but all with fixed random seed.
34
- - Evaluation metrics: Throughput (higher the better), TTFT (time to the first token, lower the better), ITL (inter-token latency, lower the better).
35
-
36
-## Known issues
37
-
38
-- TRT-LLM crashes with Llama 3.1 8B [issue](https://github.com/NVIDIA/TensorRT-LLM/issues/2105).
39
-- TGI does not support `ignore-eos` flag.
.buildkite/nightly-benchmarks/nightly-pipeline.yaml
deleted
-196
@@ -1,196 +0,0 @@
1
-common_pod_spec: &common_pod_spec
2
- priorityClassName: perf-benchmark
3
- nodeSelector:
4
- nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB
5
- volumes:
6
- - name: devshm
7
- emptyDir:
8
- medium: Memory
9
- - name: hf-cache
10
- hostPath:
11
- path: /root/.cache/huggingface
12
- type: Directory
13
-
14
-common_container_settings: &common_container_settings
15
- command:
16
- - bash .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
17
- resources:
18
- limits:
19
- nvidia.com/gpu: 8
20
- volumeMounts:
21
- - name: devshm
22
- mountPath: /dev/shm
23
- - name: hf-cache
24
- mountPath: /root/.cache/huggingface
25
- env:
26
- - name: VLLM_USAGE_SOURCE
27
- value: ci-test
28
- - name: HF_HOME
29
- value: /root/.cache/huggingface
30
- - name: VLLM_SOURCE_CODE_LOC
31
- value: /workspace/build/buildkite/vllm/performance-benchmark
32
- - name: HF_TOKEN
33
- valueFrom:
34
- secretKeyRef:
35
- name: hf-token-secret
36
- key: token
37
-
38
-steps:
39
- - block: ":rocket: Ready for comparing vllm against alternatives? This will take 4 hours."
40
-
41
-
42
-
43
- - label: "A100 vllm step 10"
44
- priority: 100
45
- agents:
46
- queue: A100
47
- plugins:
48
- - kubernetes:
49
- podSpec:
50
- <<: *common_pod_spec
51
- containers:
52
- - image: vllm/vllm-openai:v0.6.2
53
- <<: *common_container_settings
54
-
55
-
56
-
57
- - label: "A100 sglang benchmark"
58
- priority: 100
59
- agents:
60
- queue: A100
61
- plugins:
62
- - kubernetes:
63
- podSpec:
64
- <<: *common_pod_spec
65
- containers:
66
- - image: lmsysorg/sglang:v0.3.2-cu121
67
- <<: *common_container_settings
68
-
69
- - label: "A100 lmdeploy benchmark"
70
- priority: 100
71
- agents:
72
- queue: A100
73
- plugins:
74
- - kubernetes:
75
- podSpec:
76
- <<: *common_pod_spec
77
- containers:
78
- - image: openmmlab/lmdeploy:v0.6.1-cu12
79
- <<: *common_container_settings
80
-
81
-
82
-
83
-
84
- - label: "A100 trt llama-8B"
85
- priority: 100
86
- agents:
87
- queue: A100
88
- plugins:
89
- - kubernetes:
90
- podSpec:
91
- <<: *common_pod_spec
92
- containers:
93
- - image: nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3
94
- <<: *common_container_settings
95
- env:
96
- - name: VLLM_USAGE_SOURCE
97
- value: ci-test
98
- - name: HF_HOME
99
- value: /root/.cache/huggingface
100
- - name: VLLM_SOURCE_CODE_LOC
101
- value: /workspace/build/buildkite/vllm/performance-benchmark
102
- - name: HF_TOKEN
103
- valueFrom:
104
- secretKeyRef:
105
- name: hf-token-secret
106
- key: token
107
- - name: TEST_SELECTOR
108
- value: "llama8B"
109
-
110
-
111
- - label: "A100 trt llama-70B"
112
- priority: 100
113
- agents:
114
- queue: A100
115
- plugins:
116
- - kubernetes:
117
- podSpec:
118
- <<: *common_pod_spec
119
- containers:
120
- - image: nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3
121
- <<: *common_container_settings
122
- env:
123
- - name: VLLM_USAGE_SOURCE
124
- value: ci-test
125
- - name: HF_HOME
126
- value: /root/.cache/huggingface
127
- - name: VLLM_SOURCE_CODE_LOC
128
- value: /workspace/build/buildkite/vllm/performance-benchmark
129
- - name: HF_TOKEN
130
- valueFrom:
131
- secretKeyRef:
132
- name: hf-token-secret
133
- key: token
134
- - name: TEST_SELECTOR
135
- value: "llama70B"
136
-
137
-
138
- # FIXME(Kuntai): uncomment this after NVIDIA gives us their test docker image
139
- # - label: "A100 trt benchmark"
140
- # priority: 100
141
- # agents:
142
- # queue: A100
143
- # plugins:
144
- # - kubernetes:
145
- # podSpec:
146
- # <<: *common_pod_spec
147
- # containers:
148
- # - image: nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3
149
- # <<: *common_container_settings
150
-
151
-
152
- # FIXME(Kuntai): uncomment this after TGI supports `--ignore-eos`.
153
- # - label: "A100 tgi benchmark"
154
- # priority: 100
155
- # agents:
156
- # queue: A100
157
- # plugins:
158
- # - kubernetes:
159
- # podSpec:
160
- # <<: *common_pod_spec
161
- # containers:
162
- # - image: ghcr.io/huggingface/text-generation-inference:2.2.0
163
- # <<: *common_container_settings
164
-
165
- - wait
166
-
167
- - label: "Collect the results"
168
- priority: 100
169
- agents:
170
- queue: A100
171
- plugins:
172
- - kubernetes:
173
- podSpec:
174
- <<: *common_pod_spec
175
- containers:
176
- - image: vllm/vllm-openai:v0.5.0.post1
177
- command:
178
- - bash .buildkite/nightly-benchmarks/scripts/nightly-annotate.sh
179
- resources:
180
- limits:
181
- nvidia.com/gpu: 8
182
- volumeMounts:
183
- - name: devshm
184
- mountPath: /dev/shm
185
- env:
186
- - name: VLLM_USAGE_SOURCE
187
- value: ci-test
188
- - name: VLLM_SOURCE_CODE_LOC
189
- value: /workspace/build/buildkite/vllm/performance-benchmark
190
- - name: HF_TOKEN
191
- valueFrom:
192
- secretKeyRef:
193
- name: hf-token-secret
194
- key: token
195
-
196
- - block: ":rocket: check the results!"
\ No newline at end of file
.buildkite/nightly-benchmarks/scripts/download-tokenizer.py
deleted
-26
@@ -1,26 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import argparse
5
-
6
-from transformers import AutoTokenizer
7
-
8
-
9
-def main(model, cachedir):
10
- # Load the tokenizer and save it to the specified directory
11
- tokenizer = AutoTokenizer.from_pretrained(model)
12
- tokenizer.save_pretrained(cachedir)
13
- print(f"Tokenizer saved to {cachedir}")
14
-
15
-
16
-if __name__ == "__main__":
17
- parser = argparse.ArgumentParser(
18
- description="Download and save Hugging Face tokenizer"
19
- )
20
- parser.add_argument("--model", type=str, required=True, help="Name of the model")
21
- parser.add_argument(
22
- "--cachedir", type=str, required=True, help="Directory to save the tokenizer"
23
- )
24
-
25
- args = parser.parse_args()
26
- main(args.model, args.cachedir)
.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py
deleted
-97
@@ -1,97 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import argparse
5
-import json
6
-from pathlib import Path
7
-
8
-import numpy as np
9
-import pandas as pd
10
-from tabulate import tabulate
11
-
12
-
13
-def parse_arguments():
14
- parser = argparse.ArgumentParser(
15
- description="Parse command line arguments for summary-nightly-results script."
16
- )
17
- parser.add_argument(
18
- "--results-folder",
19
- type=str,
20
- required=True,
21
- help="The folder where the results are stored.",
22
- )
23
- parser.add_argument(
24
- "--description", type=str, required=True, help="Description of the results."
25
- )
26
-
27
- args = parser.parse_args()
28
- return args
29
-
30
-
31
-def get_perf(df, method, model, metric):
32
- means = []
33
-
34
- for qps in [2, 4, 8, 16, "inf"]:
35
- target = df["Test name"].str.contains(model)
36
- target = target & df["Engine"].str.contains(method)
37
- target = target & df["Test name"].str.contains("qps_" + str(qps))
38
- filtered_df = df[target]
39
-
40
- if filtered_df.empty:
41
- means.append(0.0)
42
- else:
43
- means.append(filtered_df[metric].values[0])
44
-
45
- return np.array(means)
46
-
47
-
48
-def get_perf_w_std(df, method, model, metric):
49
- if metric in ["TTFT", "ITL"]:
50
- mean = get_perf(df, method, model, "Mean " + metric + " (ms)")
51
- mean = mean.tolist()
52
- std = get_perf(df, method, model, "Std " + metric + " (ms)")
53
- if std.mean() == 0:
54
- std = None
55
- success = get_perf(df, method, model, "Successful req.")
56
- if std is not None:
57
- std = std / np.sqrt(success)
58
- std = std.tolist()
59
-
60
- else:
61
- assert metric == "Tput"
62
- mean = get_perf(df, method, model, "Input Tput (tok/s)") + get_perf(
63
- df, method, model, "Output Tput (tok/s)"
64
- )
65
- mean = mean.tolist()
66
- std = None
67
-
68
- return mean, std
69
-
70
-
71
-def main(args):
72
- results_folder = Path(args.results_folder)
73
-
74
- results = []
75
-
76
- # collect results
77
- for test_file in results_folder.glob("*_nightly_results.json"):
78
- with open(test_file) as f:
79
- results = results + json.loads(f.read())
80
-
81
- # generate markdown table
82
- df = pd.DataFrame.from_dict(results)
83
-
84
- md_table = tabulate(df, headers="keys", tablefmt="pipe", showindex=False)
85
-
86
- with open(args.description) as f:
87
- description = f.read()
88
-
89
- description = description.format(nightly_results_benchmarking_table=md_table)
90
-
91
- with open("nightly_results.md", "w") as f:
92
- f.write(description)
93
-
94
-
95
-if __name__ == "__main__":
96
- args = parse_arguments()
97
- main(args)
.buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py
deleted
-9
@@ -1,9 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from lmdeploy.serve.openai.api_client import APIClient
5
-
6
-api_client = APIClient("http://localhost:8000")
7
-model_name = api_client.available_models[0]
8
-
9
-print(model_name)
.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh
deleted
-78
@@ -1,78 +0,0 @@
1
-#!/bin/bash
2
-
3
-set -ex
4
-set -o pipefail
5
-
6
-
7
-main() {
8
-
9
- (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
10
- (which jq) || (apt-get update && apt-get -y install jq)
11
- (which zip) || (apt-get install -y zip)
12
-
13
- if [ ! -f /workspace/buildkite-agent ]; then
14
- echo "buildkite-agent binary not found. Skip plotting the results."
15
- exit 0
16
- fi
17
-
18
- # initial annotation
19
- #description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
20
-
21
- # download results
22
- cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
23
- mkdir -p results/
24
- /workspace/buildkite-agent artifact download 'results/*nightly_results.json' results/
25
- ls
26
- ls results/
27
-
28
- # upload benchmark results
29
- zip -r results.zip results/
30
- /workspace/buildkite-agent artifact upload "results.zip"
31
-
32
- # upload benchmarking scripts
33
- cd "$VLLM_SOURCE_CODE_LOC/"
34
- zip -r nightly-benchmarks.zip .buildkite/ benchmarks/
35
- /workspace/buildkite-agent artifact upload "nightly-benchmarks.zip"
36
-
37
- cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
38
- # upload benchmarking pipeline
39
- /workspace/buildkite-agent artifact upload "nightly-pipeline.yaml"
40
-
41
- cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
42
- /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly-annotation.md
43
-
44
-
45
-
46
- # The figures should be generated by a separate process outside the CI/CD pipeline
47
-
48
- # # generate figures
49
- # python3 -m pip install tabulate pandas matplotlib
50
-
51
- # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py \
52
- # --description $description \
53
- # --results-folder results/
54
-
55
-
56
- # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/plot-nightly-results.py \
57
- # --description $description \
58
- # --results-folder results/ \
59
- # --dataset sharegpt
60
-
61
- # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/plot-nightly-results.py \
62
- # --description $description \
63
- # --results-folder results/ \
64
- # --dataset sonnet_2048_128
65
-
66
- # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/plot-nightly-results.py \
67
- # --description $description \
68
- # --results-folder results/ \
69
- # --dataset sonnet_128_2048
70
-
71
- # # upload results and figures
72
- # /workspace/buildkite-agent artifact upload "nightly_results*.png"
73
- # /workspace/buildkite-agent artifact upload $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-pipeline.yaml
74
- # /workspace/buildkite-agent artifact upload $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/tests/nightly-tests.json
75
- # /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly_results.md
76
-}
77
-
78
-main "$@"
.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
deleted
-464
@@ -1,464 +0,0 @@
1
-#!/bin/bash
2
-
3
-set -o pipefail
4
-set -x
5
-
6
-check_gpus() {
7
- # check the number of GPUs and GPU type.
8
- declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l)
9
- if [[ $gpu_count -gt 0 ]]; then
10
- echo "GPU found."
11
- else
12
- echo "Need at least 1 GPU to run benchmarking."
13
- exit 1
14
- fi
15
- declare -g gpu_type="$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')"
16
- echo "GPU type is $gpu_type"
17
-}
18
-
19
-check_hf_token() {
20
- # check if HF_TOKEN is available and valid
21
- if [[ -z "$HF_TOKEN" ]]; then
22
- echo "Error: HF_TOKEN is not set."
23
- exit 1
24
- elif [[ ! "$HF_TOKEN" =~ ^hf_ ]]; then
25
- echo "Error: HF_TOKEN does not start with 'hf_'."
26
- exit 1
27
- else
28
- echo "HF_TOKEN is set and valid."
29
- fi
30
-}
31
-
32
-
33
-upload_to_buildkite() {
34
- # upload the benchmarking results to buildkite
35
-
36
- # if the agent binary is not found, skip uploading the results, exit 0
37
- if [ ! -f /workspace/buildkite-agent ]; then
38
- echo "buildkite-agent binary not found. Skip uploading the results."
39
- return 0
40
- fi
41
- # /workspace/buildkite-agent annotate --style "success" --context "benchmark-results" --append < $RESULTS_FOLDER/${CURRENT_LLM_SERVING_ENGINE}_nightly_results.md
42
- /workspace/buildkite-agent artifact upload "$RESULTS_FOLDER/*"
43
-}
44
-
45
-
46
-get_current_llm_serving_engine() {
47
-
48
- if which lmdeploy >/dev/null; then
49
- echo "Container: lmdeploy"
50
- export CURRENT_LLM_SERVING_ENGINE=lmdeploy
51
- return
52
- fi
53
-
54
- if [ -e /tgi-entrypoint.sh ]; then
55
- echo "Container: tgi"
56
- export CURRENT_LLM_SERVING_ENGINE=tgi
57
- return
58
- fi
59
-
60
- if which trtllm-build >/dev/null; then
61
- echo "Container: tensorrt-llm"
62
- export CURRENT_LLM_SERVING_ENGINE=trt
63
- return
64
- fi
65
-
66
- if [ -e /sgl-workspace ]; then
67
- echo "Container: sglang"
68
- export CURRENT_LLM_SERVING_ENGINE=sglang
69
- return
70
- fi
71
-
72
- if [ -e /vllm-workspace ]; then
73
- echo "Container: vllm"
74
- # move to a completely irrelevant directory, to avoid import vllm from current folder
75
- export CURRENT_LLM_SERVING_ENGINE=vllm
76
-
77
- return
78
- fi
79
-}
80
-
81
-json2args() {
82
- # transforms the JSON string to command line args, and '_' is replaced to '-'
83
- # example:
84
- # input: { "model": "meta-llama/Llama-2-7b-chat-hf", "tensor_parallel_size": 1 }
85
- # output: --model meta-llama/Llama-2-7b-chat-hf --tensor-parallel-size 1
86
- local json_string=$1
87
- local args=$(
88
- echo "$json_string" | jq -r '
89
- to_entries |
90
- map("--" + (.key | gsub("_"; "-")) + " " + (.value | tostring)) |
91
- join(" ")
92
- '
93
- )
94
- echo "$args"
95
-}
96
-
97
-kill_gpu_processes() {
98
- pkill -f '[p]ython'
99
- pkill -f '[p]ython3'
100
- pkill -f '[t]ritonserver'
101
- pkill -f '[p]t_main_thread'
102
- pkill -f '[t]ext-generation'
103
- pkill -f '[l]mdeploy'
104
- # vLLM now names the process with VLLM prefix after https://github.com/vllm-project/vllm/pull/21445
105
- pkill -f '[V]LLM'
106
-
107
- while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
108
- sleep 1
109
- done
110
-}
111
-
112
-wait_for_server() {
113
- # wait for vllm server to start
114
- # return 1 if vllm server crashes
115
- timeout 1200 bash -c '
116
- until curl -s localhost:8000/v1/completions > /dev/null; do
117
- sleep 1
118
- done' && return 0 || return 1
119
-}
120
-
121
-ensure_installed() {
122
- # Ensure that the given command is installed by apt-get
123
- local cmd=$1
124
- if ! which "$cmd" >/dev/null; then
125
- apt-get update && apt-get install -y "$cmd"
126
- fi
127
-}
128
-
129
-run_serving_tests() {
130
- # run serving tests using `vllm bench serve` command
131
- # $1: a json file specifying serving test cases
132
-
133
- local serving_test_file
134
- serving_test_file=$1
135
-
136
- # Iterate over serving tests
137
- jq -c '.[]' "$serving_test_file" | while read -r params; do
138
- # get the test name, and append the GPU type back to it.
139
- test_name=$(echo "$params" | jq -r '.test_name')
140
-
141
- # if TEST_SELECTOR is set, only run the test cases that match the selector
142
- if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then
143
- echo "Skip test case $test_name."
144
- continue
145
- fi
146
-
147
- # prepend the current serving engine to the test name
148
- test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name}
149
-
150
- # get common parameters
151
- common_params=$(echo "$params" | jq -r '.common_parameters')
152
- model=$(echo "$common_params" | jq -r '.model')
153
- tp=$(echo "$common_params" | jq -r '.tp')
154
- dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
155
- dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
156
- port=$(echo "$common_params" | jq -r '.port')
157
- num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
158
- reuse_server=$(echo "$common_params" | jq -r '.reuse_server')
159
-
160
- # get client and server arguments
161
- server_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_server_parameters")
162
- client_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_client_parameters")
163
- client_args=$(json2args "$client_params")
164
- qps_list=$(echo "$params" | jq -r '.qps_list')
165
- qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
166
- echo "Running over qps list $qps_list"
167
-
168
- # check if there is enough GPU to run the test
169
- if [[ $gpu_count -lt $tp ]]; then
170
- echo "Required num-shard $tp but only $gpu_count GPU found. Skip testcase $test_name."
171
- continue
172
- fi
173
-
174
- if [[ $reuse_server == "true" ]]; then
175
- echo "Reuse previous server for test case $test_name"
176
- else
177
- kill_gpu_processes
178
- bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
179
- "$server_params" "$common_params"
180
- fi
181
-
182
- if wait_for_server; then
183
- echo ""
184
- echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
185
- else
186
- echo ""
187
- echo "$CURRENT_LLM_SERVING_ENGINE failed to start within the timeout period."
188
- break
189
- fi
190
-
191
- # prepare tokenizer
192
- # this is required for lmdeploy.
193
- cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
194
- rm -rf /tokenizer_cache
195
- mkdir /tokenizer_cache
196
- python3 ../.buildkite/nightly-benchmarks/scripts/download-tokenizer.py \
197
- --model "$model" \
198
- --cachedir /tokenizer_cache
199
- cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
200
-
201
-
202
- # change model name for lmdeploy (it will not follow standard hf name)
203
- if [[ "$CURRENT_LLM_SERVING_ENGINE" == "lmdeploy" ]]; then
204
- model=$(python ../.buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py)
205
- fi
206
-
207
- # iterate over different QPS
208
- for qps in $qps_list; do
209
- # remove the surrounding single quote from qps
210
- if [[ "$qps" == *"inf"* ]]; then
211
- echo "qps was $qps"
212
- qps="inf"
213
- echo "now qps is $qps"
214
- fi
215
-
216
- new_test_name=$test_name"_qps_"$qps
217
-
218
- backend=$CURRENT_LLM_SERVING_ENGINE
219
-
220
- if [[ $backend = "trt" ]]; then
221
- backend="tensorrt-llm"
222
- fi
223
-
224
- if [[ "$backend" == *"vllm"* ]]; then
225
- backend="vllm"
226
- fi
227
-
228
- if [[ "$dataset_name" = "sharegpt" ]]; then
229
-
230
- client_command="vllm bench serve \
231
- --backend $backend \
232
- --tokenizer /tokenizer_cache \
233
- --model $model \
234
- --dataset-name $dataset_name \
235
- --dataset-path $dataset_path \
236
- --num-prompts $num_prompts \
237
- --port $port \
238
- --save-result \
239
- --result-dir $RESULTS_FOLDER \
240
- --result-filename ${new_test_name}.json \
241
- --request-rate $qps \
242
- --ignore-eos \
243
- $client_args"
244
-
245
- elif [[ "$dataset_name" = "sonnet" ]]; then
246
-
247
- sonnet_input_len=$(echo "$common_params" | jq -r '.sonnet_input_len')
248
- sonnet_output_len=$(echo "$common_params" | jq -r '.sonnet_output_len')
249
- sonnet_prefix_len=$(echo "$common_params" | jq -r '.sonnet_prefix_len')
250
-
251
- client_command="vllm bench serve \
252
- --backend $backend \
253
- --tokenizer /tokenizer_cache \
254
- --model $model \
255
- --dataset-name $dataset_name \
256
- --dataset-path $dataset_path \
257
- --num-prompts $num_prompts \
258
- --sonnet-input-len $sonnet_input_len \
259
- --sonnet-output-len $sonnet_output_len \
260
- --sonnet-prefix-len $sonnet_prefix_len \
261
- --port $port \
262
- --save-result \
263
- --result-dir $RESULTS_FOLDER \
264
- --result-filename ${new_test_name}.json \
265
- --request-rate $qps \
266
- --ignore-eos \
267
- $client_args"
268
-
269
- else
270
-
271
- echo "The dataset name must be either 'sharegpt' or 'sonnet'. Got $dataset_name."
272
- exit 1
273
-
274
- fi
275
-
276
-
277
-
278
- echo "Running test case $test_name with qps $qps"
279
- echo "Client command: $client_command"
280
-
281
- eval "$client_command"
282
-
283
- server_command="None"
284
-
285
- # record the benchmarking commands
286
- jq_output=$(jq -n \
287
- --arg server "$server_command" \
288
- --arg client "$client_command" \
289
- --arg gpu "$gpu_type" \
290
- --arg engine "$CURRENT_LLM_SERVING_ENGINE" \
291
- '{
292
- server_command: $server,
293
- client_command: $client,
294
- gpu_type: $gpu,
295
- engine: $engine
296
- }')
297
- echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands"
298
-
299
- done
300
-
301
- done
302
-
303
- kill_gpu_processes
304
-}
305
-
306
-run_genai_perf_tests() {
307
- # run genai-perf tests
308
-
309
- # $1: a json file specifying genai-perf test cases
310
- local genai_perf_test_file
311
- genai_perf_test_file=$1
312
-
313
- # Iterate over genai-perf tests
314
- jq -c '.[]' "$genai_perf_test_file" | while read -r params; do
315
- # get the test name, and append the GPU type back to it.
316
- test_name=$(echo "$params" | jq -r '.test_name')
317
-
318
- # if TEST_SELECTOR is set, only run the test cases that match the selector
319
- if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then
320
- echo "Skip test case $test_name."
321
- continue
322
- fi
323
-
324
- # prepend the current serving engine to the test name
325
- test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name}
326
-
327
- # get common parameters
328
- common_params=$(echo "$params" | jq -r '.common_parameters')
329
- model=$(echo "$common_params" | jq -r '.model')
330
- tp=$(echo "$common_params" | jq -r '.tp')
331
- dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
332
- dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
333
- port=$(echo "$common_params" | jq -r '.port')
334
- num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
335
- reuse_server=$(echo "$common_params" | jq -r '.reuse_server')
336
-
337
- # get client and server arguments
338
- server_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_server_parameters")
339
- qps_list=$(echo "$params" | jq -r '.qps_list')
340
- qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
341
- echo "Running over qps list $qps_list"
342
-
343
- # check if there is enough GPU to run the test
344
- if [[ $gpu_count -lt $tp ]]; then
345
- echo "Required num-shard $tp but only $gpu_count GPU found. Skip testcase $test_name."
346
- continue
347
- fi
348
-
349
- if [[ $reuse_server == "true" ]]; then
350
- echo "Reuse previous server for test case $test_name"
351
- else
352
- kill_gpu_processes
353
- bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
354
- "$server_params" "$common_params"
355
- fi
356
-
357
- if wait_for_server; then
358
- echo ""
359
- echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
360
- else
361
- echo ""
362
- echo "$CURRENT_LLM_SERVING_ENGINE failed to start within the timeout period."
363
- break
364
- fi
365
-
366
- # iterate over different QPS
367
- for qps in $qps_list; do
368
- # remove the surrounding single quote from qps
369
- if [[ "$qps" == *"inf"* ]]; then
370
- echo "qps was $qps"
371
- qps=$num_prompts
372
- echo "now qps is $qps"
373
- fi
374
-
375
- new_test_name=$test_name"_qps_"$qps
376
- backend=$CURRENT_LLM_SERVING_ENGINE
377
-
378
- if [[ "$backend" == *"vllm"* ]]; then
379
- backend="vllm"
380
- fi
381
- #TODO: add output dir.
382
- client_command="genai-perf profile \
383
- -m $model \
384
- --service-kind openai \
385
- --backend "$backend" \
386
- --endpoint-type chat \
387
- --streaming \
388
- --url localhost:$port \
389
- --request-rate $qps \
390
- --num-prompts $num_prompts \
391
- "
392
-
393
- echo "Client command: $client_command"
394
-
395
- eval "$client_command"
396
-
397
- #TODO: process/record outputs
398
- done
399
- done
400
-
401
- kill_gpu_processes
402
-
403
-}
404
-
405
-prepare_dataset() {
406
-
407
- # download sharegpt dataset
408
- cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
409
- wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
410
-
411
- # duplicate sonnet by 4x, to allow benchmarking with input length 2048
412
- cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
413
- echo "" > sonnet_4x.txt
414
- for _ in {1..4}
415
- do
416
- cat sonnet.txt >> sonnet_4x.txt
417
- done
418
-
419
-}
420
-
421
-main() {
422
-
423
- # check if the environment variable is successfully injected from yaml
424
-
425
- check_gpus
426
- check_hf_token
427
- get_current_llm_serving_engine
428
-
429
- pip install -U transformers
430
-
431
- pip install -r requirements/dev.txt
432
- which genai-perf
433
-
434
- # check storage
435
- df -h
436
-
437
- ensure_installed wget
438
- ensure_installed curl
439
- ensure_installed jq
440
- # genai-perf dependency
441
- ensure_installed libb64-0d
442
-
443
- prepare_dataset
444
-
445
- cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
446
- declare -g RESULTS_FOLDER=results/
447
- mkdir -p $RESULTS_FOLDER
448
- BENCHMARK_ROOT="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
449
-
450
- # run the test
451
- run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json"
452
-
453
- # run genai-perf tests
454
- run_genai_perf_tests "$BENCHMARK_ROOT/tests/genai-perf-tests.json"
455
- mv artifacts/ $RESULTS_FOLDER/
456
-
457
- # upload benchmark results to buildkite
458
- python3 -m pip install tabulate pandas
459
- python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py"
460
- upload_to_buildkite
461
-
462
-}
463
-
464
-main "$@"
.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py
deleted
-82
@@ -1,82 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import datetime
5
-import json
6
-import os
7
-from pathlib import Path
8
-
9
-import pandas as pd
10
-from tabulate import tabulate
11
-
12
-results_folder = Path("results/")
13
-
14
-# serving results and the keys that will be printed into markdown
15
-serving_results = []
16
-serving_column_mapping = {
17
- "test_name": "Test name",
18
- "gpu_type": "GPU",
19
- "completed": "Successful req.",
20
- "request_throughput": "Tput (req/s)",
21
- "mean_ttft_ms": "Mean TTFT (ms)",
22
- "std_ttft_ms": "Std TTFT (ms)",
23
- "median_ttft_ms": "Median TTFT (ms)",
24
- "mean_itl_ms": "Mean ITL (ms)",
25
- "std_itl_ms": "Std ITL (ms)",
26
- "median_itl_ms": "Median ITL (ms)",
27
- "mean_tpot_ms": "Mean TPOT (ms)",
28
- "std_tpot_ms": "Std TPOT (ms)",
29
- "median_tpot_ms": "Median TPOT (ms)",
30
- "total_token_throughput": "Total Token Tput (tok/s)",
31
- "output_throughput": "Output Tput (tok/s)",
32
- "total_input_tokens": "Total input tokens",
33
- "total_output_tokens": "Total output tokens",
34
- "engine": "Engine",
35
-}
36
-
37
-if __name__ == "__main__":
38
- # collect results
39
- for test_file in results_folder.glob("*.json"):
40
- with open(test_file) as f:
41
- raw_result = json.loads(f.read())
42
-
43
- # attach the benchmarking command to raw_result
44
- with open(test_file.with_suffix(".commands")) as f:
45
- command = json.loads(f.read())
46
- raw_result.update(command)
47
-
48
- # update the test name of this result
49
- raw_result.update({"test_name": test_file.stem})
50
-
51
- # add the result to raw_result
52
- serving_results.append(raw_result)
53
- continue
54
-
55
- serving_results = pd.DataFrame.from_dict(serving_results)
56
-
57
- if not serving_results.empty:
58
- serving_results = serving_results[list(serving_column_mapping.keys())].rename(
59
- columns=serving_column_mapping
60
- )
61
-
62
- serving_md_table_with_headers = tabulate(
63
- serving_results, headers="keys", tablefmt="pipe", showindex=False
64
- )
65
- # remove the first line of header
66
- serving_md_table_lines = serving_md_table_with_headers.split("\n")
67
- serving_md_table_without_header = "\n".join(serving_md_table_lines[2:])
68
-
69
- prefix = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
70
- prefix = prefix + "_" + os.environ.get("CURRENT_LLM_SERVING_ENGINE")
71
-
72
- # document benchmarking results in markdown
73
- with open(results_folder / f"{prefix}_nightly_results.md", "w") as f:
74
- # document results with header.
75
- # for those who wants to reproduce our benchmark.
76
- f.write(serving_md_table_with_headers)
77
- f.write("\n")
78
-
79
- # document benchmarking results in json
80
- with open(results_folder / f"{prefix}_nightly_results.json", "w") as f:
81
- results = serving_results.to_dict(orient="records")
82
- f.write(json.dumps(results))
.buildkite/nightly-benchmarks/scripts/wait-for-image.sh
deleted
-23
@@ -1,23 +0,0 @@
1
-#!/bin/sh
2
-TOKEN=$(curl -s -L "https://public.ecr.aws/token?service=public.ecr.aws&scope=repository:q9t5s3a7/vllm-ci-postmerge-repo:pull" | jq -r .token)
3
-if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
4
- URL="https://public.ecr.aws/v2/q9t5s3a7/vllm-ci-postmerge-repo/manifests/$BUILDKITE_COMMIT"
5
-else
6
- URL="https://public.ecr.aws/v2/q9t5s3a7/vllm-ci-test-repo/manifests/$BUILDKITE_COMMIT"
7
-fi
8
-
9
-TIMEOUT_SECONDS=10
10
-
11
-retries=0
12
-while [ $retries -lt 1000 ]; do
13
- if [ "$(curl -s --max-time "$TIMEOUT_SECONDS" -L -H "Authorization: Bearer $TOKEN" -o /dev/null -w "%{http_code}" "$URL")" -eq 200 ]; then
14
- exit 0
15
- fi
16
-
17
- echo "Waiting for image to be available..."
18
-
19
- retries=$((retries + 1))
20
- sleep 5
21
-done
22
-
23
-exit 1
.buildkite/performance-benchmarks/README.md
renamed
+5
-49
@@ -2,40 +2,23 @@
2
3
## Introduction
4
5
-This directory contains two sets of benchmark for vllm.
6
-
7
-- Performance benchmark: benchmark vllm's performance under various workload, for **developers** to gain clarity on whether their PR improves/degrades vllm's performance
8
-- Nightly benchmark: compare vllm's performance against alternatives (tgi, trt-llm and lmdeploy), for **the public** to know when to choose vllm.
9
-
10
-See [vLLM performance dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm) for the latest performance benchmark results and [vLLM GitHub README](https://github.com/vllm-project/vllm/blob/main/README.md) for latest nightly benchmark results.
5
+This directory contains a benchmarking suite for **developers** to run locally and gain clarity on whether their PR improves/degrades vllm's performance.
6
+vLLM also maintains a continuous performance benchmark under [perf.vllm.ai](https://perf.vllm.ai/), hosted under PyTorch CI HUD.
7
8
## Performance benchmark quick overview
9
14
-**Benchmarking Coverage**: latency, throughput and fix-qps serving on A100 (the support for FP8 benchmark on H100 is coming!) and Intel® Xeon® Processors, with different models.
10
+**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100 and Intel® Xeon® Processors, with different models.
11
12
**Benchmarking Duration**: about 1hr.
13
14
**For benchmarking developers**: please try your best to constraint the duration of benchmarking to about 1 hr so that it won't take forever to run.
15
20
-## Nightly benchmark quick overview
21
-
22
-**Benchmarking Coverage**: Fix-qps serving on A100 (the support for FP8 benchmark on H100 is coming!) on Llama-3 8B, 70B and Mixtral 8x7B.
23
-
24
-**Benchmarking engines**: vllm, TGI, trt-llm and lmdeploy.
25
-
26
-**Benchmarking Duration**: about 3.5hrs.
27
-
16
## Trigger the benchmark
17
30
-Performance benchmark will be triggered when:
31
-
32
-- A PR being merged into vllm.
33
-- Every commit for those PRs with `perf-benchmarks` label AND `ready` label.
34
-
35
-Manually Trigger the benchmark
18
+The benchmark needs to be triggered manually:
19
20
```bash
38
-bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
21
+bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
22
```
23
24
Runtime environment variables:
@@ -47,10 +30,6 @@ Runtime environment variables:
30
- `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string.
31
- `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string.
32
50
-Nightly benchmark will be triggered when:
51
-
52
-- Every commit for those PRs with `perf-benchmarks` label and `nightly-benchmarks` label.
53
-
33
## Performance benchmark details
34
35
See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
@@ -152,26 +131,3 @@ Here is an example using the script to compare result_a and result_b with Model,
131
A comparison diagram will be generated below the table.
132
Here is an example to compare between 96c/results_gnr_96c_091_tp2pp3 and 128c/results_gnr_128c_091_tp2pp3
133
<img width="1886" height="828" alt="image" src="https://github.com/user-attachments/assets/c02a43ef-25d0-4fd6-90e5-2169a28682dd" />
155
-
156
-## Nightly test details
157
-
158
-See [nightly-descriptions.md](nightly-descriptions.md) for the detailed description on test workload, models and docker containers of benchmarking other llm engines.
159
-
160
-### Workflow
161
-
162
-- The [nightly-pipeline.yaml](nightly-pipeline.yaml) specifies the docker containers for different LLM serving engines.
163
-- Inside each container, we run [scripts/run-nightly-benchmarks.sh](scripts/run-nightly-benchmarks.sh), which will probe the serving engine of the current container.
164
-- The `scripts/run-nightly-benchmarks.sh` will parse the workload described in [nightly-tests.json](tests/nightly-tests.json) and launch the right benchmark for the specified serving engine via `scripts/launch-server.sh`.
165
-- At last, we run [scripts/summary-nightly-results.py](scripts/summary-nightly-results.py) to collect and plot the final benchmarking results, and update the results to buildkite.
166
-
167
-### Nightly tests
168
-
169
-In [nightly-tests.json](tests/nightly-tests.json), we include the command line arguments for benchmarking commands, together with the benchmarking test cases. The format is highly similar to performance benchmark.
170
-
171
-### Docker containers
172
-
173
-The docker containers for benchmarking are specified in `nightly-pipeline.yaml`.
174
-
175
-WARNING: the docker versions are HARD-CODED and SHOULD BE ALIGNED WITH `nightly-descriptions.md`. The docker versions need to be hard-coded as there are several version-specific bug fixes inside `scripts/run-nightly-benchmarks.sh` and `scripts/launch-server.sh`.
176
-
177
-WARNING: populating `trt-llm` to latest version is not easy, as it requires updating several protobuf files in [tensorrt-demo](https://github.com/neuralmagic/tensorrt-demo.git).
.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md
renamed
.buildkite/performance-benchmarks/scripts/compare-json-results.py
renamed
.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py
renamed
+1
-1
@@ -392,7 +392,7 @@ if __name__ == "__main__":
392
json_file = "benchmark_results.json"
393
with open(results_folder / md_file, "w") as f:
394
results = read_markdown(
395
- "../.buildkite/nightly-benchmarks/"
395
+ "../.buildkite/performance-benchmarks/"
396
+ "performance-benchmarks-descriptions.md"
397
)
398
results = results.format(
.buildkite/performance-benchmarks/scripts/launch-server.sh
renamed
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
renamed
+1
-1
@@ -469,7 +469,7 @@ main() {
469
ensure_sharegpt_downloaded
470
declare -g RESULTS_FOLDER=results/
471
mkdir -p $RESULTS_FOLDER
472
- QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/
472
+ QUICK_BENCHMARK_ROOT=../.buildkite/performance-benchmarks/
473
474
# dump vllm info via vllm collect-env
475
env_output=$(vllm collect-env)
.buildkite/performance-benchmarks/tests/genai-perf-tests.json
renamed
.buildkite/performance-benchmarks/tests/latency-tests-cpu.json
renamed
.buildkite/performance-benchmarks/tests/latency-tests.json
renamed
.buildkite/performance-benchmarks/tests/nightly-tests.json
renamed
.buildkite/performance-benchmarks/tests/serving-tests-cpu-snc2.json
renamed
.buildkite/performance-benchmarks/tests/serving-tests-cpu-snc3.json
renamed
.buildkite/performance-benchmarks/tests/serving-tests-cpu.json
renamed
.buildkite/performance-benchmarks/tests/serving-tests.json
renamed
.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json
renamed
.buildkite/performance-benchmarks/tests/throughput-tests.json
renamed
.github/mergify.yml
+1
-1
@@ -108,7 +108,7 @@ pull_request_rules:
108
- files~=^benchmarks/
109
- files~=^vllm/benchmarks/
110
- files~=^tests/benchmarks/
111
- - files~=^\.buildkite/nightly-benchmarks/
111
+ - files~=^\.buildkite/performance-benchmarks/
112
actions:
113
label:
114
add:
docs/contributing/benchmarks.md
+2
-11
@@ -9,7 +9,6 @@ vLLM provides comprehensive benchmarking tools for performance testing and evalu
9
- **[Benchmark CLI](#benchmark-cli)**: `vllm bench` CLI tools and specialized benchmark scripts for interactive performance testing
10
- **[Parameter sweeps](#parameter-sweeps)**: Automate `vllm bench` runs for multiple configurations
11
- **[Performance benchmarks](#performance-benchmarks)**: Automated CI benchmarks for development
12
-- **[Nightly benchmarks](#nightly-benchmarks)**: Comparative benchmarks against alternatives
12
13
[Benchmark CLI]: #benchmark-cli
14
@@ -1167,7 +1166,7 @@ docker run -it --entrypoint /bin/bash -v /data/huggingface:/root/.cache/huggingf
1166
Then, run below command inside the docker instance.
1167
1168
```bash
1170
-bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
1169
+bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
1170
```
1171
1172
When run, benchmark script generates results under **benchmark/results** folder, along with the benchmark_results.md and benchmark_results.json.
@@ -1185,7 +1184,7 @@ For more results visualization, check the [visualizing the results](https://gith
1184
1185
The latest performance results are hosted on the public [vLLM Performance Dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm).
1186
1188
-More information on the performance benchmarks and their parameters can be found in [Benchmark README](https://github.com/intel-ai-tce/vllm/blob/more_cpu_models/.buildkite/nightly-benchmarks/README.md) and [performance benchmark description](../../.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md).
1187
+More information on the performance benchmarks and their parameters can be found in [Benchmark README](https://github.com/intel-ai-tce/vllm/blob/more_cpu_models/.buildkite/nightly-benchmarks/README.md) and [performance benchmark description](../../.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md).
1188
1189
### Continuous Benchmarking
1190
@@ -1210,11 +1209,3 @@ The benchmarking currently runs on a predefined set of models configured in the
1209
#### Viewing Results
1210
1211
All continuous benchmarking results are automatically published to the public [vLLM Performance Dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm).
1213
-
1214
-## Nightly Benchmarks
1215
-
1216
-These compare vLLM's performance against alternatives (`tgi`, `trt-llm`, and `lmdeploy`) when there are major updates of vLLM (e.g., bumping up to a new version). They are primarily intended for consumers to evaluate when to choose vLLM over other options and are triggered on every commit with both the `perf-benchmarks` and `nightly-benchmarks` labels.
1217
-
1218
-The latest nightly benchmark results are shared in major release blog posts such as [vLLM v0.6.0](https://blog.vllm.ai/2024/09/05/perf-update.html).
1219
-
1220
-More information on the nightly benchmarks and their parameters can be found [here](../../.buildkite/nightly-benchmarks/nightly-descriptions.md).