A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Convert `.buildkite` to `ruff format` (#17656)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 13, 2025 at 10:28 UTC
98fcba1575da8d80e47d0540898015d2906d4720
11 files changed
+182
-114
.buildkite/check-wheel-size.py
+12
-8
@@ -8,12 +8,12 @@ import zipfile
8
# Note that we have 400 MiB quota, please use it wisely.
9
# See https://github.com/pypi/support/issues/3792 .
10
# Please also sync the value with the one in Dockerfile.
11
-VLLM_MAX_SIZE_MB = int(os.environ.get('VLLM_MAX_SIZE_MB', 400))
11
+VLLM_MAX_SIZE_MB = int(os.environ.get("VLLM_MAX_SIZE_MB", 400))
12
13
14
def print_top_10_largest_files(zip_file):
15
"""Print the top 10 largest files in the given zip file."""
16
- with zipfile.ZipFile(zip_file, 'r') as z:
16
+ with zipfile.ZipFile(zip_file, "r") as z:
17
file_sizes = [(f, z.getinfo(f).file_size) for f in z.namelist()]
18
file_sizes.sort(key=lambda x: x[1], reverse=True)
19
for f, size in file_sizes[:10]:
@@ -28,14 +28,18 @@ def check_wheel_size(directory):
28
wheel_path = os.path.join(root, file_name)
29
wheel_size_mb = os.path.getsize(wheel_path) / (1024 * 1024)
30
if wheel_size_mb > VLLM_MAX_SIZE_MB:
31
- print(f"Not allowed: Wheel {wheel_path} is larger "
32
- f"({wheel_size_mb:.2f} MB) than the limit "
33
- f"({VLLM_MAX_SIZE_MB} MB).")
31
+ print(
32
+ f"Not allowed: Wheel {wheel_path} is larger "
33
+ f"({wheel_size_mb:.2f} MB) than the limit "
34
+ f"({VLLM_MAX_SIZE_MB} MB)."
35
+ )
36
print_top_10_largest_files(wheel_path)
37
return 1
38
else:
37
- print(f"Wheel {wheel_path} is within the allowed size "
38
- f"({wheel_size_mb:.2f} MB).")
39
+ print(
40
+ f"Wheel {wheel_path} is within the allowed size "
41
+ f"({wheel_size_mb:.2f} MB)."
42
+ )
43
return 0
44
45
@@ -45,4 +49,4 @@ if __name__ == "__main__":
49
sys.exit(1)
50
51
directory = sys.argv[1]
48
- sys.exit(check_wheel_size(directory))
\ No newline at end of file
52
+ sys.exit(check_wheel_size(directory))
.buildkite/generate_index.py
+2
-2
@@ -22,5 +22,5 @@ with open("index.html", "w") as f:
22
print(f"Generated index.html for {args.wheel}")
23
# cloudfront requires escaping the '+' character
24
f.write(
25
- template.format(wheel=filename,
26
- wheel_html_escaped=filename.replace("+", "%2B")))
25
+ template.format(wheel=filename, wheel_html_escaped=filename.replace("+", "%2B"))
26
+ )
.buildkite/lm-eval-harness/conftest.py
+10
-6
@@ -8,11 +8,14 @@ def pytest_addoption(parser):
8
parser.addoption(
9
"--config-list-file",
10
action="store",
11
- help="Path to the file listing model config YAMLs (one per line)")
12
- parser.addoption("--tp-size",
13
- action="store",
14
- default="1",
15
- help="Tensor parallel size to use for evaluation")
11
+ help="Path to the file listing model config YAMLs (one per line)",
12
+ )
13
+ parser.addoption(
14
+ "--tp-size",
15
+ action="store",
16
+ default="1",
17
+ help="Tensor parallel size to use for evaluation",
18
+ )
19
20
21
@pytest.fixture(scope="session")
@@ -33,7 +36,8 @@ def pytest_generate_tests(metafunc):
36
config_dir = config_list_file.parent
37
with open(config_list_file, encoding="utf-8") as f:
38
configs = [
36
- config_dir / line.strip() for line in f
39
+ config_dir / line.strip()
40
+ for line in f
41
if line.strip() and not line.startswith("#")
42
]
43
metafunc.parametrize("config_filename", configs)
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+15
-11
@@ -16,19 +16,22 @@ RTOL = 0.08
16
17
18
def launch_lm_eval(eval_config, tp_size):
19
- trust_remote_code = eval_config.get('trust_remote_code', False)
20
- model_args = f"pretrained={eval_config['model_name']}," \
21
- f"tensor_parallel_size={tp_size}," \
22
- f"enforce_eager=true," \
23
- f"add_bos_token=true," \
24
- f"trust_remote_code={trust_remote_code}"
19
+ trust_remote_code = eval_config.get("trust_remote_code", False)
20
+ model_args = (
21
+ f"pretrained={eval_config['model_name']},"
22
+ f"tensor_parallel_size={tp_size},"
23
+ f"enforce_eager=true,"
24
+ f"add_bos_token=true,"
25
+ f"trust_remote_code={trust_remote_code}"
26
+ )
27
results = lm_eval.simple_evaluate(
28
model="vllm",
29
model_args=model_args,
30
tasks=[task["name"] for task in eval_config["tasks"]],
31
num_fewshot=eval_config["num_fewshot"],
32
limit=eval_config["limit"],
31
- batch_size="auto")
33
+ batch_size="auto",
34
+ )
35
return results
36
37
@@ -42,9 +45,10 @@ def test_lm_eval_correctness_param(config_filename, tp_size):
45
for metric in task["metrics"]:
46
ground_truth = metric["value"]
47
measured_value = results["results"][task["name"]][metric["name"]]
45
- print(f'{task["name"]} | {metric["name"]}: '
46
- f'ground_truth={ground_truth} | measured={measured_value}')
47
- success = success and np.isclose(
48
- ground_truth, measured_value, rtol=RTOL)
48
+ print(
49
+ f"{task['name']} | {metric['name']}: "
50
+ f"ground_truth={ground_truth} | measured={measured_value}"
51
+ )
52
+ success = success and np.isclose(ground_truth, measured_value, rtol=RTOL)
53
54
assert success
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+46
-43
@@ -65,18 +65,18 @@ def read_markdown(file):
65
66
67
def results_to_json(latency, throughput, serving):
68
- return json.dumps({
69
- 'latency': latency.to_dict(),
70
- 'throughput': throughput.to_dict(),
71
- 'serving': serving.to_dict()
72
- })
68
+ return json.dumps(
69
+ {
70
+ "latency": latency.to_dict(),
71
+ "throughput": throughput.to_dict(),
72
+ "serving": serving.to_dict(),
73
+ }
74
+ )
75
76
77
if __name__ == "__main__":
76
-
78
# collect results
79
for test_file in results_folder.glob("*.json"):
79
-
80
with open(test_file) as f:
81
raw_result = json.loads(f.read())
82
@@ -120,7 +120,8 @@ if __name__ == "__main__":
120
for perc in [10, 25, 50, 75, 90, 99]:
121
# Multiply 1000 to convert the time unit from s to ms
122
raw_result.update(
123
- {f"P{perc}": 1000 * raw_result["percentiles"][str(perc)]})
123
+ {f"P{perc}": 1000 * raw_result["percentiles"][str(perc)]}
124
+ )
125
raw_result["avg_latency"] = raw_result["avg_latency"] * 1000
126
127
# add the result to raw_result
@@ -153,26 +154,27 @@ if __name__ == "__main__":
154
serving_results = pd.DataFrame.from_dict(serving_results)
155
throughput_results = pd.DataFrame.from_dict(throughput_results)
156
156
- raw_results_json = results_to_json(latency_results, throughput_results,
157
- serving_results)
157
+ raw_results_json = results_to_json(
158
+ latency_results, throughput_results, serving_results
159
+ )
160
161
# remapping the key, for visualization purpose
162
if not latency_results.empty:
161
- latency_results = latency_results[list(
162
- latency_column_mapping.keys())].rename(
163
- columns=latency_column_mapping)
163
+ latency_results = latency_results[list(latency_column_mapping.keys())].rename(
164
+ columns=latency_column_mapping
165
+ )
166
if not serving_results.empty:
165
- serving_results = serving_results[list(
166
- serving_column_mapping.keys())].rename(
167
- columns=serving_column_mapping)
167
+ serving_results = serving_results[list(serving_column_mapping.keys())].rename(
168
+ columns=serving_column_mapping
169
+ )
170
if not throughput_results.empty:
169
- throughput_results = throughput_results[list(
170
- throughput_results_column_mapping.keys())].rename(
171
- columns=throughput_results_column_mapping)
171
+ throughput_results = throughput_results[
172
+ list(throughput_results_column_mapping.keys())
173
+ ].rename(columns=throughput_results_column_mapping)
174
173
- processed_results_json = results_to_json(latency_results,
174
- throughput_results,
175
- serving_results)
175
+ processed_results_json = results_to_json(
176
+ latency_results, throughput_results, serving_results
177
+ )
178
179
for df in [latency_results, serving_results, throughput_results]:
180
if df.empty:
@@ -184,38 +186,39 @@ if __name__ == "__main__":
186
# The GPUs sometimes come in format of "GPUTYPE\nGPUTYPE\n...",
187
# we want to turn it into "8xGPUTYPE"
188
df["GPU"] = df["GPU"].apply(
187
- lambda x: f"{len(x.split('\n'))}x{x.split('\n')[0]}")
189
+ lambda x: f"{len(x.split('\n'))}x{x.split('\n')[0]}"
190
+ )
191
192
# get markdown tables
190
- latency_md_table = tabulate(latency_results,
191
- headers='keys',
192
- tablefmt='pipe',
193
- showindex=False)
194
- serving_md_table = tabulate(serving_results,
195
- headers='keys',
196
- tablefmt='pipe',
197
- showindex=False)
198
- throughput_md_table = tabulate(throughput_results,
199
- headers='keys',
200
- tablefmt='pipe',
201
- showindex=False)
193
+ latency_md_table = tabulate(
194
+ latency_results, headers="keys", tablefmt="pipe", showindex=False
195
+ )
196
+ serving_md_table = tabulate(
197
+ serving_results, headers="keys", tablefmt="pipe", showindex=False
198
+ )
199
+ throughput_md_table = tabulate(
200
+ throughput_results, headers="keys", tablefmt="pipe", showindex=False
201
+ )
202
203
# document the result
204
with open(results_folder / "benchmark_results.md", "w") as f:
205
-
206
- results = read_markdown("../.buildkite/nightly-benchmarks/" +
207
- "performance-benchmarks-descriptions.md")
205
+ results = read_markdown(
206
+ "../.buildkite/nightly-benchmarks/"
207
+ + "performance-benchmarks-descriptions.md"
208
+ )
209
results = results.format(
210
latency_tests_markdown_table=latency_md_table,
211
throughput_tests_markdown_table=throughput_md_table,
212
serving_tests_markdown_table=serving_md_table,
212
- benchmarking_results_in_json_string=processed_results_json)
213
+ benchmarking_results_in_json_string=processed_results_json,
214
+ )
215
f.write(results)
216
217
# document benchmarking results in json
218
with open(results_folder / "benchmark_results.json", "w") as f:
217
-
218
- results = latency_results.to_dict(
219
- orient='records') + throughput_results.to_dict(
220
- orient='records') + serving_results.to_dict(orient='records')
219
+ results = (
220
+ latency_results.to_dict(orient="records")
221
+ + throughput_results.to_dict(orient="records")
222
+ + serving_results.to_dict(orient="records")
223
+ )
224
f.write(json.dumps(results))
.buildkite/nightly-benchmarks/scripts/download-tokenizer.py
+6
-9
@@ -14,15 +14,12 @@ def main(model, cachedir):
14
15
if __name__ == "__main__":
16
parser = argparse.ArgumentParser(
17
- description="Download and save Hugging Face tokenizer")
18
- parser.add_argument("--model",
19
- type=str,
20
- required=True,
21
- help="Name of the model")
22
- parser.add_argument("--cachedir",
23
- type=str,
24
- required=True,
25
- help="Directory to save the tokenizer")
17
+ description="Download and save Hugging Face tokenizer"
18
+ )
19
+ parser.add_argument("--model", type=str, required=True, help="Name of the model")
20
+ parser.add_argument(
21
+ "--cachedir", type=str, required=True, help="Directory to save the tokenizer"
22
+ )
23
24
args = parser.parse_args()
25
main(args.model, args.cachedir)
.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py
+20
-21
@@ -11,33 +11,33 @@ from tabulate import tabulate
11
12
def parse_arguments():
13
parser = argparse.ArgumentParser(
14
- description=
15
- 'Parse command line arguments for summary-nightly-results script.')
16
- parser.add_argument('--results-folder',
17
- type=str,
18
- required=True,
19
- help='The folder where the results are stored.')
20
- parser.add_argument('--description',
21
- type=str,
22
- required=True,
23
- help='Description of the results.')
14
+ description="Parse command line arguments for summary-nightly-results script."
15
+ )
16
+ parser.add_argument(
17
+ "--results-folder",
18
+ type=str,
19
+ required=True,
20
+ help="The folder where the results are stored.",
21
+ )
22
+ parser.add_argument(
23
+ "--description", type=str, required=True, help="Description of the results."
24
+ )
25
26
args = parser.parse_args()
27
return args
28
29
30
def get_perf(df, method, model, metric):
30
-
31
means = []
32
33
for qps in [2, 4, 8, 16, "inf"]:
34
- target = df['Test name'].str.contains(model)
35
- target = target & df['Engine'].str.contains(method)
36
- target = target & df['Test name'].str.contains("qps_" + str(qps))
34
+ target = df["Test name"].str.contains(model)
35
+ target = target & df["Engine"].str.contains(method)
36
+ target = target & df["Test name"].str.contains("qps_" + str(qps))
37
filtered_df = df[target]
38
39
if filtered_df.empty:
40
- means.append(0.)
40
+ means.append(0.0)
41
else:
42
means.append(filtered_df[metric].values[0])
43
@@ -45,7 +45,6 @@ def get_perf(df, method, model, metric):
45
46
47
def get_perf_w_std(df, method, model, metric):
48
-
48
if metric in ["TTFT", "ITL"]:
49
mean = get_perf(df, method, model, "Mean " + metric + " (ms)")
50
mean = mean.tolist()
@@ -60,7 +59,8 @@ def get_perf_w_std(df, method, model, metric):
59
else:
60
assert metric == "Tput"
61
mean = get_perf(df, method, model, "Input Tput (tok/s)") + get_perf(
63
- df, method, model, "Output Tput (tok/s)")
62
+ df, method, model, "Output Tput (tok/s)"
63
+ )
64
mean = mean.tolist()
65
std = None
66
@@ -80,18 +80,17 @@ def main(args):
80
# generate markdown table
81
df = pd.DataFrame.from_dict(results)
82
83
- md_table = tabulate(df, headers='keys', tablefmt='pipe', showindex=False)
83
+ md_table = tabulate(df, headers="keys", tablefmt="pipe", showindex=False)
84
85
with open(args.description) as f:
86
description = f.read()
87
88
- description = description.format(
89
- nightly_results_benchmarking_table=md_table)
88
+ description = description.format(nightly_results_benchmarking_table=md_table)
89
90
with open("nightly_results.md", "w") as f:
91
f.write(description)
92
93
95
-if __name__ == '__main__':
94
+if __name__ == "__main__":
95
args = parse_arguments()
96
main(args)
.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py
+10
-14
@@ -34,10 +34,8 @@ serving_column_mapping = {
34
}
35
36
if __name__ == "__main__":
37
-
37
# collect results
38
for test_file in results_folder.glob("*.json"):
40
-
39
with open(test_file) as f:
40
raw_result = json.loads(f.read())
41
@@ -56,17 +54,16 @@ if __name__ == "__main__":
54
serving_results = pd.DataFrame.from_dict(serving_results)
55
56
if not serving_results.empty:
59
- serving_results = serving_results[list(
60
- serving_column_mapping.keys())].rename(
61
- columns=serving_column_mapping)
57
+ serving_results = serving_results[list(serving_column_mapping.keys())].rename(
58
+ columns=serving_column_mapping
59
+ )
60
63
- serving_md_table_with_headers = tabulate(serving_results,
64
- headers='keys',
65
- tablefmt='pipe',
66
- showindex=False)
61
+ serving_md_table_with_headers = tabulate(
62
+ serving_results, headers="keys", tablefmt="pipe", showindex=False
63
+ )
64
# remove the first line of header
68
- serving_md_table_lines = serving_md_table_with_headers.split('\n')
69
- serving_md_table_without_header = '\n'.join(serving_md_table_lines[2:])
65
+ serving_md_table_lines = serving_md_table_with_headers.split("\n")
66
+ serving_md_table_without_header = "\n".join(serving_md_table_lines[2:])
67
68
prefix = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
69
prefix = prefix + "_" + os.environ.get("CURRENT_LLM_SERVING_ENGINE")
@@ -76,10 +73,9 @@ if __name__ == "__main__":
73
# document results with header.
74
# for those who wants to reproduce our benchmark.
75
f.write(serving_md_table_with_headers)
79
- f.write('\n')
76
+ f.write("\n")
77
78
# document benchmarking results in json
79
with open(results_folder / f"{prefix}_nightly_results.json", "w") as f:
83
-
84
- results = serving_results.to_dict(orient='records')
80
+ results = serving_results.to_dict(orient="records")
81
f.write(json.dumps(results))
.buildkite/pyproject.toml
new
+55
@@ -0,0 +1,55 @@
1
+# This local pyproject file is part of the migration from yapf to ruff format.
2
+# It uses the same core rules as the main pyproject.toml file, but with the
3
+# following differences:
4
+# - isort profile is set to black
5
+# - ruff line length is overridden to 88
6
+# - deprecated typing ignores (UP006, UP035) have been removed
7
+
8
+[tool.isort]
9
+profile = "black"
10
+
11
+[tool.ruff]
12
+line-length = 88
13
+exclude = [
14
+ # External file, leaving license intact
15
+ "examples/other/fp8/quantizer/quantize.py",
16
+ "vllm/vllm_flash_attn/flash_attn_interface.pyi"
17
+]
18
+
19
+[tool.ruff.lint.per-file-ignores]
20
+"vllm/third_party/**" = ["ALL"]
21
+"vllm/version.py" = ["F401"]
22
+"vllm/_version.py" = ["ALL"]
23
+
24
+[tool.ruff.lint]
25
+select = [
26
+ # pycodestyle
27
+ "E",
28
+ # Pyflakes
29
+ "F",
30
+ # pyupgrade
31
+ "UP",
32
+ # flake8-bugbear
33
+ "B",
34
+ # flake8-simplify
35
+ "SIM",
36
+ # isort
37
+ "I",
38
+ # flake8-logging-format
39
+ "G",
40
+]
41
+ignore = [
42
+ # star imports
43
+ "F405", "F403",
44
+ # lambda expression assignment
45
+ "E731",
46
+ # Loop control variable not used within loop body
47
+ "B007",
48
+ # f-string format
49
+ "UP032",
50
+ # Can remove once 3.10+ is the minimum Python version
51
+ "UP007",
52
+]
53
+
54
+[tool.ruff.format]
55
+docstring-code-format = true
.pre-commit-config.yaml
+4
@@ -16,6 +16,8 @@ repos:
16
hooks:
17
- id: ruff
18
args: [--output-format, github, --fix]
19
+ - id: ruff-format
20
+ files: ^(.buildkite).*
21
- repo: https://github.com/codespell-project/codespell
22
rev: v2.4.1
23
hooks:
@@ -26,6 +28,8 @@ repos:
28
rev: 6.0.1
29
hooks:
30
- id: isort
31
+ # necessary during the transition from yapf to ruff format
32
+ args: [--resolve-all-configs, --config-root, .]
33
- repo: https://github.com/pre-commit/mirrors-clang-format
34
rev: v20.1.3
35
hooks:
pyproject.toml
+2
@@ -53,6 +53,7 @@ include = ["vllm*"]
53
54
[tool.yapfignore]
55
ignore_patterns = [
56
+ ".buildkite/**",
57
"build/**",
58
]
59
@@ -107,6 +108,7 @@ select = [
108
"SIM",
109
# isort
110
# "I",
111
+ # flake8-logging-format
112
"G",
113
]
114
ignore = [