A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[CI] fix silent error in nightly wheel index generation script, add generation time to HTML index (#30060)
Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
Shengqi Chen committed
Dec 5, 2025 at 08:48 UTC
aaddc9c82a6df73f0f93912d3aee987859d28a53
2 files changed
+30
-8
.buildkite/scripts/generate-nightly-index.py
+26
-7
@@ -9,6 +9,7 @@ import argparse
9
import json
10
import sys
11
from dataclasses import asdict, dataclass
12
+from datetime import datetime
13
from pathlib import Path
14
from typing import Any
15
from urllib.parse import quote
@@ -20,6 +21,7 @@ if not sys.version_info >= (3, 12):
21
22
INDEX_HTML_TEMPLATE = """<!DOCTYPE html>
23
<html>
24
+ <!-- {comment} -->
25
<meta name="pypi:repository-version" content="1.0">
26
<body>
27
{items}
@@ -90,7 +92,7 @@ def parse_from_filename(file: str) -> WheelFileInfo:
92
)
93
94
93
-def generate_project_list(subdir_names: list[str]) -> str:
95
+def generate_project_list(subdir_names: list[str], comment: str = "") -> str:
96
"""
97
Generate project list HTML content linking to each project & variant sub-directory.
98
"""
@@ -98,11 +100,14 @@ def generate_project_list(subdir_names: list[str]) -> str:
100
for name in sorted(subdir_names):
101
name = name.strip("/").strip(".")
102
href_tags.append(f' <a href="{name}/">{name}/</a><br/>')
101
- return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags))
103
+ return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
104
105
106
def generate_package_index_and_metadata(
105
- wheel_files: list[WheelFileInfo], wheel_base_dir: Path, index_base_dir: Path
107
+ wheel_files: list[WheelFileInfo],
108
+ wheel_base_dir: Path,
109
+ index_base_dir: Path,
110
+ comment: str = "",
111
) -> tuple[str, str]:
112
"""
113
Generate package index HTML content for a specific package, linking to actual wheel files.
@@ -120,7 +125,7 @@ def generate_package_index_and_metadata(
125
file_meta = asdict(file)
126
file_meta["path"] = file_path_quoted
127
metadata.append(file_meta)
123
- index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags))
128
+ index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
129
metadata_str = json.dumps(metadata, indent=2)
130
return index_str, metadata_str
131
@@ -131,6 +136,7 @@ def generate_index_and_metadata(
136
index_base_dir: Path,
137
default_variant: str | None = None,
138
alias_to_default: str | None = None,
139
+ comment: str = "",
140
):
141
"""
142
Generate index for all wheel files.
@@ -141,6 +147,7 @@ def generate_index_and_metadata(
147
index_base_dir (Path): Base directory to store index files.
148
default_variant (str | None): The default variant name, if any.
149
alias_to_default (str | None): Alias variant name for the default variant, if any.
150
+ comment (str | None): Optional comment to include in the generated HTML files.
151
152
First, parse all wheel files to extract metadata.
153
We need to collect all wheel files for each variant, and generate an index for it (in a sub-directory).
@@ -234,6 +241,10 @@ def generate_index_and_metadata(
241
variant_to_files[alias_to_default] = variant_to_files["default"].copy()
242
print(f"Alias variant '{alias_to_default}' created for default variant.")
243
244
+ # Generate comment in HTML header
245
+ comment_str = f" ({comment})" if comment else ""
246
+ comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}"
247
+
248
# Generate index for each variant
249
subdir_names = set()
250
for variant, files in variant_to_files.items():
@@ -253,7 +264,7 @@ def generate_index_and_metadata(
264
subdir_names = subdir_names.union(packages)
265
else:
266
# generate project list for this variant directly
256
- project_list_str = generate_project_list(sorted(packages))
267
+ project_list_str = generate_project_list(sorted(packages), comment_tmpl)
268
with open(variant_dir / "index.html", "w") as f:
269
f.write(project_list_str)
270
@@ -263,7 +274,7 @@ def generate_index_and_metadata(
274
package_dir = variant_dir / package
275
package_dir.mkdir(parents=True, exist_ok=True)
276
index_str, metadata_str = generate_package_index_and_metadata(
266
- package_files, wheel_base_dir, package_dir
277
+ package_files, wheel_base_dir, package_dir, comment
278
)
279
with open(package_dir / "index.html", "w") as f:
280
f.write(index_str)
@@ -271,7 +282,7 @@ def generate_index_and_metadata(
282
f.write(metadata_str)
283
284
# Generate top-level project list index
274
- project_list_str = generate_project_list(sorted(subdir_names))
285
+ project_list_str = generate_project_list(sorted(subdir_names), comment_tmpl)
286
with open(index_base_dir / "index.html", "w") as f:
287
f.write(project_list_str)
288
@@ -283,6 +294,7 @@ if __name__ == "__main__":
294
--current-objects <path_to_json> : path to JSON file containing current S3 objects listing in this version directory
295
--output-dir <output_directory> : directory to store generated index files
296
--alias-to-default <alias_variant_name> : (optional) alias variant name for the default variant
297
+ --comment <comment_string> : (optional) comment string to include in generated HTML files
298
"""
299
300
parser = argparse.ArgumentParser(
@@ -312,6 +324,12 @@ if __name__ == "__main__":
324
default=None,
325
help="Alias variant name for the default variant",
326
)
327
+ parser.add_argument(
328
+ "--comment",
329
+ type=str,
330
+ default="",
331
+ help="Optional comment string to include in generated HTML files",
332
+ )
333
334
args = parser.parse_args()
335
@@ -366,5 +384,6 @@ if __name__ == "__main__":
384
index_base_dir=index_base_dir,
385
default_variant=None,
386
alias_to_default=args.alias_to_default,
387
+ comment=args.comment.strip(),
388
)
389
print(f"Successfully generated index and metadata in {output_dir}")
.buildkite/scripts/upload-wheels.sh
+4
-1
@@ -81,7 +81,10 @@ else
81
alias_arg=""
82
fi
83
84
-$PYTHON pip install regex && .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" $alias_arg
84
+# HACK: we do not need regex module here, but it is required by pre-commit hook
85
+# To avoid any external dependency, we simply replace it back to the stdlib re module
86
+sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
87
+$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" $alias_arg
88
89
# copy indices to /<commit>/ unconditionally
90
echo "Uploading indices to $S3_COMMIT_PREFIX"