[CI][AMD] Persist the openai-harmony tiktoken vocab cache across jobs (#51666)
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
stefankoncarevic committed
Aug 11, 2026 at 08:11 UTC
75903e506743180c391bc01ad9bf06a1bab68048
2 files changed
+44
-5
.buildkite/scripts/ci-bake-rocm.sh
+37
-4
@@ -302,6 +302,8 @@ validate_ci_build_context_source() {
302
local mode=""
303
local stage=""
304
local path=""
305
+ local worktree_diff_status=0
306
+ local staged_diff_status=0
307
local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes"
308
local info_attributes=""
309
@@ -319,15 +321,46 @@ validate_ci_build_context_source() {
321
return 1
322
fi
323
ROCM_BUILD_CONTEXT_COMMIT="${context_commit}"
322
- if ! git -C "${source_root}" diff \
323
- --quiet --no-ext-diff --ignore-submodules=none --; then
324
+ # Shared AMD workspaces may present tracked files with inflated executable
325
+ # bits. Context modes come from the pinned Git tree, so ignore only that
326
+ # filesystem drift while continuing to reject content changes.
327
+ git -C "${source_root}" -c core.fileMode=false diff \
328
+ --quiet --no-ext-diff --ignore-submodules=none -- \
329
+ || worktree_diff_status=$?
330
+ if (( worktree_diff_status != 0 )); then
331
+ if (( worktree_diff_status > 1 )); then
332
+ printf 'Failed to inspect tracked CI worktree changes (git diff exited %s)\n' \
333
+ "${worktree_diff_status}" >&2
334
+ return 1
335
+ fi
336
echo "Tracked worktree changes cannot be omitted from the CI Docker context" \
337
>&2
338
+ echo "Tracked worktree diff (first 50 entries):" >&2
339
+ git -C "${source_root}" -c core.fileMode=false diff \
340
+ --no-ext-diff --ignore-submodules=none --name-status -- \
341
+ | sed -n '1,50p' >&2 || true
342
+ git -C "${source_root}" -c core.fileMode=false diff \
343
+ --no-ext-diff --ignore-submodules=none --summary -- \
344
+ | sed -n '1,50p' >&2 || true
345
return 1
346
fi
328
- if ! git -C "${source_root}" diff --cached \
329
- --quiet --no-ext-diff --ignore-submodules=none HEAD --; then
347
+ git -C "${source_root}" diff --cached \
348
+ --quiet --no-ext-diff --ignore-submodules=none HEAD -- \
349
+ || staged_diff_status=$?
350
+ if (( staged_diff_status != 0 )); then
351
+ if (( staged_diff_status > 1 )); then
352
+ printf 'Failed to inspect staged CI changes (git diff exited %s)\n' \
353
+ "${staged_diff_status}" >&2
354
+ return 1
355
+ fi
356
echo "Staged changes cannot be omitted from the CI Docker context" >&2
357
+ echo "Staged diff (first 50 entries):" >&2
358
+ git -C "${source_root}" diff --cached \
359
+ --no-ext-diff --ignore-submodules=none --name-status HEAD -- \
360
+ | sed -n '1,50p' >&2 || true
361
+ git -C "${source_root}" diff --cached \
362
+ --no-ext-diff --ignore-submodules=none --summary HEAD -- \
363
+ | sed -n '1,50p' >&2 || true
364
return 1
365
fi
366
# Untracked and ignored worker outputs are intentionally absent: the
.buildkite/scripts/hardware_ci/run-amd-test.sh
+7
-1
@@ -429,11 +429,15 @@ initialize_native_environment() {
429
# datasets uses POSIX locks that are unsupported by the shared HF NFS cache.
430
# Keep processed datasets job-local while retaining the persistent Hub cache.
431
HF_DATASETS_CACHE="${native_root}/cache/huggingface/datasets"
432
+ # openai-harmony downloads its tiktoken vocab on first use and caches it under
433
+ # $TMPDIR by default, which is job-local; keep it with the persistent Hub cache.
434
+ TIKTOKEN_RS_CACHE_DIR="${HF_HOME}/tiktoken-rs-cache"
435
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
436
: "${HF_HUB_ETAG_TIMEOUT:=60}"
437
export TMPDIR VLLM_RPC_BASE_PATH
438
export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
439
export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
440
+ export TIKTOKEN_RS_CACHE_DIR
441
export PYTORCH_ROCM_ARCH=""
442
443
mkdir -p "${TMPDIR}" \
@@ -442,6 +446,7 @@ initialize_native_environment() {
446
"${VLLM_CACHE_ROOT}" \
447
"${XDG_CACHE_HOME}" \
448
"${HF_HOME}" \
449
+ "${TIKTOKEN_RS_CACHE_DIR}" \
450
"${HF_DATASETS_CACHE}" || return 1
451
452
echo "Native compile caches: VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT} TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR}"
@@ -1092,7 +1097,7 @@ container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}"
1097
container_job_id_short="${container_job_id:0:8}"
1098
CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}"
1099
CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache"
1095
-CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
1100
+CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TIKTOKEN_RS_CACHE_DIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
1101
1102
# Verify GPU access
1103
render_gid=$(getent group render | cut -d: -f3)
@@ -1210,6 +1215,7 @@ else
1215
-e "HF_HOME=${HF_MOUNT}" \
1216
-e "PYTHONPATH=${MYPYTHONPATH}" \
1217
-e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \
1218
+ -e "TIKTOKEN_RS_CACHE_DIR=${HF_MOUNT}/tiktoken-rs-cache" \
1219
-e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \
1220
-e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \
1221
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \