| 1 | #!/bin/bash |
| 2 | |
| 3 | # This script runs tests inside the Intel XPU docker container. |
| 4 | # It mirrors the structure of run-amd-test.sh while keeping Intel-specific |
| 5 | # container setup and allowing commands to be sourced from YAML or env. |
| 6 | # |
| 7 | # Command sources (in priority order): |
| 8 | # 1) VLLM_TEST_COMMANDS env var (preferred, preserves quoting) |
| 9 | # 2) Positional args (legacy) |
| 10 | # 3) One or more YAML files with a commands list (test-area style) |
| 11 | ############################################################################### |
| 12 | set -o pipefail |
| 13 | |
| 14 | DRY_RUN=${DRY_RUN:-0} |
| 15 | if [[ "${1:-}" == "--dry-run" ]]; then |
| 16 | DRY_RUN=1 |
| 17 | shift |
| 18 | fi |
| 19 | |
| 20 | # Export Python path |
| 21 | # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed |
| 22 | export PYTHONPATH=".." |
| 23 | export VLLM_DISABLE_COMPILE_CACHE=1 |
| 24 | |
| 25 | if [[ "${BUILDKITE_PARALLEL_JOB_COUNT:-1}" -gt 1 ]]; then |
| 26 | PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--num-shards=${BUILDKITE_PARALLEL_JOB_COUNT} --shard-id=${BUILDKITE_PARALLEL_JOB:-0}" |
| 27 | export PYTEST_ADDOPTS |
| 28 | fi |
| 29 | |
| 30 | ############################################################################### |
| 31 | # Helper Functions |
| 32 | ############################################################################### |
| 33 | |
| 34 | cleanup_docker() { |
| 35 | # Share the same lock with image pull to avoid cleanup/pull races on one node. |
| 36 | local docker_lock="/tmp/docker-pull.lock" |
| 37 | exec 9>"$docker_lock" |
| 38 | flock 9 |
| 39 | |
| 40 | docker_root=$(docker info -f '{{.DockerRootDir}}') |
| 41 | if [ -z "$docker_root" ]; then |
| 42 | echo "Failed to determine Docker root directory." >&2 |
| 43 | flock -u 9 |
| 44 | return 1 |
| 45 | fi |
| 46 | echo "Docker root directory: $docker_root" |
| 47 | |
| 48 | disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') |
| 49 | threshold=70 |
| 50 | if [ "$disk_usage" -gt "$threshold" ]; then |
| 51 | echo "Disk usage is above $threshold%. Running aggressive CI image cleanup..." |
| 52 | cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 1 |
| 53 | else |
| 54 | echo "Disk usage is below $threshold%. Checking old CI images anyway." |
| 55 | cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 0 |
| 56 | fi |
| 57 | echo "Old CI image cleanup completed." |
| 58 | |
| 59 | flock -u 9 |
| 60 | } |
| 61 | |
| 62 | cleanup_old_ci_images() { |
| 63 | local repo_prefix="$1" |
| 64 | local current_image_ref="$2" |
| 65 | local ttl_hours="$3" |
| 66 | local aggressive_cleanup="$4" |
| 67 | |
| 68 | if [[ -z "$repo_prefix" || "$repo_prefix" == "/" ]]; then |
| 69 | echo "Skip old-image cleanup: invalid repo prefix '${repo_prefix}'" |
| 70 | return 0 |
| 71 | fi |
| 72 | |
| 73 | if ! [[ "$ttl_hours" =~ ^[0-9]+$ ]]; then |
| 74 | echo "Invalid DOCKER_IMAGE_CLEANUP_HOURS='${ttl_hours}', fallback to 72" |
| 75 | ttl_hours=72 |
| 76 | fi |
| 77 | |
| 78 | local now_epoch cutoff_epoch |
| 79 | now_epoch=$(date +%s) |
| 80 | cutoff_epoch=$((now_epoch - ttl_hours * 3600)) |
| 81 | |
| 82 | local -a used_image_ids |
| 83 | mapfile -t used_image_ids < <(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u) |
| 84 | |
| 85 | local removed_count=0 |
| 86 | local examined_count=0 |
| 87 | declare -A seen_ids=() |
| 88 | |
| 89 | while read -r image_ref image_id; do |
| 90 | [[ -z "$image_ref" || -z "$image_id" ]] && continue |
| 91 | ((examined_count++)) |
| 92 | |
| 93 | # Keep the image this job is going to use. |
| 94 | if [[ "$image_ref" == "$current_image_ref" ]]; then |
| 95 | continue |
| 96 | fi |
| 97 | |
| 98 | # Avoid duplicate deletes when multiple tags point to same image id. |
| 99 | if [[ -n "${seen_ids[$image_id]:-}" ]]; then |
| 100 | continue |
| 101 | fi |
| 102 | seen_ids[$image_id]=1 |
| 103 | |
| 104 | # Never delete images that are used by any container on this node. |
| 105 | if printf '%s\n' "${used_image_ids[@]}" | grep -qx "$image_id"; then |
| 106 | continue |
| 107 | fi |
| 108 | |
| 109 | local created created_epoch |
| 110 | created=$(docker image inspect -f '{{.Created}}' "$image_id" 2>/dev/null || true) |
| 111 | [[ -z "$created" ]] && continue |
| 112 | created_epoch=$(date -d "$created" +%s 2>/dev/null || true) |
| 113 | [[ -z "$created_epoch" ]] && continue |
| 114 | |
| 115 | if (( created_epoch < cutoff_epoch )) || [[ "$aggressive_cleanup" == "1" ]]; then |
| 116 | if docker image rm -f "$image_id" >/dev/null 2>&1; then |
| 117 | ((removed_count++)) |
| 118 | fi |
| 119 | fi |
| 120 | done < <(docker image ls --no-trunc "$repo_prefix" --format '{{.Repository}}:{{.Tag}} {{.ID}}') |
| 121 | |
| 122 | # Also trim old dangling layers; this is safe and does not remove referenced images. |
| 123 | docker image prune -f --filter "until=${ttl_hours}h" >/dev/null 2>&1 || true |
| 124 | |
| 125 | if [[ "$aggressive_cleanup" == "1" ]]; then |
| 126 | echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} unused images under disk pressure." |
| 127 | else |
| 128 | echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} old images (>${ttl_hours}h)." |
| 129 | fi |
| 130 | } |
| 131 | |
| 132 | re_quote_pytest_markers() { |
| 133 | local input="$1" |
| 134 | local output="" |
| 135 | local collecting=false |
| 136 | local marker_buf="" |
| 137 | |
| 138 | local flat="${input//$'\n'/ }" |
| 139 | local restore_glob |
| 140 | restore_glob="$(shopt -p -o noglob 2>/dev/null || true)" |
| 141 | set -o noglob |
| 142 | local -a words |
| 143 | read -ra words <<< "$flat" |
| 144 | eval "$restore_glob" |
| 145 | |
| 146 | for word in "${words[@]}"; do |
| 147 | if $collecting; then |
| 148 | if [[ "$word" == *"'"* ]]; then |
| 149 | if [[ -n "$marker_buf" ]]; then |
| 150 | output+="${marker_buf} " |
| 151 | marker_buf="" |
| 152 | fi |
| 153 | output+="${word} " |
| 154 | collecting=false |
| 155 | continue |
| 156 | fi |
| 157 | |
| 158 | local is_boundary=false |
| 159 | case "$word" in |
| 160 | "&&"|"||"|";"|"|") |
| 161 | is_boundary=true ;; |
| 162 | --*) |
| 163 | is_boundary=true ;; |
| 164 | -[a-zA-Z]) |
| 165 | is_boundary=true ;; |
| 166 | */*) |
| 167 | is_boundary=true ;; |
| 168 | *.py|*.py::*) |
| 169 | is_boundary=true ;; |
| 170 | *=*) |
| 171 | if [[ "$word" =~ ^[A-Z_][A-Z0-9_]*= ]]; then |
| 172 | is_boundary=true |
| 173 | fi |
| 174 | ;; |
| 175 | esac |
| 176 | |
| 177 | if $is_boundary; then |
| 178 | if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then |
| 179 | output+="'${marker_buf}' " |
| 180 | else |
| 181 | output+="${marker_buf} " |
| 182 | fi |
| 183 | collecting=false |
| 184 | marker_buf="" |
| 185 | if [[ "$word" == "-m" || "$word" == "-k" ]]; then |
| 186 | output+="${word} " |
| 187 | collecting=true |
| 188 | else |
| 189 | output+="${word} " |
| 190 | fi |
| 191 | else |
| 192 | if [[ -n "$marker_buf" ]]; then |
| 193 | marker_buf+=" ${word}" |
| 194 | else |
| 195 | marker_buf="${word}" |
| 196 | fi |
| 197 | fi |
| 198 | elif [[ "$word" == "-m" || "$word" == "-k" ]]; then |
| 199 | output+="${word} " |
| 200 | collecting=true |
| 201 | marker_buf="" |
| 202 | else |
| 203 | output+="${word} " |
| 204 | fi |
| 205 | done |
| 206 | |
| 207 | if $collecting && [[ -n "$marker_buf" ]]; then |
| 208 | if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then |
| 209 | output+="'${marker_buf}'" |
| 210 | else |
| 211 | output+="${marker_buf}" |
| 212 | fi |
| 213 | fi |
| 214 | |
| 215 | echo "${output% }" |
| 216 | } |
| 217 | |
| 218 | apply_intel_test_overrides() { |
| 219 | local cmds="$1" |
| 220 | # Placeholder for Intel-specific exclusions/overrides. |
| 221 | echo "$cmds" |
| 222 | } |
| 223 | |
| 224 | is_yaml_file() { |
| 225 | local p="$1" |
| 226 | [[ -f "$p" && "$p" == *.yaml ]] |
| 227 | } |
| 228 | |
| 229 | extract_yaml_commands() { |
| 230 | local yaml_path="$1" |
| 231 | awk ' |
| 232 | $1 == "commands:" { in_cmds=1; next } |
| 233 | in_cmds && $0 ~ /^[[:space:]]*-[[:space:]]/ { |
| 234 | sub(/^[[:space:]]*-[[:space:]]/, ""); |
| 235 | print; |
| 236 | next |
| 237 | } |
| 238 | in_cmds && $0 ~ /^[^[:space:]]/ { exit } |
| 239 | ' "$yaml_path" |
| 240 | } |
| 241 | |
| 242 | ############################################################################### |
| 243 | # Main |
| 244 | ############################################################################### |
| 245 | |
| 246 | default_image_name="${REGISTRY}/${REPO}:${BUILDKITE_COMMIT}-xpu" |
| 247 | #default_image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-xpu" |
| 248 | image_name="${IMAGE_TAG_XPU:-${default_image_name}}" |
| 249 | container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" |
| 250 | |
| 251 | # ---- Command source selection ---- |
| 252 | commands="" |
| 253 | commands_source="" |
| 254 | if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then |
| 255 | commands="${VLLM_TEST_COMMANDS}" |
| 256 | commands_source="env" |
| 257 | echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)" |
| 258 | elif [[ $# -gt 0 ]]; then |
| 259 | all_yaml=true |
| 260 | for arg in "$@"; do |
| 261 | if ! is_yaml_file "$arg"; then |
| 262 | all_yaml=false |
| 263 | break |
| 264 | fi |
| 265 | done |
| 266 | |
| 267 | if $all_yaml; then |
| 268 | for yaml in "$@"; do |
| 269 | mapfile -t COMMANDS < <(extract_yaml_commands "$yaml") |
| 270 | if [[ ${#COMMANDS[@]} -eq 0 ]]; then |
| 271 | echo "Error: No commands found in ${yaml}" >&2 |
| 272 | exit 1 |
| 273 | fi |
| 274 | for cmd in "${COMMANDS[@]}"; do |
| 275 | if [[ -z "$commands" ]]; then |
| 276 | commands="${cmd}" |
| 277 | else |
| 278 | commands+=" && ${cmd}" |
| 279 | fi |
| 280 | done |
| 281 | done |
| 282 | echo "Commands sourced from YAML files: $*" |
| 283 | else |
| 284 | commands="$*" |
| 285 | echo "Commands sourced from positional args (legacy mode)" |
| 286 | fi |
| 287 | else |
| 288 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 289 | DEFAULT_YAML="${SCRIPT_DIR}/intel-test.yaml" |
| 290 | if [[ ! -f "${DEFAULT_YAML}" ]]; then |
| 291 | echo "Error: YAML file not found: ${DEFAULT_YAML}" >&2 |
| 292 | exit 1 |
| 293 | fi |
| 294 | mapfile -t COMMANDS < <(extract_yaml_commands "${DEFAULT_YAML}") |
| 295 | if [[ ${#COMMANDS[@]} -eq 0 ]]; then |
| 296 | echo "Error: No commands found in ${DEFAULT_YAML}" >&2 |
| 297 | exit 1 |
| 298 | fi |
| 299 | for cmd in "${COMMANDS[@]}"; do |
| 300 | if [[ -z "$commands" ]]; then |
| 301 | commands="${cmd}" |
| 302 | else |
| 303 | commands+=" && ${cmd}" |
| 304 | fi |
| 305 | done |
| 306 | echo "Commands sourced from default YAML: ${DEFAULT_YAML}" |
| 307 | fi |
| 308 | |
| 309 | if [[ -z "$commands" ]]; then |
| 310 | echo "Error: No test commands provided." >&2 |
| 311 | exit 1 |
| 312 | fi |
| 313 | |
| 314 | echo "Raw commands: $commands" |
| 315 | if [[ "$commands_source" != "env" ]]; then |
| 316 | commands=$(re_quote_pytest_markers "$commands") |
| 317 | echo "After re-quoting: $commands" |
| 318 | else |
| 319 | echo "Skipping re-quoting for VLLM_TEST_COMMANDS input" |
| 320 | fi |
| 321 | commands=$(apply_intel_test_overrides "$commands") |
| 322 | echo "Final commands: $commands" |
| 323 | |
| 324 | # Dry-run mode prints final commands and exits before Docker. |
| 325 | if [[ "$DRY_RUN" == "1" ]]; then |
| 326 | echo "DRY_RUN=1 set, skipping Docker execution." |
| 327 | exit 0 |
| 328 | fi |
| 329 | |
| 330 | # --- Docker housekeeping --- |
| 331 | cleanup_docker |
| 332 | |
| 333 | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" |
| 334 | |
| 335 | # --- Build or pull test image --- |
| 336 | IMAGE="${IMAGE_TAG_XPU:-${image_name}}" |
| 337 | |
| 338 | echo "Using image: ${IMAGE}" |
| 339 | |
| 340 | remove_docker_container() { |
| 341 | docker rm -f "${container_name}" || true |
| 342 | } |
| 343 | trap remove_docker_container EXIT |
| 344 | |
| 345 | # --- Single-node job --- |
| 346 | |
| 347 | if [[ -z "${ZE_AFFINITY_MASK:-}" ]]; then |
| 348 | echo "Warning: ZE_AFFINITY_MASK is not set. Proceeding without device affinity." >&2 |
| 349 | fi |
| 350 | |
| 351 | export CMDS="${commands}" |
| 352 | export HF_TOKEN ZE_AFFINITY_MASK VLLM_DISABLE_COMPILE_CACHE |
| 353 | |
| 354 | { |
| 355 | flock 9 |
| 356 | if docker image inspect "${IMAGE}" >/dev/null 2>&1; then |
| 357 | echo "Image already exists locally, skipping pull" |
| 358 | else |
| 359 | echo "Image not found locally, pulling image..." |
| 360 | timeout 900 docker pull "${IMAGE}" |
| 361 | echo "Pull step completed" |
| 362 | fi |
| 363 | |
| 364 | docker create \ |
| 365 | --device /dev/dri:/dev/dri \ |
| 366 | --net=host \ |
| 367 | --ipc=host \ |
| 368 | --privileged \ |
| 369 | -v /dev/dri/by-path:/dev/dri/by-path \ |
| 370 | -v "/data/huggingface:/root/.cache/huggingface" \ |
| 371 | --entrypoint='' \ |
| 372 | -e HF_TOKEN \ |
| 373 | -e ZE_AFFINITY_MASK \ |
| 374 | -e VLLM_DISABLE_COMPILE_CACHE \ |
| 375 | -e BUILDKITE_PARALLEL_JOB \ |
| 376 | -e BUILDKITE_PARALLEL_JOB_COUNT \ |
| 377 | -e PYTEST_ADDOPTS \ |
| 378 | -e CMDS \ |
| 379 | --name "${container_name}" \ |
| 380 | "${IMAGE}" \ |
| 381 | bash -c 'set -e; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \ |
| 382 | >/dev/null |
| 383 | } 9>/tmp/docker-pull.lock |
| 384 | |
| 385 | docker start -a "${container_name}" |