| 1 | #!/bin/bash |
| 2 | |
| 3 | INPUT_JSON="$1" |
| 4 | GCS_PATH="$2" # Optional GCS path for uploading results for each run |
| 5 | |
| 6 | SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) |
| 7 | AUTOTUNE_SCRIPT="$SCRIPT_DIR/auto_tune.sh" |
| 8 | |
| 9 | if [[ -z "$INPUT_JSON" ]]; then |
| 10 | echo "Error: Input JSON file not provided." |
| 11 | echo "Usage: $0 <path_to_json_file> [gcs_upload_path]" |
| 12 | exit 1 |
| 13 | fi |
| 14 | |
| 15 | if [[ ! -f "$INPUT_JSON" ]]; then |
| 16 | echo "Error: File not found at '$INPUT_JSON'" |
| 17 | exit 1 |
| 18 | fi |
| 19 | |
| 20 | if ! command -v jq &> /dev/null; then |
| 21 | echo "Error: 'jq' command not found. Please install jq to process the JSON input." |
| 22 | exit 1 |
| 23 | fi |
| 24 | |
| 25 | if [[ -n "$GCS_PATH" ]] && ! command -v gcloud &> /dev/null; then |
| 26 | echo "Error: 'gcloud' command not found, but a GCS_PATH was provided." |
| 27 | exit 1 |
| 28 | fi |
| 29 | |
| 30 | SUCCESS_COUNT=0 |
| 31 | FAILURE_COUNT=0 |
| 32 | FAILED_RUNS=() |
| 33 | SCRIPT_START_TIME=$(date +%s) |
| 34 | |
| 35 | json_content=$(cat "$INPUT_JSON") |
| 36 | if ! num_runs=$(echo "$json_content" | jq 'length'); then |
| 37 | echo "Error: Invalid JSON in $INPUT_JSON. 'jq' failed to get array length." >&2 |
| 38 | exit 1 |
| 39 | fi |
| 40 | |
| 41 | echo "Found $num_runs benchmark configurations in $INPUT_JSON." |
| 42 | echo "Starting benchmark runs..." |
| 43 | echo "--------------------------------------------------" |
| 44 | |
| 45 | for i in $(seq 0 $(($num_runs - 1))); do |
| 46 | run_object=$(echo "$json_content" | jq ".[$i]") |
| 47 | |
| 48 | RUN_START_TIME=$(date +%s) |
| 49 | ENV_VARS_ARRAY=() |
| 50 | # Dynamically create env vars from the JSON object's keys |
| 51 | for key in $(echo "$run_object" | jq -r 'keys_unsorted[]'); do |
| 52 | value=$(echo "$run_object" | jq -r ".$key") |
| 53 | var_name=$(echo "$key" | tr '[:lower:]' '[:upper:]' | tr -cd 'A-Z0-9_') |
| 54 | ENV_VARS_ARRAY+=("${var_name}=${value}") |
| 55 | done |
| 56 | |
| 57 | echo "Executing run #$((i+1))/$num_runs with parameters: ${ENV_VARS_ARRAY[*]}" |
| 58 | |
| 59 | # Execute auto_tune.sh and capture output |
| 60 | RUN_OUTPUT_FILE=$(mktemp) |
| 61 | if env "${ENV_VARS_ARRAY[@]}" bash "$AUTOTUNE_SCRIPT" > >(tee -a "$RUN_OUTPUT_FILE") 2>&1; then |
| 62 | STATUS="SUCCESS" |
| 63 | ((SUCCESS_COUNT++)) |
| 64 | else |
| 65 | STATUS="FAILURE" |
| 66 | ((FAILURE_COUNT++)) |
| 67 | FAILED_RUNS+=("Run #$((i+1)): $(echo "$run_object" | jq -c .)") |
| 68 | fi |
| 69 | |
| 70 | RUN_OUTPUT=$(<"$RUN_OUTPUT_FILE") |
| 71 | rm "$RUN_OUTPUT_FILE" |
| 72 | |
| 73 | # Parse results and optionally upload them to GCS |
| 74 | RUN_ID="" |
| 75 | RESULTS="" |
| 76 | GCS_RESULTS_URL="" |
| 77 | if [[ "$STATUS" == "SUCCESS" ]]; then |
| 78 | RESULT_FILE_PATH=$(echo "$RUN_OUTPUT" | grep 'RESULT_FILE=' | tail -n 1 | cut -d'=' -f2 | tr -s '/' || true) |
| 79 | |
| 80 | if [[ -n "$RESULT_FILE_PATH" && -f "$RESULT_FILE_PATH" ]]; then |
| 81 | RUN_ID=$(basename "$(dirname "$RESULT_FILE_PATH")") |
| 82 | RESULT_DIR=$(dirname "$RESULT_FILE_PATH") |
| 83 | RESULTS=$(cat "$RESULT_FILE_PATH") |
| 84 | |
| 85 | if [[ -n "$GCS_PATH" ]]; then |
| 86 | GCS_RESULTS_URL="${GCS_PATH}/${RUN_ID}" |
| 87 | echo "Uploading results to GCS..." |
| 88 | if gcloud storage rsync --recursive "$RESULT_DIR/" "$GCS_RESULTS_URL"; then |
| 89 | echo "GCS upload successful." |
| 90 | else |
| 91 | echo "Warning: GCS upload failed for RUN_ID $RUN_ID." |
| 92 | fi |
| 93 | fi |
| 94 | else |
| 95 | echo "Warning: Could not find result file for a successful run." |
| 96 | STATUS="WARNING_NO_RESULT_FILE" |
| 97 | fi |
| 98 | fi |
| 99 | |
| 100 | # Add the results back into the JSON object for this run |
| 101 | json_content=$(echo "$json_content" | jq --argjson i "$i" --arg run_id "$RUN_ID" --arg status "$STATUS" --arg results "$RESULTS" --arg gcs_results "$GCS_RESULTS_URL" \ |
| 102 | '.[$i] += {run_id: $run_id, status: $status, results: $results, gcs_results: $gcs_results}') |
| 103 | |
| 104 | RUN_END_TIME=$(date +%s) |
| 105 | echo "Run finished in $((RUN_END_TIME - RUN_START_TIME)) seconds. Status: $STATUS" |
| 106 | echo "--------------------------------------------------" |
| 107 | |
| 108 | # Save intermediate progress back to the file |
| 109 | echo "$json_content" > "$INPUT_JSON.tmp" && mv "$INPUT_JSON.tmp" "$INPUT_JSON" |
| 110 | |
| 111 | done |
| 112 | |
| 113 | SCRIPT_END_TIME=$(date +%s) |
| 114 | echo "All benchmark runs completed in $((SCRIPT_END_TIME - SCRIPT_START_TIME)) seconds." |
| 115 | echo |
| 116 | echo "====================== SUMMARY ======================" |
| 117 | echo "Successful runs: $SUCCESS_COUNT" |
| 118 | echo "Failed runs: $FAILURE_COUNT" |
| 119 | echo "===================================================" |
| 120 | |
| 121 | if [[ $FAILURE_COUNT -gt 0 ]]; then |
| 122 | echo "Details of failed runs (see JSON file for full parameters):" |
| 123 | for failed in "${FAILED_RUNS[@]}"; do |
| 124 | echo " - $failed" |
| 125 | done |
| 126 | fi |
| 127 | |
| 128 | echo "Updated results have been saved to '$INPUT_JSON'." |