| 1 | name: Sync publish data to main |
| 2 | |
| 3 | on: # zizmor: ignore[dangerous-triggers] intentional chaining off crawl-and-publish; sync job guarded by workflow_run.conclusion==success |
| 4 | workflow_run: |
| 5 | workflows: ["Crawl and publish weekly data"] |
| 6 | types: [completed] |
| 7 | branches: [main] |
| 8 | workflow_dispatch: |
| 9 | |
| 10 | # Prevent overlapping force-pushes to sync/publish-to-main |
| 11 | concurrency: |
| 12 | group: sync-publish-to-main |
| 13 | cancel-in-progress: false |
| 14 | |
| 15 | permissions: |
| 16 | contents: read |
| 17 | |
| 18 | jobs: |
| 19 | sync: |
| 20 | runs-on: ubuntu-latest |
| 21 | permissions: |
| 22 | contents: write # force-push the sync/publish-to-main branch |
| 23 | pull-requests: write # open/update the sync PR into main |
| 24 | if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} |
| 25 | steps: |
| 26 | - name: Check out main # zizmor: ignore[artipacked] sync job force-pushes the sync branch; checkout token is reused by git push |
| 27 | uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 |
| 28 | with: |
| 29 | ref: main |
| 30 | fetch-depth: 0 |
| 31 | |
| 32 | - name: Sync data from publish |
| 33 | id: sync |
| 34 | env: |
| 35 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 36 | SYNC_BRANCH: sync/publish-to-main |
| 37 | run: | |
| 38 | set -euo pipefail |
| 39 | git config user.name "github-actions[bot]" |
| 40 | git config user.email "github-actions[bot]@users.noreply.github.com" |
| 41 | |
| 42 | # Fetch publish |
| 43 | git fetch origin publish |
| 44 | |
| 45 | # Create or reset sync branch from main |
| 46 | git checkout -B "$SYNC_BRANCH" origin/main |
| 47 | |
| 48 | # Wipe target generated dirs so deletions on publish propagate to main |
| 49 | rm -rf data/raw data/analyzed data/metrics content/weekly content/monthly content/yearly |
| 50 | |
| 51 | # REQUIRED — these must exist on publish; fail loudly if not |
| 52 | git checkout origin/publish -- \ |
| 53 | data/raw/ \ |
| 54 | data/analyzed/ \ |
| 55 | data/metrics/ \ |
| 56 | content/weekly/ \ |
| 57 | content/monthly/ \ |
| 58 | content/yearly/ |
| 59 | |
| 60 | # Rebuild rollups from the synced analyzed summaries so monthly/yearly |
| 61 | # pages cannot drift from the weekly content in the generated sync PR. |
| 62 | python3 scripts/generate_rollups.py |
| 63 | |
| 64 | # Stage restored/generated files before evaluating the cached diff. |
| 65 | git add -A |
| 66 | if git diff --cached --quiet; then |
| 67 | echo "No changes to sync." |
| 68 | exit 0 |
| 69 | fi |
| 70 | |
| 71 | if git diff --cached --name-only | grep -E '^\.squad/' >/dev/null; then |
| 72 | echo "::error::Refusing to sync .squad state from publish to main." |
| 73 | git diff --cached --name-only | grep -E '^\.squad/' || true |
| 74 | exit 1 |
| 75 | fi |
| 76 | |
| 77 | if git diff --cached --quiet; then |
| 78 | echo "No changes to sync after staging." |
| 79 | exit 0 |
| 80 | fi |
| 81 | |
| 82 | git commit -m "sync: publish data → main |
| 83 | |
| 84 | Automated sync of crawl data, analysis, and content |
| 85 | from the publish branch." |
| 86 | |
| 87 | git push -f origin "$SYNC_BRANCH" |
| 88 | |
| 89 | # Create or update PR |
| 90 | PR_NUMBER=$(gh pr list --head "$SYNC_BRANCH" --base main --json number --jq '.[0].number // empty') |
| 91 | if [ -n "$PR_NUMBER" ]; then |
| 92 | echo "Updated existing PR #${PR_NUMBER}" |
| 93 | else |
| 94 | gh pr create \ |
| 95 | --base main \ |
| 96 | --head "$SYNC_BRANCH" \ |
| 97 | --title "sync: publish data → main" \ |
| 98 | --body "Automated sync of generated content and data subtrees from the publish branch to main. |
| 99 | |
| 100 | This PR keeps main up-to-date with generated data and content from the crawl/publish pipeline. |
| 101 | |
| 102 | **Synced paths (required — workflow fails if missing on publish):** |
| 103 | - \`data/raw/\` — raw crawl data |
| 104 | - \`data/analyzed/\` — weekly analysis summaries |
| 105 | - \`data/metrics/\` — token usage metrics |
| 106 | - \`content/weekly/\`, \`content/monthly/\`, \`content/yearly/\` — generated Hugo pages |
| 107 | |
| 108 | **Explicitly NOT synced:** |
| 109 | - \`.squad/**\` — Squad memory, decisions, run counters, skills, logs, and agent histories are not authoritative on publish. |
| 110 | |
| 111 | **Notes:** |
| 112 | - Publish is authoritative for synced paths. Hand-edits on main to these paths WILL be overwritten. |
| 113 | - Deletions on publish propagate to main (target dirs are wiped before checkout). |
| 114 | |
| 115 | Safe to merge — contains only generated data, no code changes." \ |
| 116 | --label "squad" |
| 117 | PR_NUMBER=$(gh pr list --head "$SYNC_BRANCH" --base main --json number --jq '.[0].number') |
| 118 | fi |
| 119 | |
| 120 | if [ -z "$PR_NUMBER" ]; then |
| 121 | echo "::error::Could not determine PR number for sync branch $SYNC_BRANCH" |
| 122 | exit 1 |
| 123 | fi |
| 124 | |
| 125 | # Short-circuit if the PR has already been merged (e.g. by a prior run). |
| 126 | if [ "$(gh pr view "$PR_NUMBER" --json state --jq '.state')" = "MERGED" ]; then |
| 127 | echo "::notice::PR #${PR_NUMBER} is already merged; nothing to do." |
| 128 | exit 0 |
| 129 | fi |
| 130 | |
| 131 | HEAD_SHA=$(git rev-parse HEAD) |
| 132 | CHECK_COUNT=0 |
| 133 | for _ in $(seq 1 30); do |
| 134 | # `gh pr checks` exits non-zero ("no checks reported") until checks |
| 135 | # register; tolerate that under `set -e` so we keep polling instead of |
| 136 | # killing the whole sync job (the bug that left the sync PR unmerged). |
| 137 | CHECK_COUNT=$(gh pr checks "$PR_NUMBER" --json name --jq 'length' || echo 0) |
| 138 | if [ "$CHECK_COUNT" -gt 0 ]; then |
| 139 | break |
| 140 | fi |
| 141 | echo "Waiting for PR #${PR_NUMBER} checks to register..." |
| 142 | sleep 10 |
| 143 | done |
| 144 | |
| 145 | if [ "$CHECK_COUNT" -eq 0 ]; then |
| 146 | echo "::error::No status checks registered for PR #${PR_NUMBER}; refusing to merge without validation." |
| 147 | exit 1 |
| 148 | fi |
| 149 | |
| 150 | # Only now that we have confirmed required checks are configured do we |
| 151 | # enable auto-merge, so the PR cannot merge immediately by bypassing the |
| 152 | # "refusing to merge without validation" guard above. Emit a warning (not |
| 153 | # a silent `|| true`) if auto-merge cannot be enabled. |
| 154 | gh pr merge "$PR_NUMBER" --auto --squash || echo "::warning::Could not enable auto-merge for PR #${PR_NUMBER} — will attempt explicit merge after checks pass." |
| 155 | |
| 156 | gh pr checks "$PR_NUMBER" --watch --fail-fast |
| 157 | # Auto-merge may have already squashed the PR once checks passed; if the |
| 158 | # explicit merge fails, confirm the PR did in fact merge before succeeding. |
| 159 | gh pr merge "$PR_NUMBER" --squash --match-head-commit "$HEAD_SHA" || \ |
| 160 | gh pr view "$PR_NUMBER" --json state --jq '.state' | grep -qx MERGED |
| 161 | |
| 162 | # Signal that the weekly content is now merged to main so the Podcaster |
| 163 | # handoff (next step) can fire — and only now, never before the merge. |
| 164 | echo "merged=true" >> "$GITHUB_OUTPUT" |
| 165 | |
| 166 | - name: Set up Python |
| 167 | if: ${{ steps.sync.outputs.merged == 'true' }} |
| 168 | uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 |
| 169 | with: |
| 170 | python-version: '3.12' |
| 171 | |
| 172 | - name: Trigger Podcaster after merge |
| 173 | if: ${{ steps.sync.outputs.merged == 'true' }} |
| 174 | env: |
| 175 | PODCASTER_ENDPOINT: ${{ vars.PODCASTER_ENDPOINT }} |
| 176 | PODCASTER_API_KEY: ${{ secrets.PODCASTER_API_KEY }} |
| 177 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 178 | run: | |
| 179 | set -euo pipefail |
| 180 | if [ -z "$PODCASTER_ENDPOINT" ] || [ -z "$PODCASTER_API_KEY" ]; then |
| 181 | echo "::notice::Podcaster handoff skipped — PODCASTER_ENDPOINT/PODCASTER_API_KEY not configured." |
| 182 | exit 0 |
| 183 | fi |
| 184 | |
| 185 | # main now contains the merged weekly content. Identify the latest week. |
| 186 | git fetch origin main |
| 187 | git checkout -B main origin/main |
| 188 | LATEST=$(git ls-files 'content/weekly/*/W*.md' | sort -V | tail -1) |
| 189 | if [ -z "$LATEST" ]; then |
| 190 | echo "::notice::No weekly article on main; skipping Podcaster handoff." |
| 191 | exit 0 |
| 192 | fi |
| 193 | YEAR=$(basename "$(dirname "$LATEST")") |
| 194 | SHORT=$(basename "$LATEST" .md) |
| 195 | WEEK="${YEAR}-${SHORT}" |
| 196 | # Derive the canonical URL via the shared helper to avoid slug drift. |
| 197 | ARTICLE_URL=$(python3 - "$LATEST" <<'PY' |
| 198 | import sys |
| 199 | from scripts.podcaster_handoff import article_url_from_page_path |
| 200 | |
| 201 | print(article_url_from_page_path("https://claracle.com/", sys.argv[1])) |
| 202 | PY |
| 203 | ) |
| 204 | |
| 205 | # Locate the publish manifest produced by crawl-and-publish for this week. |
| 206 | # Surface publish access problems as warnings instead of swallowing them — |
| 207 | # a broken publish branch should be visible in the Actions log, not silent. |
| 208 | git fetch origin publish |
| 209 | # Clear any stale candidates and fail closed: if the publish checkout |
| 210 | # fails we skip the handoff rather than risk using an incorrect manifest. |
| 211 | rm -rf data/candidates/ |
| 212 | if ! git checkout origin/publish -- data/candidates/; then |
| 213 | echo "::warning::Could not check out data/candidates/ from publish; skipping Podcaster handoff." |
| 214 | exit 0 |
| 215 | fi |
| 216 | MANIFEST=$(find "data/candidates/${WEEK}" -name 'publish-manifest.json' -type f 2>/dev/null | sort -V | tail -1) |
| 217 | if [ -z "$MANIFEST" ]; then |
| 218 | echo "::warning::No manifest for ${WEEK}; skipping Podcaster handoff." |
| 219 | exit 0 |
| 220 | fi |
| 221 | if ! python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST"; then |
| 222 | echo "::notice::Manifest not eligible; skipping Podcaster handoff." |
| 223 | exit 0 |
| 224 | fi |
| 225 | |
| 226 | # --require-merged fails closed unless the merged article exists and its |
| 227 | # sha256 matches the manifest, so the Podcaster is never triggered for a stub. |
| 228 | python3 scripts/podcaster_handoff.py \ |
| 229 | --week "$WEEK" \ |
| 230 | --article-url "$ARTICLE_URL" \ |
| 231 | --article-path "$LATEST" \ |
| 232 | --publish-run-id "$(basename "$(dirname "$MANIFEST")")" \ |
| 233 | --publish-mode normal \ |
| 234 | --manifest "$MANIFEST" \ |
| 235 | --podcast-config config/podcast.json \ |
| 236 | --require-merged |