| 1 | name: Sync GitHub Release Assets |
| 2 | |
| 3 | on: |
| 4 | workflow_dispatch: |
| 5 | schedule: |
| 6 | - cron: '0 0 * * *' |
| 7 | |
| 8 | concurrency: |
| 9 | group: release-assets-dist-sync |
| 10 | cancel-in-progress: true |
| 11 | |
| 12 | permissions: |
| 13 | contents: write # to upload release asset |
| 14 | |
| 15 | jobs: |
| 16 | dist-ipfs-tech: |
| 17 | if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' |
| 18 | runs-on: "ubuntu-latest" |
| 19 | timeout-minutes: 15 |
| 20 | steps: |
| 21 | - uses: ipfs/download-ipfs-distribution-action@v1 |
| 22 | - uses: ipfs/start-ipfs-daemon-action@v1 |
| 23 | with: |
| 24 | args: --init --init-profile=flatfs,server --enable-gc=false |
| 25 | - uses: actions/setup-node@v6 |
| 26 | with: |
| 27 | node-version: 14 |
| 28 | - name: Sync the latest 5 github releases |
| 29 | uses: actions/github-script@v9 |
| 30 | with: |
| 31 | script: | |
| 32 | const fs = require('fs').promises |
| 33 | const max_synced = 5 |
| 34 | |
| 35 | // fetch github releases |
| 36 | resp = await github.rest.repos.listReleases({ |
| 37 | owner: context.repo.owner, |
| 38 | repo: context.repo.repo, |
| 39 | page: 1, |
| 40 | per_page: max_synced |
| 41 | }) |
| 42 | const release_assets = []; |
| 43 | num_synced = 0; |
| 44 | for (const release of resp.data) { |
| 45 | console.log("checking release tagged", release.tag_name) |
| 46 | if (num_synced > max_synced) { |
| 47 | console.log("done: synced", max_synced, "latest releases") |
| 48 | break; |
| 49 | } |
| 50 | num_synced += 1 |
| 51 | |
| 52 | const github_assets = new Set() |
| 53 | for (const asset of release.assets) { |
| 54 | github_assets.add(asset.name) |
| 55 | } |
| 56 | |
| 57 | // fetch asset info from dist.ipfs.tech |
| 58 | p = '/ipns/dist.ipfs.tech/kubo/' + release.tag_name |
| 59 | let stdout = '' |
| 60 | const options = {} |
| 61 | options.listeners = { |
| 62 | stdout: (data) => { |
| 63 | stdout += data.toString(); |
| 64 | } |
| 65 | } |
| 66 | await exec.exec('ipfs', ['ls', p], options) |
| 67 | |
| 68 | const dist_assets = new Set() |
| 69 | const missing_files = [] |
| 70 | for (const raw_line of stdout.split("\n")) { |
| 71 | line = raw_line.trim(); |
| 72 | if (line.length != 0) { |
| 73 | file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2] |
| 74 | dist_assets.add(file) |
| 75 | if (!github_assets.has(file)) { |
| 76 | missing_files.push(file) |
| 77 | } |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | // if dist.ipfs.tech has files not found in github, copy them over |
| 82 | for (const file of missing_files) { |
| 83 | file_sha = file + ".sha512" |
| 84 | file_cid = file + ".cid" |
| 85 | |
| 86 | // skip files that don't have .cid and .sha512 checksum files |
| 87 | if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) { |
| 88 | if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :) |
| 89 | console.log(`skipping "${file}" as dist.ipfs.tech does not provide .cid and .sha512 checksum files for it`) |
| 90 | } |
| 91 | continue |
| 92 | } |
| 93 | |
| 94 | console.log("fetching", file, "from dist.ipfs.tech") |
| 95 | await exec.exec('ipfs', ['get', p + '/' + file]) |
| 96 | await exec.exec('ipfs', ['get', p + '/' + file_sha]) |
| 97 | await exec.exec('ipfs', ['get', p + '/' + file_cid]) |
| 98 | console.log("verifying contents of", file) |
| 99 | |
| 100 | // compute sha512 output for file |
| 101 | let sha_stdout = '' |
| 102 | const sha_options = {} |
| 103 | sha_options.listeners = { |
| 104 | stdout: (data) => { |
| 105 | sha_stdout += data.toString(); |
| 106 | } |
| 107 | } |
| 108 | await exec.exec('sha512sum', [file], sha_options) |
| 109 | // read expected sha512 output |
| 110 | const sha_data = await fs.readFile(file_sha, "utf8") |
| 111 | const digest = (s) => s.split(' ').shift() |
| 112 | if (digest(sha_data) != digest(sha_stdout)) { |
| 113 | console.log(`${file}.sha512: ${sha_data}`) |
| 114 | console.log(`sha512sum ${file}: ${sha_stdout}`) |
| 115 | throw "checksum verification failed for " + file |
| 116 | } |
| 117 | |
| 118 | console.log("uploading", file, "to github release", release.tag_name) |
| 119 | const uploadReleaseAsset = async (file) => github.rest.repos.uploadReleaseAsset({ |
| 120 | owner: context.repo.owner, |
| 121 | repo: context.repo.repo, |
| 122 | release_id: release.id, |
| 123 | headers: { |
| 124 | "content-type": "application/octet-stream", |
| 125 | "content-length": `${(await fs.stat(file)).size}` |
| 126 | }, |
| 127 | name: file, |
| 128 | data: await fs.readFile(file) |
| 129 | }) |
| 130 | await uploadReleaseAsset(file) |
| 131 | await uploadReleaseAsset(file_sha) |
| 132 | await uploadReleaseAsset(file_cid) |
| 133 | |
| 134 | } |
| 135 | // summary of assets on both sides |
| 136 | release_assets.push({ tag: release.tag_name, github_assets, dist_assets }) |
| 137 | } |
| 138 | console.log(release_assets) |
| 139 | return release_assets |