chore: verify checksums during dist.ipfs.io sync (#8587)
* only copy files from dist.ipfs.io which have associated checksum files (also copy the checksum files) * verify checksums before copying * also, ignore path from sha512sum output, which sometimes has absolute path on dist.ipfs.io website Co-authored-by: Marcin Rataj <lidel@lidel.org>
Petar Maymounkov committed
Dec 7, 2021 at 16:45 UTC
8cfc88961d4081b2749fba135b63ffcca5a62456
1 file changed
+47
-12
.github/workflows/sync-release-assets.yml
+47
-12
@@ -72,11 +72,9 @@ jobs:
72
}
73
num_synced += 1
74
75
- const github_assets = [];
76
- github_map = {};
75
+ const github_assets = new Set()
76
for (const asset of release.assets) {
78
- github_assets.push(asset.name);
79
- github_map[asset.name] = true;
77
+ github_assets.add(asset.name)
78
}
79
80
// fetch asset info from dist.ipfs.io
@@ -90,14 +88,14 @@ jobs:
88
}
89
await exec.exec('ipfs', ['ls', p], options)
90
93
- const dist_assets = []
94
- missing_files = []
91
+ const dist_assets = new Set()
92
+ const missing_files = []
93
for (const raw_line of stdout.split("\n")) {
94
line = raw_line.trim();
95
if (line.length != 0) {
96
file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2]
99
- dist_assets.push(file)
100
- if (!github_map[file]) {
97
+ dist_assets.add(file)
98
+ if (!github_assets.has(file)) {
99
missing_files.push(file)
100
}
101
}
@@ -105,23 +103,60 @@ jobs:
103
104
// if dist.ipfs.io has files not found in github, copy them over
105
for (const file of missing_files) {
106
+ file_sha = file + ".sha512"
107
+ file_cid = file + ".cid"
108
+
109
+ // skip files that don't have .cid and .sha512 checksum files
110
+ if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) {
111
+ if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :)
112
+ console.log(`skipping "${file}" as dist.ipfs.io does not provide .cid and .sha512 checksum files for it`)
113
+ }
114
+ continue
115
+ }
116
+
117
console.log("fetching", file, "from dist.ipfs.io")
118
await exec.exec('ipfs', ['get', p + '/' + file])
119
+ await exec.exec('ipfs', ['get', p + '/' + file_sha])
120
+ await exec.exec('ipfs', ['get', p + '/' + file_cid])
121
+ console.log("verifying contents of", file)
122
+
123
+ // compute sha512 output for file
124
+ let sha_stdout = ''
125
+ const sha_options = {}
126
+ sha_options.listeners = {
127
+ stdout: (data) => {
128
+ sha_stdout += data.toString();
129
+ }
130
+ }
131
+ await exec.exec('sha512sum', [file], sha_options)
132
+ // read expected sha512 output
133
+ const sha_data = await fs.readFile(file_sha, "utf8")
134
+ const digest = (s) => s.split(' ').shift()
135
+ if (digest(sha_data) != digest(sha_stdout)) {
136
+ console.log(`${file}.sha512: ${sha_data}`)
137
+ console.log(`sha512sum ${file}: ${sha_stdout}`)
138
+ throw "checksum verification failed for " + file
139
+ }
140
+
141
console.log("uploading", file, "to github release", release.tag_name)
111
- resp = await github.repos.uploadReleaseAsset({
142
+ const uploadReleaseAsset = async (file) => github.repos.uploadReleaseAsset({
143
owner: context.repo.owner,
144
repo: context.repo.repo,
145
release_id: release.id,
146
headers: {
147
"content-type": "application/octet-stream",
148
"content-length": `${(await fs.stat(file)).size}`
118
- },
149
+ },
150
name: file,
120
- data: await fs.readFile(file),
151
+ data: await fs.readFile(file)
152
})
153
+ await uploadReleaseAsset(file)
154
+ await uploadReleaseAsset(file_sha)
155
+ await uploadReleaseAsset(file_cid)
156
+
157
}
158
// summary of assets on both sides
124
- release_assets.push({ tag: release.tag_name, github_assets: github_assets, dist_assets: dist_assets })
159
+ release_assets.push({ tag: release.tag_name, github_assets, dist_assets })
160
}
161
console.log(release_assets)
162
return release_assets