fix: get npm v6 content from tarball
Luke Karrys committed
Dec 21, 2022 at 14:26 UTC
da0656f554188fe70becc27cb594c41dea593f13
8 files changed
+31
-90
.github/workflows/ci-cli.yml
-2
@@ -100,5 +100,3 @@ jobs:
100
run: echo "::add-matcher::.github/matchers/tap.json"
101
- name: Test
102
run: npm test --ignore-scripts -w cli
103
- env:
104
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CONTRIBUTING.md
-5
@@ -179,11 +179,6 @@ adding a new major version to the site.
179
fetch the latest version of the documentation from GitHub.
180
* `spec`: The registry spec for the version. This will be used
181
to fetch the latest version in that range from the registry.
182
- * `useBranch`: A boolean that controls whether the content for this
183
- version will be fetched from GitHub. The default is false, which
184
- means the content will be fetched directly from the registry tarball.
185
- It is preferred to use the registry but for some legacy versions,
186
- the content was only updated on GitHub and never published.
182
* `resolved`: This should not be edited manually. This is a reference
183
to the last fetched version of the content for this release. If
184
a future fetch is done and this field matches what is returned
cli/lib/extract.js
+1
-39
@@ -43,40 +43,6 @@ const unpackTarball = async ({ release, cwd, dir }) => {
43
return result
44
}
45
46
-const unpackTree = async ({ release, cwd, dir }) => {
47
- const dirParts = dir.split(sep)
48
- const child = dirParts.pop()
49
- const parent = join(...dirParts)
50
-
51
- // to get the sha of the dir, we have to get the parent
52
- // and find the child as an entry and get its sha
53
- const sha = await gh.getDirectory(release.branch, parent)
54
- .then(paths => paths.find((p) => p.name === child).sha)
55
-
56
- const files = await gh.getAllFiles(sha)
57
-
58
- // tar makes the directories for us when unpacking but we
59
- // need to to that manually here
60
- const dirs = [...new Set(files.map((f) => join(cwd, dirname(f.path))))]
61
- await Promise.all(dirs.map((d) => fs.mkdir(d, { recursive: true })))
62
-
63
- await Promise.all(
64
- files.map(async (file) => {
65
- const buffer = await gh.getFile({ sha: file.sha })
66
- return fs.writeFile(
67
- join(cwd, file.path),
68
- Transform.sync(buffer, {
69
- path: file.path,
70
- release,
71
- }),
72
- 'utf-8'
73
- )
74
- })
75
- )
76
-
77
- return files.map((f) => f.path)
78
-}
79
-
46
const getNav = async ({ path, release }) => {
47
const nav = await gh.getFile({ ref: release.branch, path })
48
@@ -157,11 +123,7 @@ const unpackRelease = async (
123
// the tree of the doc directory's sha which has all the docs
124
// we need in it. Note that this requires the docs to all be
125
// built in source, which is true for v6 but not for v9 and later.
160
- const files = release.useBranch ? await unpackTree({
161
- release,
162
- cwd,
163
- dir: release.src,
164
- }) : await unpackTarball({
126
+ const files = await unpackTarball({
127
release,
128
cwd,
129
dir: builtPath,
cli/lib/gh.js
-33
@@ -20,34 +20,6 @@ const getFile = async ({ sha, ref, path }) => {
20
return Buffer.from(data.content, data.encoding)
21
}
22
23
-const getAllFiles = async (sha) => {
24
- const {
25
- data: { tree },
26
- } = await octokit.git.getTree({
27
- ...opts,
28
- tree_sha: sha,
29
- recursive: true,
30
- })
31
-
32
- return tree
33
- .filter((f) => f.type === 'blob')
34
- .map((f) => ({
35
- ...f,
36
- // return file paths that can be used on the
37
- // system to write files
38
- path: f.path.split(posix.sep).join(sep),
39
- }))
40
-}
41
-
42
-const getDirectory = async (ref, dir) => {
43
- const { data } = await octokit.repos.getContent({
44
- ...opts,
45
- ref,
46
- path: dir.split(sep).join(posix.sep),
47
- })
48
- return data
49
-}
50
-
23
const pathExists = async (ref, path) => {
24
try {
25
await octokit.repos.getContent({
@@ -66,12 +38,7 @@ const pathExists = async (ref, path) => {
38
}
39
40
module.exports = {
69
- octokit,
41
getFile,
71
- getAllFiles,
72
- getDirectory,
42
pathExists,
74
- owner,
75
- repo,
43
nwo: `${owner}/${repo}`,
44
}
cli/releases.json
+1
-2
@@ -1,8 +1,7 @@
1
[
2
{
3
"id": "v6",
4
- "branch": "release/v6",
5
- "useBranch": true
4
+ "branch": "release/v6"
5
},
6
{
7
"id": "v7",
cli/scripts/template-oss/_step-test.yml
deleted
-3
@@ -1,3 +0,0 @@
1
-{{> defaultStepTest }}
2
- env:
3
- GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
cli/test/index.js
+26
-5
@@ -1,7 +1,9 @@
1
const t = require('tap')
2
-const { resolve, join } = require('path')
2
+const { resolve, join, posix } = require('path')
3
const fs = require('fs/promises')
4
const pacote = require('pacote')
5
+const yaml = require('yaml')
6
+const semver = require('semver')
7
8
const navPath = resolve(
9
__dirname,
@@ -16,7 +18,6 @@ const getReleases = () => [
18
{
19
id: 'v6',
20
branch: 'release/v6',
19
- useBranch: true,
21
},
22
{
23
id: 'v7',
@@ -33,9 +34,12 @@ const getReleases = () => [
34
]
35
36
const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) => {
37
+ const rawNav = await fs.readFile(navPath, 'utf-8')
38
+ const nav = yaml.parse(rawNav)
39
+
40
const testdir = t.testdir({
41
'releases.json': JSON.stringify(releases),
38
- 'nav.yml': await fs.readFile(navPath, 'utf-8'),
42
+ 'nav.yml': rawNav,
43
content: {},
44
...testdirOpts,
45
})
@@ -46,7 +50,7 @@ const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) =>
50
// so by default they all need to exist
51
switch (r.id.slice(1)) {
52
case '6':
49
- return '6.14.17'
53
+ return '6.14.18'
54
case '7':
55
return '7.24.2'
56
case '8':
@@ -61,6 +65,13 @@ const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) =>
65
packument.latest = packument.versions[packument.versions.length - 1]
66
}
67
68
+ const navSection = (ref) => {
69
+ const id = ref === 'latest' ? `v${semver.major(packument.latest)}` : posix.basename(ref)
70
+ const { variants } = nav.find(c => c.url === '/cli')
71
+ const { children } = variants.find(v => posix.basename(v.url) === id)
72
+ return yaml.stringify(children).replace(new RegExp(`/cli/${id}/`, 'g'), '/')
73
+ }
74
+
75
const build = t.mock('../lib/build', {
76
pacote: {
77
...pacote,
@@ -76,6 +87,16 @@ const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) =>
87
}
88
},
89
},
90
+ '../lib/gh.js': {
91
+ getFile: async ({ ref }) => navSection(ref),
92
+ pathExists: async (ref, p) => {
93
+ if (ref.includes('v6') && p.includes('docs/lib/content')) {
94
+ return null
95
+ }
96
+ return p
97
+ },
98
+ nwo: `npm/cli`,
99
+ },
100
})
101
102
return {
@@ -116,7 +137,7 @@ t.test('prereleases', async (t) => {
137
const releases = getReleases()
138
const { build, testdir } = await mockBuild({
139
releases,
119
- packument: { versions: ['6.14.17', '7.24.2', '8.19.3', '9.0.0-pre.2'], latest: '8.19.3' },
140
+ packument: { versions: ['6.14.18', '7.24.2', '8.19.3', '9.0.0-pre.2'], latest: '8.19.3' },
141
})
142
143
await build({ prerelease: false })
cli/test/transform.js
+3
-1
@@ -1,8 +1,10 @@
1
const t = require('tap')
2
const fm = require('front-matter')
3
-const Transform = require('../lib/transform')
3
4
const transform = ({ id, path }) => {
5
+ const Transform = t.mock('../lib/transform', {
6
+ '../lib/gh.js': { nwo: 'npm/cli' },
7
+ })
8
const transformed = Transform.sync('---\n---\n', {
9
release: {
10
id: id,