@cryptotaxi247 / documentation
Code Commits Issues Pulls Sessions CI/CD
reggi/dev-engines
@reggi/path-to-regexp dependabot/npm_and_yarn/main/copy-to-clipboard-4.0.2 dependabot/npm_and_yarn/main/eslint-10.4.0 dependabot/npm_and_yarn/main/npmcli/eslint-config-7.0.0 dependabot/npm_and_yarn/main/proc-log-7.0.0 dependabot/npm_and_yarn/npm_and_yarn-826852524d dependabot/npm_and_yarn/npm_and_yarn-ab9a7f4bc2 deprecate-totp-2fa dhei/classic-tokens gat-bypass-2fa-docs jpg619/fix-accessibility-content-flow jpg619/version-bump-tar-2 kartykp/gat-bypass-2fa-docs kartykp/upgrade-path-to-regex main maitxn/version-bump-tar patch-1 reggi/cache-based-on-version reggi/dev-engines reggi/fix-transform-prettier reggi/overrides update-search-sensitivity
js 225 lines 6.61 KB
Raw
1 const pacote = require('pacote')
2 const tar = require('tar')
3 const {join, sep, dirname, posix} = require('path')
4 const fs = require('fs/promises')
5 const yaml = require('yaml')
6 const Transform = require('./transform')
7 const gh = require('./gh')
8 const log = require('./log')
9
10 const whackAMoleReplace = (s, replacements) => {
11 // some known content in changelogs that is problematic for mdx this is
12 // a bit of whack-a-mole but is necessary since markdown in the CLI is
13 // different from the mdx v2 we parse for the docs site
14 for (const rep of replacements) {
15 s = s.replace(rep, rep.replace(/([<>{}])/g, '\\$1'))
16 }
17 return s
18 }
19
20 const unpackTarball = async ({release, cwd, dir}) => {
21 const strip = 1
22 const result = []
23 const dirParts = dir.split(sep)
24
25 log.verbose('tarball', release.resolved, {cwd, dir})
26
27 const format = s => {
28 // a few specific replacements that cause problems for mdx
29 return (
30 whackAMoleReplace(s, [
31 '{npm-version} node/{node-version} {platform} {arch} workspaces/{workspaces} {ci}',
32 'node/{process.version} {process.platform} {process.arch}',
33 'Default: {prefix}/etc/npmrc',
34 ])
35 // we cant remove all emails since all except this one are in code blocks
36 .replace(/(:: )<(i@izs\.me)>/g, '$1[$2](mailto:$2)')
37 // the v6 version of the funding page has json not inside a code
38 .replace(/(:\n\n)(\s{4}"funding": {)/g, '$1```json\n$2')
39 .replace(/^(\s{4}]$)(\n\n)/gm, '$1\n```$2')
40 // anchor links to markdown. this regex does need to match spaces and newlines since the source markdown
41 // already has some links where attributes are separated by newlines
42 .replace(
43 /<a[\s\n]href="(.*?)"(?:[\s\n]target="_blank")?(?:[\s\n]rel="[a-z\s]+")?>(.*?)<\/a>/g,
44 (_, href, text) => `[${href}](${text.trim()})`,
45 )
46 )
47 }
48
49 const extract = () =>
50 tar.x({
51 cwd,
52 strip: dirParts.length + strip,
53 transform: ({path}) => {
54 result.push(path)
55 log.verbose(release.id, path)
56 return new Transform({
57 path,
58 release,
59 format,
60 })
61 },
62 filter: path => {
63 const pathParts = path.split(posix.sep)
64 const prefixParts = pathParts.slice(strip, dirParts.length + strip)
65 return join(...prefixParts) === join(...dirParts)
66 },
67 })
68
69 await pacote.tarball.stream(
70 `npm@${release.version}`,
71 stream =>
72 new Promise((res, rej) => {
73 stream.on('end', res)
74 stream.on('error', rej)
75 stream.pipe(extract())
76 }),
77 )
78
79 await Promise.all(result.map(f => fs.rename(join(cwd, f), join(cwd, f.replace('.md', '.mdx')))))
80
81 return result
82 }
83
84 const getNav = async ({path, release}) => {
85 const nav = await gh.getFile({ref: release.branch, path})
86
87 const rewriteUrls = nodes =>
88 nodes?.map(n => {
89 n.url = release.url + n.url
90 n.children = rewriteUrls(n.children)
91 return n
92 })
93
94 return {
95 path,
96 children: rewriteUrls(yaml.parse(nav.toString())),
97 }
98 }
99
100 const writeChangelog = async ({release, nav, cwd, srcPath, contentPath}) => {
101 const title = 'Changelog'
102 const changelog = await gh.getFile({ref: release.branch, path: srcPath})
103
104 await fs.writeFile(
105 join(cwd, contentPath + '.mdx'),
106 Transform.sync(changelog, {
107 release,
108 path: contentPath,
109 frontmatter: {
110 github_path: srcPath,
111 title,
112 },
113 format: s => {
114 // some known content in changelogs that is problematic for mdx this is
115 // a bit of whack-a-mole but is necessary since markdown in the CLI is
116 // different from the mdx v2 we parse for the docs site
117 return (
118 whackAMoleReplace(s, [
119 ' support for node <=16.13 ',
120 '<->',
121 ' npm install <folder> ',
122 ' --replace-registry-host=<npmjs|always|never> ',
123 'bundledDependencies -> bundleDependencies ',
124 ' bump knownBroken to <12.5.0 ',
125 ])
126 // remove changelog h1 so it doesnt double render the title
127 .replace(/^#\s+Changelog\s+$\n/gm, '')
128 )
129 },
130 }),
131 'utf-8',
132 )
133
134 nav.children[nav.children.length - 1].children.push({
135 title,
136 url: `${release.url}/${contentPath}`,
137 description: 'Changelog notes for each version',
138 })
139 }
140
141 const unpackRelease = async (release, {contentPath, baseNav, prerelease = false}) => {
142 if (release.prerelease && !prerelease) {
143 log.info(`Skipping ${release.id} due to prerelease ${release.version}`)
144 return
145 }
146
147 log.info(release.id, release)
148
149 const cwd = join(contentPath, release.id)
150
151 const builtPath = join('docs', 'content')
152 const srcPath = join('docs', 'lib', 'content')
153
154 // this is the src dir for the docs that we link to for the edit links
155 release.src = (await gh.pathExists(release.branch, srcPath)) ?? (await gh.pathExists(release.branch, builtPath))
156
157 /* istanbul ignore next */
158 if (!release.src) {
159 throw new Error(`Could not find source dir for ${release.id}`)
160 }
161
162 const nav = await getNav({
163 release,
164 // the nav file can also be in a few different places
165 path:
166 (await gh.pathExists(release.branch, join(srcPath, 'nav.yml'))) ??
167 (await gh.pathExists(release.branch, join('docs', 'nav.yml'))),
168 })
169
170 await fs.rm(cwd, {force: true, recursive: true}).then(() => fs.mkdir(cwd, {recursive: true}))
171
172 const files = await unpackTarball({
173 release,
174 cwd,
175 dir: builtPath,
176 })
177
178 const dirs = ['', ...new Set(files.map(f => dirname(f)))]
179
180 // The docs in the cli contains all the content pagess and the nav
181 // but no index pages. So this builts empty index pages for each
182 // directory with the correct frontmatter. The context is an mdx
183 // component which will end up showing the nav for this directory.
184 const indexes = await Promise.all(
185 dirs.map(async dir => {
186 const path = join(dir, 'index.mdx')
187 const navSection = dir
188 ? nav.children.find(c => posix.basename(c.url) === dir)
189 : baseNav.find(c => posix.basename(c.url) === release.urlPrefix)
190
191 await fs.writeFile(
192 join(cwd, path),
193 Transform.sync('<Index depth="1" />\n', {
194 release,
195 path,
196 frontmatter: {
197 github_path: nav.path,
198 title: navSection.title,
199 shortName: navSection.shortName,
200 },
201 }),
202 'utf-8',
203 )
204
205 return path
206 }),
207 )
208
209 await writeChangelog({
210 release,
211 nav,
212 cwd,
213 srcPath: 'CHANGELOG.md',
214 contentPath: posix.join('using-npm', 'changelog'),
215 })
216
217 log.info(release.id, `${[...files, ...indexes].length} files`)
218
219 return {
220 ...release,
221 nav: nav.children,
222 }
223 }
224
225 module.exports = unpackRelease