@cryptotaxi247 / documentation-1
Code Commits Issues Pulls Sessions CI/CD
reggi/cache-based-on-version
@reggi/path-to-regexp dependabot/npm_and_yarn/main/copy-to-clipboard-4.0.2 dependabot/npm_and_yarn/main/eslint-10.4.0 dependabot/npm_and_yarn/main/npmcli/eslint-config-7.0.0 dependabot/npm_and_yarn/main/proc-log-7.0.0 dependabot/npm_and_yarn/npm_and_yarn-826852524d dependabot/npm_and_yarn/npm_and_yarn-ab9a7f4bc2 deprecate-totp-2fa dhei/classic-tokens gat-bypass-2fa-docs jpg619/fix-accessibility-content-flow jpg619/version-bump-tar-2 kartykp/gat-bypass-2fa-docs kartykp/upgrade-path-to-regex main maitxn/version-bump-tar patch-1 reggi/cache-based-on-version reggi/dev-engines reggi/fix-transform-prettier reggi/overrides update-search-sensitivity
js 236 lines 6.92 KB
Raw
1 const pacote = require('pacote')
2 const tar = require('tar')
3 const {join, sep, dirname, posix} = require('path')
4 const fs = require('fs/promises')
5 const yaml = require('yaml')
6 const Transform = require('./transform')
7 const gh = require('./gh')
8 const log = require('./log')
9
10 const whackAMoleReplace = (s, replacements) => {
11 // some known content in changelogs that is problematic for mdx this is
12 // a bit of whack-a-mole but is necessary since markdown in the CLI is
13 // different from the mdx v2 we parse for the docs site
14 for (const rep of replacements) {
15 s = s.replace(rep, rep.replace(/([<>{}])/g, '\\$1'))
16 }
17 return s
18 }
19
20 const unpackTarball = async ({release, cwd, dir}) => {
21 const strip = 1
22 const result = []
23 const dirParts = dir.split(sep)
24
25 log.verbose('tarball', release.resolved, {cwd, dir})
26
27 const format = s => {
28 // a few specific replacements that cause problems for mdx
29 return (
30 whackAMoleReplace(s, [
31 '{npm-version} node/{node-version} {platform} {arch} workspaces/{workspaces} {ci}',
32 'node/{process.version} {process.platform} {process.arch}',
33 'Default: {prefix}/etc/npmrc',
34 ])
35 // we cant remove all emails since all except this one are in code blocks
36 .replace(/(:: )<(i@izs\.me)>/g, '$1[$2](mailto:$2)')
37 // the v6 version of the funding page has json not inside a code
38 .replace(/(:\n\n)(\s{4}"funding": {)/g, '$1```json\n$2')
39 .replace(/^(\s{4}]$)(\n\n)/gm, '$1\n```$2')
40 // anchor links to markdown. this regex does need to match spaces and newlines since the source markdown
41 // already has some links where attributes are separated by newlines
42 .replace(
43 /<a[\s\n]href="(.*?)"(?:[\s\n]target="_blank")?(?:[\s\n]rel="[a-z\s]+")?>(.*?)<\/a>/g,
44 (_, href, text) => `[${href}](${text.trim()})`,
45 )
46 )
47 }
48
49 const extract = () =>
50 tar.x({
51 cwd,
52 strip: dirParts.length + strip,
53 transform: ({path}) => {
54 result.push(path)
55 log.verbose(release.id, path)
56 return new Transform({
57 path,
58 release,
59 format,
60 })
61 },
62 filter: path => {
63 const pathParts = path.split(posix.sep)
64 const prefixParts = pathParts.slice(strip, dirParts.length + strip)
65 return join(...prefixParts) === join(...dirParts)
66 },
67 })
68
69 await pacote.tarball.stream(
70 `npm@${release.version}`,
71 stream =>
72 new Promise((res, rej) => {
73 stream.on('end', res)
74 stream.on('error', rej)
75 stream.pipe(extract())
76 }),
77 )
78
79 await Promise.all(result.map(f => fs.rename(join(cwd, f), join(cwd, f.replace('.md', '.mdx')))))
80
81 return result
82 }
83
84 const getNav = async ({path, release}) => {
85 const nav = await gh.getFile({ref: release.branch, path})
86
87 const rewriteUrls = nodes =>
88 nodes?.map(n => {
89 n.url = release.url + n.url
90 n.children = rewriteUrls(n.children)
91 return n
92 })
93
94 return {
95 path,
96 children: rewriteUrls(yaml.parse(nav.toString())),
97 }
98 }
99
100 const writeChangelog = async ({release, nav, cwd, srcPath, contentPath}) => {
101 const title = 'Changelog'
102 const changelog = await gh.getFile({ref: release.branch, path: srcPath})
103
104 await fs.writeFile(
105 join(cwd, contentPath + '.mdx'),
106 Transform.sync(changelog, {
107 release,
108 path: contentPath,
109 frontmatter: {
110 github_path: srcPath,
111 title,
112 },
113 format: s => {
114 // some known content in changelogs that is problematic for mdx this is
115 // a bit of whack-a-mole but is necessary since markdown in the CLI is
116 // different from the mdx v2 we parse for the docs site
117 return (
118 whackAMoleReplace(s, [
119 ' support for node <=16.13 ',
120 '<->',
121 ' npm install <folder> ',
122 ' --replace-registry-host=<npmjs|always|never> ',
123 'bundledDependencies -> bundleDependencies ',
124 ' bump knownBroken to <12.5.0 ',
125 ])
126 // remove changelog h1 so it doesnt double render the title
127 .replace(/^#\s+Changelog\s+$\n/gm, '')
128 )
129 },
130 }),
131 'utf-8',
132 )
133
134 nav.children[nav.children.length - 1].children.push({
135 title,
136 url: `${release.url}/${contentPath}`,
137 description: 'Changelog notes for each version',
138 })
139 }
140
141 const unpackRelease = async (release, {cache, contentPath, baseNav, prerelease = false}) => {
142 if (cache) {
143 const currentVersion = await pacote.manifest(`npm@${release.version}`).then(manifest => manifest.version)
144 if (cache.same(release.id, currentVersion)) {
145 log.info(`Skipping ${release.id} due to cache`)
146 return
147 }
148 cache.set(release.id, currentVersion)
149 }
150
151 if (release.prerelease && !prerelease) {
152 log.info(`Skipping ${release.id} due to prerelease ${release.version}`)
153 return
154 }
155
156 log.info(release.id, release)
157
158 const cwd = join(contentPath, release.id)
159
160 const builtPath = join('docs', 'content')
161 const srcPath = join('docs', 'lib', 'content')
162
163 // this is the src dir for the docs that we link to for the edit links
164 release.src = (await gh.pathExists(release.branch, srcPath)) ?? (await gh.pathExists(release.branch, builtPath))
165
166 /* istanbul ignore next */
167 if (!release.src) {
168 throw new Error(`Could not find source dir for ${release.id}`)
169 }
170
171 const nav = await getNav({
172 release,
173 // the nav file can also be in a few different places
174 path:
175 (await gh.pathExists(release.branch, join(srcPath, 'nav.yml'))) ??
176 (await gh.pathExists(release.branch, join('docs', 'nav.yml'))),
177 })
178
179 await fs.rm(cwd, {force: true, recursive: true}).then(() => fs.mkdir(cwd, {recursive: true}))
180
181 const files = await unpackTarball({
182 release,
183 cwd,
184 dir: builtPath,
185 })
186
187 const dirs = ['', ...new Set(files.map(f => dirname(f)))]
188
189 // The docs in the cli contains all the content pagess and the nav
190 // but no index pages. So this builts empty index pages for each
191 // directory with the correct frontmatter. The context is an mdx
192 // component which will end up showing the nav for this directory.
193 const indexes = await Promise.all(
194 dirs.map(async dir => {
195 const path = join(dir, 'index.mdx')
196 const navSection = dir
197 ? nav.children.find(c => posix.basename(c.url) === dir)
198 : baseNav.find(c => posix.basename(c.url) === release.urlPrefix)
199
200 await fs.writeFile(
201 join(cwd, path),
202 Transform.sync('<Index depth="1" />\n', {
203 release,
204 path,
205 frontmatter: {
206 github_path: nav.path,
207 title: navSection.title,
208 shortName: navSection.shortName,
209 },
210 }),
211 'utf-8',
212 )
213
214 return path
215 }),
216 )
217
218 await writeChangelog({
219 release,
220 nav,
221 cwd,
222 srcPath: 'CHANGELOG.md',
223 contentPath: posix.join('using-npm', 'changelog'),
224 })
225
226 log.info(release.id, `${[...files, ...indexes].length} files`)
227
228 return {
229 ...release,
230 nav: nav.children,
231 }
232 }
233
234 module.exports = {
235 unpackRelease,
236 }