better code: split files
Massimo Melina committed
May 14, 2022 at 14:48 UTC
1c1f8c2f25db1ed6caf9341d523067c42388f230
2 files changed
+139
-127
server/src/api.plugins.ts
+27
-127
@@ -4,20 +4,16 @@ import {
4
getAvailablePlugins,
5
getPluginConfigFields,
6
mapPlugins,
7
- parsePluginSource,
7
Plugin, pluginsConfig,
9
- PATH as PLUGINS_PATH, isPluginRunning, enablePlugin, getPluginInfo, rescan
8
+ PATH as PLUGINS_PATH, isPluginRunning, enablePlugin, getPluginInfo
9
} from './plugins'
10
import _ from 'lodash'
11
import assert from 'assert'
13
-import { httpsString, httpsStream, objSameKeys, onOff, same, wait } from './misc'
14
-import { ApiError, ApiHandlers, sendList } from './apiMiddleware'
12
+import { objSameKeys, onOff, same, wait } from './misc'
13
+import { ApiHandlers, sendList } from './apiMiddleware'
14
import events from './events'
16
-import unzipper from 'unzipper'
17
-import { createWriteStream, mkdirSync } from 'fs'
15
import { rm } from 'fs/promises'
19
-
20
-const DIST_ROOT = 'dist/'
16
+import { downloadPlugin, getRepo2folder, getRepoInfo, readOnlinePlugin, searchPlugins } from './github'
17
18
const apis: ApiHandlers = {
19
@@ -86,62 +82,42 @@ const apis: ApiHandlers = {
82
search_online_plugins({ text }, ctx) {
83
const list = sendList()
84
const repo2folder = getRepo2folder()
89
- // do our job in then() so api can return in the meantime
90
- apiGithub('search/repositories?q=topic:hfs-plugin+' + encodeURI(text)).then(async res => {
91
- for (const it of res.items) {
92
- const repo = it.full_name
93
- let pl = await readOnlinePlugin(it)
94
- if (!pl.apiRequired) continue // mandatory field
95
- if (pl.badApi) { // we try other branches (starting with 'api')
96
- const branches: string[] = (await apiGithub('repos/' + it.full_name + '/branches'))
97
- .map((x:any) => x.name).filter((x: string) => x.startsWith('api')).sort()
98
- for (const branch of branches) {
99
- pl = await readOnlinePlugin(it, branch)
100
- if (!pl.apiRequired)
101
- pl.badApi = '-'
102
- if (!pl.badApi)
103
- break
104
- }
85
+ setTimeout(async () => {
86
+ try {
87
+ for await (const pl of searchPlugins(text)) {
88
+ const repo = pl.id
89
+ Object.assign(pl, { installed: repo2folder[repo] })
90
+ list.add(pl)
91
+ // watch for events about this plugin, until this request is closed
92
+ ctx.req.on('close', onOff(events, {
93
+ pluginInstalled: p => {
94
+ if (p.repo === repo)
95
+ list.update({ id: repo }, { installed: true })
96
+ },
97
+ pluginUninstalled: id => {
98
+ if (repo === _.findKey(repo2folder, x => x === id))
99
+ list.update({ id: repo }, { installed: false })
100
+ },
101
+ ['pluginDownload_'+repo](status) {
102
+ list.update({ id: repo }, { downloading: status ?? null })
103
+ }
104
+ }) )
105
}
106
- if (pl.badApi)
107
- continue
108
- Object.assign(pl, { // inject some extra useful fields
109
- downloading: downloading[repo],
110
- installed: repo2folder[repo]
111
- })
112
- list.add(pl)
113
- // watch for events about this plugin, until this request is closed
114
- ctx.req.on('close', onOff(events, {
115
- pluginInstalled: p => {
116
- if (p.repo === repo)
117
- list.update({ id: repo }, { installed: true })
118
- },
119
- pluginUninstalled: id => {
120
- if (repo === _.findKey(repo2folder, x => x === id))
121
- list.update({ id: repo }, { installed: false })
122
- },
123
- ['pluginDownload_'+repo](status) {
124
- list.update({ id: repo }, { downloading: status ?? null })
125
- }
126
- }) )
106
+ }
107
+ catch (err: any) {
108
+ list.error(err.code || err.message)
109
}
110
list.end()
129
- }, (err: any) => {
130
- list.error(err.code || err.message)
111
})
112
return list.return
113
},
114
115
async download_plugin(pl) {
136
- if (downloading[pl.id])
137
- return new ApiError(409, "already downloading")
116
await downloadPlugin(pl.id, pl.branch)
117
return {}
118
},
119
120
async update_plugin(pl) {
143
- if (downloading[pl.id])
144
- return new ApiError(409, "already downloading")
121
await downloadPlugin(pl.id, pl.branch, true)
122
return {}
123
},
@@ -157,80 +133,4 @@ const apis: ApiHandlers = {
133
134
}
135
160
-type DownloadStatus = true | undefined
161
-const downloading: Record<string, DownloadStatus> = {}
162
-
163
-function downloadProgress(id: string, status: DownloadStatus) {
164
- if (status === undefined)
165
- delete downloading[id]
166
- else
167
- downloading[id] = status
168
- events.emit('pluginDownload_'+id, status)
169
-}
170
-
171
-async function downloadPlugin(repo: string, branch='', overwrite?: boolean) {
172
- downloadProgress(repo, true)
173
- const rec = await getRepoInfo(repo)
174
- if (!branch)
175
- branch = rec.default_branch
176
- const url = `https://github.com/${repo}/archive/refs/heads/${branch}.zip`
177
- const res = await httpsStream(url)
178
- const repo2 = repo.split('/')[1] // second part, repo without the owner
179
- const repo2clash = !overwrite
180
- && (getAvailablePlugins().find(x => x.id === repo2) || mapPlugins(x => x.id === repo2).some(Boolean))
181
- const pluginFolder = repo2clash ? repo.replace('/','-') : repo2 // longer form only if necessary
182
- const installFolder = PLUGINS_PATH + '/' + pluginFolder
183
- const GITHUB_ZIP_ROOT = repo2 + '-' + (branch) // github puts everything within this folder
184
- const rootWithinZip = GITHUB_ZIP_ROOT + '/' + DIST_ROOT
185
- return new Promise(resolve =>
186
- res.pipe(unzipper.Parse())
187
- .on('entry', entry => {
188
- const { path, type } = entry
189
- if (!path.startsWith(rootWithinZip))
190
- return entry.autodrain()
191
- const dest = installFolder + '/' + path.slice(rootWithinZip.length)
192
- if (type === 'File')
193
- return entry.pipe(createWriteStream(dest))
194
- mkdirSync(dest, { recursive: true }) // easy way be sure to have the folder ready before proceeding
195
- })
196
- .on('close', () => {
197
- rescan() // workaround: for some reason, operations above are not triggering the rescan of the watched folder. Let's invoke it.
198
- resolve(undefined)
199
- downloadProgress(repo, undefined)
200
- }))
201
-}
202
-
136
export default apis
204
-
205
-async function apiGithub(uri: string) {
206
- const res = await httpsString('https://api.github.com/'+uri, {
207
- headers: {
208
- 'User-Agent': 'HFS',
209
- Accept: 'application/vnd.github.v3+json',
210
- }
211
- })
212
- if (!res.ok)
213
- throw res.statusCode
214
- return JSON.parse(res.body)
215
-}
216
-
217
-function getRepoInfo(id: string) {
218
- return apiGithub('repos/'+id)
219
-}
220
-
221
-async function readOnlinePlugin(repoInfo: { full_name: string, default_branch: string }, branch='') {
222
- const url = `https://raw.githubusercontent.com/${repoInfo.full_name}/${branch || repoInfo.default_branch}/${DIST_ROOT}plugin.js`
223
- const res = await httpsString(url)
224
- if (!res.ok)
225
- throw res.statusCode
226
- const pl = parsePluginSource(repoInfo.full_name, res.body) // use 'repo' as 'id' client-side
227
- pl.branch = branch || undefined
228
- return pl
229
-}
230
-
231
-function getRepo2folder() {
232
- const ret = Object.fromEntries(getAvailablePlugins().map(x => [x.repo, x.id]))
233
- Object.assign(ret, Object.fromEntries(mapPlugins(x => [x.getData().repo, x.id]))) // started ones
234
- delete ret.undefined
235
- return ret
236
-}
server/src/github.ts
new
+112
@@ -0,0 +1,112 @@
1
+import events from './events'
2
+import { httpsStream, httpsString } from './misc'
3
+import { getAvailablePlugins, mapPlugins, parsePluginSource, PATH as PLUGINS_PATH, rescan } from './plugins'
4
+import unzipper from 'unzipper'
5
+import { createWriteStream, mkdirSync } from 'fs'
6
+import { ApiError } from './apiMiddleware'
7
+
8
+const DIST_ROOT = 'dist/'
9
+
10
+type DownloadStatus = true | undefined
11
+const downloading: Record<string, DownloadStatus> = {}
12
+
13
+function downloadProgress(id: string, status: DownloadStatus) {
14
+ if (status === undefined)
15
+ delete downloading[id]
16
+ else
17
+ downloading[id] = status
18
+ events.emit('pluginDownload_'+id, status)
19
+}
20
+
21
+export async function downloadPlugin(repo: string, branch='', overwrite?: boolean) {
22
+ if (downloading[repo])
23
+ return new ApiError(409, "already downloading")
24
+ downloadProgress(repo, true)
25
+ const rec = await getRepoInfo(repo)
26
+ if (!branch)
27
+ branch = rec.default_branch
28
+ const url = `https://github.com/${repo}/archive/refs/heads/${branch}.zip`
29
+ const res = await httpsStream(url)
30
+ const repo2 = repo.split('/')[1] // second part, repo without the owner
31
+ const repo2clash = !overwrite
32
+ && (getAvailablePlugins().find(x => x.id === repo2) || mapPlugins(x => x.id === repo2).some(Boolean))
33
+ const pluginFolder = repo2clash ? repo.replace('/','-') : repo2 // longer form only if necessary
34
+ const installFolder = PLUGINS_PATH + '/' + pluginFolder
35
+ const GITHUB_ZIP_ROOT = repo2 + '-' + (branch) // github puts everything within this folder
36
+ const rootWithinZip = GITHUB_ZIP_ROOT + '/' + DIST_ROOT
37
+ return new Promise(resolve =>
38
+ res.pipe(unzipper.Parse())
39
+ .on('entry', entry => {
40
+ const { path, type } = entry
41
+ if (!path.startsWith(rootWithinZip))
42
+ return entry.autodrain()
43
+ const dest = installFolder + '/' + path.slice(rootWithinZip.length)
44
+ if (type === 'File')
45
+ return entry.pipe(createWriteStream(dest))
46
+ mkdirSync(dest, { recursive: true }) // easy way be sure to have the folder ready before proceeding
47
+ })
48
+ .on('close', () => {
49
+ rescan() // workaround: for some reason, operations above are not triggering the rescan of the watched folder. Let's invoke it.
50
+ resolve(undefined)
51
+ downloadProgress(repo, undefined)
52
+ }))
53
+}
54
+
55
+export function getRepoInfo(id: string) {
56
+ return apiGithub('repos/'+id)
57
+}
58
+
59
+export async function readOnlinePlugin(repoInfo: { full_name: string, default_branch: string }, branch='') {
60
+ const url = `https://raw.githubusercontent.com/${repoInfo.full_name}/${branch || repoInfo.default_branch}/${DIST_ROOT}plugin.js`
61
+ const res = await httpsString(url)
62
+ if (!res.ok)
63
+ throw res.statusCode
64
+ const pl = parsePluginSource(repoInfo.full_name, res.body) // use 'repo' as 'id' client-side
65
+ pl.branch = branch || undefined
66
+ return pl
67
+}
68
+
69
+export function getRepo2folder() {
70
+ const ret = Object.fromEntries(getAvailablePlugins().map(x => [x.repo, x.id]))
71
+ Object.assign(ret, Object.fromEntries(mapPlugins(x => [x.getData().repo, x.id]))) // started ones
72
+ delete ret.undefined
73
+ return ret
74
+}
75
+
76
+async function apiGithub(uri: string) {
77
+ const res = await httpsString('https://api.github.com/'+uri, {
78
+ headers: {
79
+ 'User-Agent': 'HFS',
80
+ Accept: 'application/vnd.github.v3+json',
81
+ }
82
+ })
83
+ if (!res.ok)
84
+ throw res.statusCode
85
+ return JSON.parse(res.body)
86
+}
87
+
88
+export async function* searchPlugins(text: string) {
89
+ const res = await apiGithub('search/repositories?q=topic:hfs-plugin+' + encodeURI(text))
90
+ for (const it of res.items) {
91
+ const repo = it.full_name
92
+ let pl = await readOnlinePlugin(it)
93
+ if (!pl.apiRequired) continue // mandatory field
94
+ if (pl.badApi) { // we try other branches (starting with 'api')
95
+ const branches: string[] = (await apiGithub('repos/' + it.full_name + '/branches'))
96
+ .map((x: any) => x.name).filter((x: string) => x.startsWith('api')).sort()
97
+ for (const branch of branches) {
98
+ pl = await readOnlinePlugin(it, branch)
99
+ if (!pl.apiRequired)
100
+ pl.badApi = '-'
101
+ if (!pl.badApi)
102
+ break
103
+ }
104
+ }
105
+ if (pl.badApi)
106
+ continue
107
+ Object.assign(pl, { // inject some extra useful fields
108
+ downloading: downloading[repo],
109
+ })
110
+ yield pl
111
+ }
112
+}