support for non-github plugins #254
Massimo Melina committed
Aug 29, 2023 at 14:48 UTC
0152da6ac413cc25ad610f018b3d7e0f8e1cbd41
8 files changed
+150
-86
admin/src/InstalledPlugins.ts
+7
-3
@@ -5,7 +5,7 @@ import { createElement as h, Fragment, ReactNode } from 'react'
5
import { Alert, Box, Link, Tooltip } from '@mui/material'
6
import { DataTable } from './DataTable'
7
import { Delete, Error, PlayCircle, Settings, StopCircle, Upgrade } from '@mui/icons-material'
8
-import { IconBtn, xlate } from './misc'
8
+import { IconBtn, with_, xlate } from './misc'
9
import { formDialog, toast } from './dialog'
10
import _ from 'lodash'
11
import { BoolField, Field, MultiSelectField, NumberField, SelectField, StringField } from '@hfs/mui-grid-form'
@@ -101,11 +101,15 @@ export default function InstalledPlugins({ updates }: { updates?: true }) {
101
102
export function renderName({ row, value }: any) {
103
const { repo } = row
104
- const arr = repo?.split('/')
104
return h(Fragment, {},
105
errorIcon(row.badApi, true),
106
errorIcon(row.error),
108
- ...!repo ? [value] : [ h(Link, { href: 'https://github.com/' + repo, target: 'plugin' }, arr[1]), '\xa0by ', arr[0] ]
107
+ repo?.includes('//') ? h(Link, { href: repo, target: 'plugin' }, value)
108
+ : !repo ? value
109
+ : with_(repo?.split('/'), arr => h(Fragment, {},
110
+ h(Link, { href: 'https://github.com/' + repo, target: 'plugin' }, arr[1]),
111
+ '\xa0by ', arr[0]
112
+ ))
113
)
114
115
function errorIcon(msg: ReactNode, warning=false) {
dev-plugins.md
+21
-3
@@ -45,10 +45,27 @@ All the following properties are optional unless otherwise specified.
45
- `description: string` try to explain what this plugin is for.
46
- `version: number` use progressive numbers to distinguish each release
47
- `apiRequired: number | [min:number,max:number]` declare version(s) for which the plugin is designed for. Mandatory. [Refer to API version history](#api-version-history)
48
-- `repo: string` pointer to a GitHub repo where this plugin is hosted.
48
- `depend: { repo: string, version: number }[]` declare what other plugins this depends on.
49
+- `repo: string | object` pointer to a GitHub repo where this plugin is hosted.
50
+ - the string form is for GitHub repos. Example: "rejetto/file-icons"
51
+ - the object form will point to other custom repo. Object properties:
52
+ - `web: string` link to a web page
53
+ - `main: string` link to the plugin.js (can be relative to `web`)
54
+ - `zip: string` link to the zip with the whole plugin (can be relative to `web`)
55
+ - `zipRoot: string` optional, in case the plugin in the zip is inside a folder
56
+
57
+ Example:
58
+ ```
59
+ {
60
+ "web": "https://github.com/rejetto/file-icons",
61
+ "zip": "/archive/refs/heads/main.zip",
62
+ "zipRoot: "file-icons/main",
63
+ "main": "https://raw.githubusercontent.com/rejetto/file-icons/main/dist/plugin.js"
64
+ }
65
+ ```
66
+ Plugins with custom repos are not included in search results, but the update feature will still work.
67
51
-All the properties above are a bit special and must go in `exports` only (thus, not returned in `init`) and the syntax
68
+WARNING: All the properties above are a bit special and must go in `exports` only (thus, not returned in `init`) and the syntax
69
used must be strictly JSON (thus, no single quotes, only double quotes for strings and objects).
70
71
- `init` described in the previous section.
@@ -146,7 +163,8 @@ The `api` object you get as parameter of the `init` contains the following:
163
164
- `getConnections: Connections[]` retrieve current list of active connections.
165
149
-- `storageDir: string` folder where a plugin is supposed to store run-time data.
166
+- `storageDir: string` folder where a plugin is supposed to store run-time data. This folder is preserved during
167
+ an update of the plugin, while the rest could be deleted.
168
169
- `events: EventEmitter` this is the main events emitter used by HFS.
170
src/api.net.ts
+2
-2
@@ -8,7 +8,7 @@ import {parse} from 'node-html-parser'
8
import _ from 'lodash'
9
import { getIps, getServerStatus } from './listen'
10
import { getProjectInfo } from './github'
11
-import { httpsString } from './util-http'
11
+import { httpString } from './util-http'
12
import { exec } from 'child_process'
13
14
async function getNatInfo() {
@@ -40,7 +40,7 @@ async function getPublicIp() {
40
const prjInfo = await getProjectInfo()
41
for (const urls of _.chunk(_.shuffle(prjInfo.publicIpServices), 2)) // small parallelization
42
try {
43
- return await Promise.any(urls.map(url => httpsString(url).then(res => {
43
+ return await Promise.any(urls.map(url => httpString(url).then(res => {
44
const ip = res.body?.trim()
45
if (!/[.:0-9a-fA-F]/.test(ip))
46
throw Error("bad result: " + ip)
src/api.plugins.ts
+25
-23
@@ -28,7 +28,7 @@ import { HTTP_FAILED_DEPENDENCY, HTTP_NOT_FOUND, HTTP_SERVER_ERROR } from './con
28
const apis: ApiHandlers = {
29
30
get_plugins({}, ctx) {
31
- const list = new SendListReadable({ addAtStart: [ ...mapPlugins(serialize), ...getAvailablePlugins() ] })
31
+ const list = new SendListReadable({ addAtStart: [ ...mapPlugins(serialize), ...getAvailablePlugins().map(serialize) ] })
32
return list.events(ctx, {
33
pluginInstalled: p => list.add(serialize(p)),
34
'pluginStarted pluginStopped pluginUpdated': p => {
@@ -41,34 +41,35 @@ const apis: ApiHandlers = {
41
function serialize(p: Readonly<Plugin> | AvailablePlugin) {
42
const o = 'getData' in p ? Object.assign(_.pick(p, ['id','started']), p.getData())
43
: { ...p } // _.defaults mutates object, and we don't want that
44
+ if (typeof o.repo === 'object') // custom repo
45
+ o.repo = o.repo.web
46
return _.defaults(o, { started: null, badApi: null }) // nulls should be used to be sure to overwrite previous values,
47
}
48
},
49
50
async get_plugin_updates() {
49
- const list = new SendListReadable()
50
- setTimeout(async () => {
51
- const errs = await Promise.all(_.map(getFolder2repo(), async (repo, folder) => {
52
- try {
53
- if (!repo) return
54
- //TODO shouldn't we consider other branches here?
55
- const online = await readOnlinePlugin(repo)
56
- if (!online.apiRequired || online.badApi) return
57
- const disk = getPluginInfo(folder)
58
- if (online.version! > disk.version)
59
- list.add(online)
60
- }
61
- catch (err:any) {
62
- if (err.message === '404') // the plugin is declaring a wrong repo
63
- return
64
- return err.code || err.message
65
- }
66
- }))
67
- for (const x of _.uniq(onlyTruthy(errs)))
68
- list.error(x)
69
- list.close()
51
+ return new SendListReadable({
52
+ async doAtStart(list) {
53
+ const errs = await Promise.all(_.map(getFolder2repo(), async (repo, folder) => {
54
+ try {
55
+ if (!repo) return
56
+ //TODO shouldn't we consider other branches here?
57
+ const online = await readOnlinePlugin(repo)
58
+ if (!online?.apiRequired || online.badApi) return
59
+ const disk = getPluginInfo(folder)
60
+ if (online.version! > disk.version)
61
+ list.add(online)
62
+ } catch (err: any) {
63
+ if (err.message === '404') // the plugin is declaring a wrong repo
64
+ return
65
+ return err.code || err.message
66
+ }
67
+ }))
68
+ for (const x of _.uniq(onlyTruthy(errs)))
69
+ list.error(x)
70
+ list.close()
71
+ }
72
})
71
- return list
73
},
74
75
async start_plugin({ id }) {
@@ -184,6 +185,7 @@ export default apis
185
186
async function checkDependencies(repo: string, branch: string) {
187
const rec = await readOnlinePlugin(repo, branch)
188
+ if (!rec) return
189
const miss = rec.depend && rec.depend.map((dep: any) => {
190
const res = findPluginByRepo(dep.repo)
191
const error = !res ? 'missing'
src/github.ts
+76
-41
@@ -1,17 +1,18 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
import events from './events'
4
-import { httpsString, httpsStream, unzip } from './misc'
4
+import { httpString, httpStream, unzip } from './misc'
5
import {
6
- DISABLING_POSTFIX,
6
+ DISABLING_POSTFIX, findPluginByRepo,
7
getAvailablePlugins,
8
+ getPluginInfo,
9
mapPlugins,
10
parsePluginSource,
10
- PATH as PLUGINS_PATH,
11
+ PATH as PLUGINS_PATH, Repo,
12
} from './plugins'
13
import { ApiError } from './apiMiddleware'
14
import _ from 'lodash'
14
-import { DAY, HFS_REPO, HTTP_BAD_REQUEST, HTTP_CONFLICT } from './const'
15
+import { DAY, HFS_REPO, HTTP_BAD_REQUEST, HTTP_CONFLICT, HTTP_FAILED_DEPENDENCY, HTTP_SERVER_ERROR } from './const'
16
import { rename, rm } from 'fs/promises'
17
import { join } from 'path'
18
import { readFileSync } from 'fs'
@@ -35,39 +36,54 @@ export async function downloadPlugin(repo: string, branch='', overwrite?: boolea
36
console.log('downloading plugin', repo)
37
downloadProgress(repo, true)
38
try {
39
+ if (repo.includes('//')) { // custom repo
40
+ const pl = findPluginByRepo(repo)
41
+ if (!pl)
42
+ return new ApiError(HTTP_BAD_REQUEST, "bad repo")
43
+ const customRepo = ((pl as any).getData?.() || pl).repo
44
+ let url = customRepo?.zip
45
+ if (!url)
46
+ return new ApiError(HTTP_SERVER_ERROR, "bad plugin")
47
+ if (!url.includes('//'))
48
+ url = customRepo.web + url
49
+ return await go(url, pl?.id, customRepo.zipRoot ?? DIST_ROOT)
50
+ }
51
const rec = await getRepoInfo(repo)
52
if (!branch)
53
branch = rec.default_branch
54
const short = repo.split('/')[1] // second part, repo without the owner
55
if (!short)
56
return new ApiError(HTTP_BAD_REQUEST, "bad repo")
44
- const folder2repo = getFolder2repo()
45
- const folder = overwrite ? _.findKey(folder2repo, x => x===repo)! // use existing folder
46
- : folder2repo.hasOwnProperty(short) ? repo.replace('/','-') // longer form only if another plugin is using short form
57
+ const folder = overwrite ? _.findKey(getFolder2repo(), x => x===repo)! // use existing folder
58
+ : getFolder2repo().hasOwnProperty(short) ? repo.replace('/','-') // longer form only if another plugin is using short form
59
: short
48
- const installPath = PLUGINS_PATH + '/' + folder
60
const GITHUB_ZIP_ROOT = short + '-' + branch // GitHub puts everything within this folder
50
- const rootWithinZip = GITHUB_ZIP_ROOT + '/' + DIST_ROOT
51
- const foldersToCopy = [ // from longer to shorter, so we first test the longer
52
- rootWithinZip + '-' + process.platform + '-' + process.arch,
53
- rootWithinZip + '-' + process.platform,
54
- rootWithinZip,
55
- ].map(x => x + '/')
56
- // this zip doesn't have content-length, so we cannot produce progress event
57
- const stream = await httpsStream(`https://github.com/${repo}/archive/refs/heads/${branch}.zip`)
58
- const MAIN = 'plugin.js'
59
- await unzip(stream, async path => {
60
- const folder = foldersToCopy.find(x => path.startsWith(x))
61
- if (!folder || path.endsWith('/')) return false
62
- let dest = path.slice(folder.length)
63
- if (dest === MAIN) // avoid being possibly loaded before the download is complete
64
- dest += DISABLING_POSTFIX
65
- dest = join(installPath, dest)
66
- return rm(dest, { force: true }).then(() => dest, () => false)
67
- })
68
- const main = join(installPath, MAIN)
69
- await rename(main + DISABLING_POSTFIX, main) // we are good now, restore name
70
- return folder
61
+ return await go(`https://github.com/${repo}/archive/refs/heads/${branch}.zip`, folder, GITHUB_ZIP_ROOT + '/' + DIST_ROOT)
62
+
63
+ async function go(url: string, folder: string, zipRoot: string) {
64
+ const installPath = PLUGINS_PATH + '/' + folder
65
+ const foldersToCopy = [ // from longer to shorter, so we first test the longer
66
+ zipRoot + '-' + process.platform + '-' + process.arch,
67
+ zipRoot + '-' + process.platform,
68
+ zipRoot,
69
+ ].map(x => x + '/')
70
+ // github zip doesn't have content-length, so we cannot produce progress event
71
+ const stream = await httpStream(url)
72
+ const MAIN = 'plugin.js'
73
+ await unzip(stream, async path => {
74
+ const folder = foldersToCopy.find(x => path.startsWith(x))
75
+ if (!folder || path.endsWith('/')) return false
76
+ let dest = path.slice(folder.length)
77
+ if (dest === MAIN) // avoid being possibly loaded before the download is complete
78
+ dest += DISABLING_POSTFIX
79
+ dest = join(installPath, dest)
80
+ return rm(dest, { force: true }).then(() => dest, () => false)
81
+ })
82
+ const main = join(installPath, MAIN)
83
+ await rename(main + DISABLING_POSTFIX, main) // we are good now, restore name
84
+ .catch(e => { throw e.code !== 'ENOENT' ? e : new ApiError(HTTP_FAILED_DEPENDENCY, "missing main file") })
85
+ return folder
86
+ }
87
}
88
finally {
89
downloadProgress(repo, undefined)
@@ -79,16 +95,34 @@ export function getRepoInfo(id: string) {
95
}
96
97
export function readGithubFile(uri: string) {
82
- return httpsString('https://raw.githubusercontent.com/' + uri)
98
+ return httpString('https://raw.githubusercontent.com/' + uri)
99
.then(res => res.body)
100
}
101
86
-export async function readOnlinePlugin(repo: string, branch='') {
87
- branch ||= (await getRepoInfo(repo)).default_branch
88
- const res = await readGithubFile(`${repo}/${branch}/${DIST_ROOT}/plugin.js`)
89
- const pl = parsePluginSource(repo, res) // use 'repo' as 'id' client-side
90
- pl.branch = branch || undefined
91
- return pl
102
+export async function readOnlinePlugin(repo: Repo, branch='') {
103
+ if (typeof repo !== 'string') { // non-github plugin
104
+ const folder = _.findKey(getFolder2repo(), x => x === repo)
105
+ if (!folder) throw Error()
106
+ const pl = getPluginInfo(folder)
107
+ let { main } = pl.repo
108
+ if (!main) throw Error("missing repo.main")
109
+ if (!main.includes('//'))
110
+ main = pl.repo.web + main
111
+ const res = await httpString(main)
112
+ if (!res.ok) throw Error("bad repo.main")
113
+ return parsePluginSource(main, res.body) // use 'repo' as 'id' client-side
114
+ }
115
+ const branches = branch ? [branch] : (async function*() {
116
+ yield 'main' // getRepoInfo consumes github-api-quota, so give 'main' a shot first, and if it fails we'll ask
117
+ yield (await getRepoInfo(repo))?.default_branch
118
+ })()
119
+ for await (const b of branches) {
120
+ const res = await readGithubFile(`${repo}/${b}/${DIST_ROOT}/plugin.js`)
121
+ if (!res) continue
122
+ const pl = parsePluginSource(repo, res) // use 'repo' as 'id' client-side
123
+ pl.branch = b || undefined
124
+ return pl
125
+ }
126
}
127
128
export function getFolder2repo() {
@@ -99,7 +133,7 @@ export function getFolder2repo() {
133
134
async function apiGithub(uri: string) {
135
try {
102
- const res = await httpsString('https://api.github.com/'+uri, {
136
+ const res = await httpString('https://api.github.com/'+uri, {
137
headers: {
138
'User-Agent': 'HFS',
139
Accept: 'application/vnd.github.v3+json',
@@ -120,24 +154,25 @@ export async function* searchPlugins(text='') {
154
const projectInfo = await getProjectInfo()
155
const res = await apiGithub('search/repositories?q=topic:hfs-plugin+' + encodeURI(text))
156
for (const it of res.items) {
123
- const repo = it.full_name
157
+ const repo = it.full_name as string
158
if (projectInfo?.plugins_blacklist?.includes(repo)) continue
159
let pl = await readOnlinePlugin(repo, it.default_branch)
126
- if (!pl.apiRequired) continue // mandatory field
160
+ if (!pl?.apiRequired) continue // mandatory field
161
if (pl.badApi) { // we try other branches (starting with 'api')
162
const res = await apiGithub('repos/' + it.full_name + '/branches')
163
const branches: string[] = res.map((x: any) => x?.name)
164
.filter((x: any) => typeof x === 'string' && x.startsWith('api'))
165
.sort().reverse()
166
for (const branch of branches) {
133
- pl = await readOnlinePlugin(it, branch)
167
+ pl = await readOnlinePlugin(repo, branch)
168
+ if (!pl) continue
169
if (!pl.apiRequired)
170
pl.badApi = '-'
171
if (!pl.badApi)
172
break
173
}
174
}
140
- if (pl.badApi)
175
+ if (!pl || pl.badApi)
176
continue
177
Object.assign(pl, { // inject some extra useful fields
178
downloading: downloading[repo],
src/plugins.ts
+9
-4
@@ -103,8 +103,12 @@ export function mapPlugins<T>(cb:(plugin:Readonly<Plugin>, pluginName:string)=>
103
}
104
105
export function findPluginByRepo<T>(repo: string) {
106
- return _.find(plugins, pl => pl.getData()?.repo === repo)
107
- || _.find(availablePlugins, { repo })
106
+ return _.find(plugins, pl => match(pl.getData()))
107
+ || _.find(availablePlugins, match)
108
+
109
+ function match(rec: any) {
110
+ return repo === (rec?.repo?.main ?? rec?.repo)
111
+ }
112
}
113
114
export function getPluginConfigFields(id: string) {
@@ -211,12 +215,13 @@ type PluginMiddleware = (ctx:Koa.Context) => void | Stop | CallMeAfter
215
type Stop = true
216
type CallMeAfter = ()=>any
217
218
+export type Repo = string | { web?: string, main: string, zip?: string, zipRoot?: string }
219
export interface AvailablePlugin {
220
id: string
221
description?: string
222
version?: number
223
apiRequired?: number | [number,number]
219
- repo?: string
224
+ repo?: Repo
225
depend?: { repo: string, version?: number }[]
226
branch?: string
227
badApi?: string
@@ -428,7 +433,7 @@ onProcessExit(() =>
433
export function parsePluginSource(id: string, source: string) {
434
const pl: AvailablePlugin = { id }
435
pl.description = tryJson(/exports.description *= *(".*")/.exec(source)?.[1])
431
- pl.repo = /exports.repo *= *"(.*)"/.exec(source)?.[1]
436
+ pl.repo = tryJson(/exports.repo *= *(.*);? *$/m.exec(source)?.[1])
437
pl.version = Number(/exports.version *= *(\d*\.?\d+)/.exec(source)?.[1]) ?? undefined
438
pl.apiRequired = tryJson(/exports.apiRequired *= *([ \d.,[\]]+)/.exec(source)?.[1]) ?? undefined
439
pl.depend = tryJson(/exports.depend *= *(\[.*\])/m.exec(source)?.[1])?.filter((x: any) =>
src/update.ts
+2
-2
@@ -4,7 +4,7 @@ import { getRepoInfo } from './github'
4
import { argv, HFS_REPO, IS_BINARY, IS_WINDOWS, RUNNING_BETA } from './const'
5
import { dirname, join } from 'path'
6
import { spawn, spawnSync } from 'child_process'
7
-import { httpsStream, onProcessExit, unzip } from './misc'
7
+import { httpStream, onProcessExit, unzip } from './misc'
8
import { createReadStream, renameSync, unlinkSync } from 'fs'
9
import { pluginsWatcher } from './plugins'
10
import { access, chmod, stat } from 'fs/promises'
@@ -84,7 +84,7 @@ export async function update(tag?: string) {
84
throw "asset not found"
85
const url = asset.browser_download_url
86
console.log("downloading", url)
87
- updateSource = await httpsStream(url)
87
+ updateSource = await httpStream(url)
88
}
89
90
const bin = process.execPath
src/util-http.ts
+8
-8
@@ -1,12 +1,12 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
import { RequestOptions } from 'https'
4
-import { IncomingMessage } from 'node:http'
4
+import http, { IncomingMessage } from 'node:http'
5
import https from 'node:https'
6
import { HTTP_TEMPORARY_REDIRECT } from './const'
7
8
-export function httpsString(url: string, options?: XRequestOptions): Promise<IncomingMessage & { ok: boolean, body: string }> {
9
- return httpsStream(url, options).then(res =>
8
+export function httpString(url: string, options?: XRequestOptions): Promise<IncomingMessage & { ok: boolean, body: string }> {
9
+ return httpStream(url, options).then(res =>
10
new Promise(resolve => {
11
let buf = ''
12
res.on('data', chunk => buf += chunk.toString())
@@ -19,17 +19,17 @@ export function httpsString(url: string, options?: XRequestOptions): Promise<Inc
19
}
20
21
export interface XRequestOptions extends RequestOptions { body?: string | Buffer }
22
-export function httpsStream(url: string, { body, ...options }:XRequestOptions ={}): Promise<IncomingMessage> {
22
+export function httpStream(url: string, { body, ...options }:XRequestOptions ={}): Promise<IncomingMessage> {
23
return new Promise((resolve, reject) => {
24
if (body)
25
options.method ||= 'POST'
26
- console.debug("making http request", url)
27
- const req = https.request(url, options, res => {
28
- console.debug("http responded", res.statusCode)
26
+ const proto = url.startsWith('https:') ? https : http
27
+ const req = proto.request(url, options, res => {
28
+ console.debug("http responded", res.statusCode, "to", url)
29
if (!res.statusCode || res.statusCode >= 400)
30
return reject(new Error(String(res.statusCode), { cause: res }))
31
if (res.statusCode === HTTP_TEMPORARY_REDIRECT && res.headers.location)
32
- return resolve(httpsStream(res.headers.location, options))
32
+ return resolve(httpStream(res.headers.location, options))
33
resolve(res)
34
}).on('error', e => {
35
reject((req as any).res || e)