plugins: middleware
Massimo Melina committed
Jan 4, 2022 at 18:41 UTC
49d9a3650084829805406506d367266440a57f4e
16 files changed
+218
-92
README.md
+22
-3
@@ -86,8 +86,27 @@ You'll find the output in `dist` folder.
86
87
Now to run it you should `cd dist` and `node .`
88
89
-# Plugins
89
+# Plug-ins
90
91
We are slowly introducing a plug-ins system.
92
-Each plug-in is a `plugin.yaml` file sub-folder of `plugins` folder.
93
-You can quickly disable a plug-in by appending `-disabled` to the plug-in folder.
92
+Each plug-in is a sub-folder of `plugins` folder.
93
+You can quickly disable a plug-in by appending `-disabled` to the plug-in's folder name.
94
+Plug-ins can be hot-swapped, and at some extent can be edited without restarting the server.
95
+
96
+Each plug-in has access to the same set of features.
97
+Normally you'll have a plug-in that's a theme, and another that's a firewall,
98
+but nothing is preventing a single plug-in from doing both tasks.
99
+
100
+## For plug-in makers
101
+
102
+What a plug-in does is declared in its `plugin.yaml` file.
103
+Supported keys are:
104
+
105
+- `middleware` javascript file exporting a function that will be used as a middleware: it can interfere with http activity.
106
+
107
+ If the function returns `true`, other executions on this http request will be interrupted.
108
+ Return another function if you want to execute it in the "upstream" of middlewares.
109
+
110
+- `frontend_css` path to one or more css files that you want the frontend to load.
111
+
112
+Each plug-in can have a `public` folder, and its files will be accessible at `/~/plugins/PLPUGIN_NAME/FILENAME`.
package-lock.json
+12
-12
@@ -607,9 +607,9 @@
607
}
608
},
609
"node_modules/@types/koa-bodyparser": {
610
- "version": "4.3.4",
611
- "resolved": "https://registry.npmjs.org/@types/koa-bodyparser/-/koa-bodyparser-4.3.4.tgz",
612
- "integrity": "sha512-eNyHGkIxwJKYaZhyxzAVP3taZVWNgAw+Gzl1Zc9vdkfzoBjypTANLwOAb6tC2hGOXS5uw4O08N6jaL6MdrltqQ==",
610
+ "version": "4.3.5",
611
+ "resolved": "https://registry.npmjs.org/@types/koa-bodyparser/-/koa-bodyparser-4.3.5.tgz",
612
+ "integrity": "sha512-NRqqoTtt7cfdDk/KNo+EwCIKRuzPAu/wsaZ7tgIvSIBtNfxuZHYueaLoWdxX3ZftWavQv07NE46TcpyoZGqpgQ==",
613
"dev": true,
614
"dependencies": {
615
"@types/koa": "*"
@@ -3080,9 +3080,9 @@
3080
}
3081
},
3082
"node_modules/object-inspect": {
3083
- "version": "1.11.1",
3084
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.1.tgz",
3085
- "integrity": "sha512-If7BjFlpkzzBeV1cqgT3OSWT3azyoxDGajR+iGnFBfVV2EWyDyWaZZW2ERDjUaY2QM8i5jI3Sj7mhsM4DDAqWA==",
3083
+ "version": "1.12.0",
3084
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz",
3085
+ "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==",
3086
"funding": {
3087
"url": "https://github.com/sponsors/ljharb"
3088
}
@@ -4615,9 +4615,9 @@
4615
}
4616
},
4617
"@types/koa-bodyparser": {
4618
- "version": "4.3.4",
4619
- "resolved": "https://registry.npmjs.org/@types/koa-bodyparser/-/koa-bodyparser-4.3.4.tgz",
4620
- "integrity": "sha512-eNyHGkIxwJKYaZhyxzAVP3taZVWNgAw+Gzl1Zc9vdkfzoBjypTANLwOAb6tC2hGOXS5uw4O08N6jaL6MdrltqQ==",
4618
+ "version": "4.3.5",
4619
+ "resolved": "https://registry.npmjs.org/@types/koa-bodyparser/-/koa-bodyparser-4.3.5.tgz",
4620
+ "integrity": "sha512-NRqqoTtt7cfdDk/KNo+EwCIKRuzPAu/wsaZ7tgIvSIBtNfxuZHYueaLoWdxX3ZftWavQv07NE46TcpyoZGqpgQ==",
4621
"dev": true,
4622
"requires": {
4623
"@types/koa": "*"
@@ -6474,9 +6474,9 @@
6474
"dev": true
6475
},
6476
"object-inspect": {
6477
- "version": "1.11.1",
6478
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.1.tgz",
6479
- "integrity": "sha512-If7BjFlpkzzBeV1cqgT3OSWT3azyoxDGajR+iGnFBfVV2EWyDyWaZZW2ERDjUaY2QM8i5jI3Sj7mhsM4DDAqWA=="
6477
+ "version": "1.12.0",
6478
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz",
6479
+ "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g=="
6480
},
6481
"on-finished": {
6482
"version": "2.3.0",
plugins/example-disabled/plugin.yaml
deleted
-4
@@ -1,4 +0,0 @@
1
-ver: 1
2
-frontend:
3
- load:
4
- css: style.css
plugins/middleware-example-disabled/blocker.js
new
+4
@@ -0,0 +1,4 @@
1
+module.exports = function(ctx) {
2
+ ctx.body = 'This plugin is stopping you ;)'
3
+ return true // true = please stop
4
+}
plugins/middleware-example-disabled/plugin.yaml
new
+2
@@ -0,0 +1,2 @@
1
+ver: 1
2
+middleware: import(blocker.js)
plugins/theme-example-disabled/plugin.yaml
new
+2
@@ -0,0 +1,2 @@
1
+ver: 1
2
+frontend_css: style.css
plugins/theme-example-disabled/public/star.png
renamed
plugins/theme-example-disabled/public/style.css
renamed
src/config.ts
+1
-1
@@ -1,6 +1,6 @@
1
import EventEmitter from 'events'
2
import { argv } from './const'
3
-import { watchLoad } from './misc'
3
+import { watchLoad } from './watchLoad'
4
5
const PATH = 'config.yaml'
6
src/frontEndApis.ts
+3
-3
@@ -6,7 +6,6 @@ import { getCurrentUsername, updateAccount, verifyLogin } from './perm'
6
import { stat } from 'fs/promises'
7
import { ApiHandlers } from './apis'
8
import { plugins } from './plugins'
9
-import { wantArray } from './misc'
9
import { PLUGINS_PUB_URI } from './const'
10
11
export const frontEndApis: ApiHandlers = {
@@ -93,8 +92,9 @@ export const frontEndApis: ApiHandlers = {
92
93
async extras_to_load() {
94
const css = []
96
- for (const k in plugins)
97
- css.push(...wantArray(plugins[k].data.frontend?.load?.css).map(f => PLUGINS_PUB_URI + k + '/' + f))
95
+ for (const [k,plug] of Object.entries(plugins))
96
+ if (plug.frontend_css)
97
+ css.push( ...plug.frontend_css.map(f => PLUGINS_PUB_URI + k + '/' + f) )
98
return { css }
99
},
100
}
src/index.ts
+3
-10
@@ -3,7 +3,7 @@ import mount from 'koa-mount'
3
import bodyParser from 'koa-bodyparser'
4
import { apiMiddleware } from './apis'
5
import { serveFrontend } from './serveFrontend'
6
-import { API_URI, DEV, FRONTEND_URI, PLUGINS_PUB_URI } from './const'
6
+import { API_URI, DEV, FRONTEND_URI} from './const'
7
import { serveFile } from './serveFile'
8
import { vfs } from './vfs'
9
import { isDirectory } from './misc'
@@ -16,9 +16,7 @@ import { zipStreamFromFolder } from './zip'
16
import { frontEndApis } from './frontEndApis'
17
import { log } from './log'
18
import './plugins'
19
-import { createReadStream } from 'fs'
20
-import { PATH as PLUGINS_PATH } from './plugins'
21
-import mime from 'mime-types'
19
+import { pluginsMiddleware } from './plugins'
20
21
const BUILD_TIMESTAMP = ""
22
@@ -32,6 +30,7 @@ app.use(session({
30
maxAge: 30*60_000,
31
}, app))
32
app.use(log())
33
+app.use(pluginsMiddleware())
34
35
// serve apis
36
app.use(mount(API_URI, new Koa()
@@ -55,12 +54,6 @@ app.use(async (ctx, next) => {
54
return await next()
55
if (path.startsWith(FRONTEND_URI))
56
return await serveFrontendPrefixed(ctx,next)
58
- if (path.startsWith(PLUGINS_PUB_URI)) { // expose public plugins' files
59
- const a = path.substring(PLUGINS_PUB_URI.length).split('/')
60
- a.splice(1,0,'public')
61
- ctx.type = mime.lookup(path) || ''
62
- return ctx.body = createReadStream(PLUGINS_PATH + '/' + a.join('/'))
63
- }
57
const decoded = decodeURI(path)
58
const node = await vfs.urlToNode(decoded, ctx)
59
if (!node)
src/misc.ts
+1
-43
@@ -1,8 +1,5 @@
1
import fs from 'fs/promises'
2
import { objSameKeys } from './obj'
3
-import { FSWatcher, watch } from 'fs'
4
-import yaml from 'yaml'
5
-import _ from 'lodash'
3
4
export function enforceFinal(sub:string, s:string) {
5
return s.endsWith(sub) ? s : s+sub
@@ -26,7 +23,7 @@ export function prefix(pre:string, v:string|number, post:string='') {
23
return v ? pre+v+post : ''
24
}
25
29
-export function setHidden(dest: object, src:object) {
26
+export function setHidden(dest: object, src:Record<string,any>) {
27
Object.defineProperties(dest, objSameKeys(src, value => ({ enumerable:false, value })))
28
}
29
@@ -47,45 +44,6 @@ export function wantArray(x:any) {
44
return x == null ? [] : Array.isArray(x) ? x : [x]
45
}
46
50
-// return canceler
51
-export function watchLoad(path:string, parser:(data:any)=>void|Promise<void>) {
52
- let doing = false
53
- let watcher: FSWatcher
54
- const debounced = _.debounce(load, 100)
55
- const timer = setInterval(()=>{
56
- try {
57
- watcher = watch(path, debounced)
58
- debounced()
59
- clearInterval(timer)
60
- }
61
- catch(e){
62
- }
63
- }, 1000)
64
- let running = true
65
- return () => {
66
- if (!running) return
67
- running = false
68
- clearInterval(timer)
69
- watcher?.close()
70
- }
71
-
72
- async function load(){
73
- if (doing) return
74
- doing = true
75
- console.debug('loading', path)
76
- let data: any
77
- try {
78
- data = yaml.parse(await readFileBusy(path))
79
- } catch (e) {
80
- doing = false
81
- console.warn('cannot read', path, String(e))
82
- return
83
- }
84
- await parser(data)
85
- doing = false
86
- }
87
-}
88
-
47
// callback can return undefined to skip element
48
export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterator<IN>, filterMap: (el: IN) => Promise<OUT>) {
49
for await (const x of generator) {
src/perm.ts
+2
-1
@@ -2,7 +2,8 @@ import fs from 'fs/promises'
2
import _ from 'lodash'
3
import yaml from 'yaml'
4
import { hashPassword, verifyPassword } from './crypt'
5
-import { setHidden, wantArray, watchLoad } from './misc'
5
+import { setHidden, wantArray } from './misc'
6
+import { watchLoad } from './watchLoad'
7
import Koa from 'koa'
8
import { subscribeConfig } from './config'
9
src/plugins.ts
+118
-14
@@ -1,9 +1,42 @@
1
-import { watch } from 'fs'
1
+import { createReadStream, watch } from 'fs'
2
import glob from 'fast-glob'
3
-import { watchLoad } from './misc'
3
+import { watchLoad } from './watchLoad'
4
import _ from 'lodash'
5
+import path from 'path'
6
+import { PLUGINS_PUB_URI } from './const'
7
+import mime from 'mime-types'
8
+import Koa from 'koa'
9
+import { WatchLoadCanceller } from './watchLoad'
10
6
-export const PATH = 'plugins'
11
+const PATH = 'plugins'
12
+
13
+export const plugins: Record<string, Plugin> = {}
14
+
15
+export function pluginsMiddleware(): Koa.Middleware {
16
+ return async (ctx, next) => {
17
+ const { path } = ctx
18
+ const after = []
19
+ // run middleware plugins
20
+ for (const pl of Object.values(plugins)) {
21
+ const res = await pl.middleware?.(ctx)
22
+ if (res === true)
23
+ ctx.pluginStopped = true
24
+ if (typeof res === 'function')
25
+ after.push(res)
26
+ }
27
+ // expose public plugins' files
28
+ if (path.startsWith(PLUGINS_PUB_URI)) {
29
+ const a = path.substring(PLUGINS_PUB_URI.length).split('/')
30
+ a.splice(1,0,'public')
31
+ ctx.type = mime.lookup(path) || ''
32
+ return ctx.body = createReadStream(PATH + '/' + a.join('/'))
33
+ }
34
+ if (!ctx.pluginStopped)
35
+ await next()
36
+ for (const f of after)
37
+ await f()
38
+ }
39
+}
40
41
try {
42
const debounced = _.debounce(rescan, 1000)
@@ -14,11 +47,41 @@ catch(e){
47
console.debug('plugins not found')
48
}
49
17
-interface Plugin {
18
- data: any
19
- unwatch: ()=>void
50
+class Plugin {
51
+ constructor(readonly k:string, private data:any, private unwatch:()=>void){
52
+ plugins[k] = this // track this
53
+
54
+ // some validation
55
+ let v = data.frontend_css
56
+ if (typeof v === 'string')
57
+ data.frontend_css = [v]
58
+ else if (v && !Array.isArray(v)) {
59
+ delete data.frontend_css
60
+ console.warn('invalid frontend_css')
61
+ }
62
+ v = data.middleware
63
+ if (v && !(v instanceof Function)) {
64
+ delete data.middleware
65
+ console.warn('invalid middleware')
66
+ }
67
+ }
68
+ get middleware(): undefined | PluginMiddleware {
69
+ return this.data.middleware
70
+ }
71
+ get frontend_css(): undefined | string[] {
72
+ return this.data.frontend_css
73
+ }
74
+
75
+ unload() {
76
+ console.log('unloading plugin', this.k)
77
+ delete plugins[this.k]
78
+ this.unwatch()
79
+ }
80
}
21
-export const plugins: Record<string, Plugin> = {}
81
+
82
+type PluginMiddleware = (ctx:Koa.Context) => void | Stop | CallMeAfter
83
+type Stop = true
84
+type CallMeAfter = ()=>void
85
86
async function rescan() {
87
console.debug('scanning plugins')
@@ -29,18 +92,59 @@ async function rescan() {
92
found.push(k)
93
if (plugins[k]) // already loaded
94
continue
32
- const unwatch = watchLoad(f, data => {
95
+ const unwatch = watchLoad(f, async data => {
96
console.log('loading plugin', k)
34
- plugins[k] = { data, unwatch }
97
+ const importCanceller = await resolveImport(data, path.resolve(PATH)+'/'+k+'/')
98
+ new Plugin(k, data, () => {
99
+ unwatch()
100
+ importCanceller()
101
+ })
102
})
103
}
104
for (const k in plugins)
105
if (!found.includes(k))
39
- unloadPlugin(k)
106
+ plugins[k].unload()
107
+}
108
+
109
+const re = /^ *import\((.+)\) *$/
110
+async function resolveImport(x: Record<string,any>, basePath: string) {
111
+ const cancellers: WatchLoadCanceller[] = []
112
+ for (const k in x) {
113
+ let v = x[k]
114
+ if (!v)
115
+ continue
116
+ if (typeof v === 'object')
117
+ await resolveImport(v, basePath)
118
+ else if (typeof v === 'string' && re.test(v)) {
119
+ const fn = re.exec(v)![1].replace(/\\/g, '//')
120
+ if (fn.includes('..') || fn.startsWith('/') || fn.includes(':'))
121
+ continue
122
+ const path = basePath + fn
123
+ await new Promise(resolve => // wait for first execution, so that the caller sees imported stuff instead of string
124
+ cancellers.push(watchLoad(path, async () => {
125
+ try {
126
+ x[k] = (await import(path)).default
127
+ deleteModule(path) // avoid caching
128
+ console.log('plugin imported', fn)
129
+ } catch (e) {
130
+ console.log('plugin error importing', fn, String(e))
131
+ delete x[k]
132
+ }
133
+ resolve(0)
134
+ })) )
135
+ }
136
+ }
137
+ return () => {
138
+ for (const c of cancellers)
139
+ c()
140
+ }
141
}
142
42
-function unloadPlugin(k: string) {
43
- console.log('unloading plugin', k)
44
- plugins[k]?.unwatch()
45
- delete plugins[k]
143
+function deleteModule(name: string) {
144
+ let solvedName = require.resolve(name)
145
+ let mod = require.cache[solvedName]
146
+ if (!mod) return
147
+ for (const child of mod.children)
148
+ deleteModule(child.filename)
149
+ delete require.cache[solvedName]
150
}
src/watchLoad.ts
new
+47
@@ -0,0 +1,47 @@
1
+import { FSWatcher, watch } from 'fs'
2
+import _ from 'lodash'
3
+import yaml from 'yaml'
4
+import { readFileBusy } from './misc'
5
+
6
+export type WatchLoadCanceller = () => void
7
+
8
+export function watchLoad(path:string, parser:(data:any)=>void|Promise<void>): WatchLoadCanceller {
9
+ let doing = false
10
+ let watcher: FSWatcher
11
+ const debounced = _.debounce(load, 100)
12
+ const timer = setInterval(()=>{
13
+ try {
14
+ watcher = watch(path, debounced)
15
+ debounced()
16
+ clearInterval(timer)
17
+ }
18
+ catch(e){
19
+ }
20
+ }, 1000)
21
+ let running = true
22
+ return () => {
23
+ if (!running) return
24
+ running = false
25
+ clearInterval(timer)
26
+ watcher?.close()
27
+ }
28
+
29
+ async function load(){
30
+ if (doing) return
31
+ doing = true
32
+ console.debug('loading', path)
33
+ let data: any
34
+ try {
35
+ data = await readFileBusy(path)
36
+ if (path.endsWith('.yaml'))
37
+ data = yaml.parse(data)
38
+ } catch (e) {
39
+ doing = false
40
+ console.warn('cannot read', path, String(e))
41
+ return
42
+ }
43
+ await parser(data)
44
+ doing = false
45
+ }
46
+}
47
+
todo.md
+1
-1
@@ -9,7 +9,7 @@
9
- user.redirect
10
- config: bans
11
- config: min disk space
12
-- link to parent folder in the list (as an option of the frontend?)
12
+- link to parent folder in the list (as an option of the frontend? plugin?)
13
- archive for search results
14
- archive only selected files
15
- https