max-kbps
Massimo Melina committed
Jan 8, 2022 at 11:48 UTC
398534e22bece19b82d26847bfba662bf000b4d4
10 files changed
+238
-4
README.md
+4
-1
@@ -20,6 +20,7 @@ You won't find all previous features here (yet), but still we got:
20
- simple website serving
21
- plug-ins
22
- log file
23
+- speed throttler
24
25
# Configuration
26
@@ -38,7 +39,9 @@ Supported entries are:
39
E.g.: `"*.jpg": image/jpeg`
40
You can specify multiple entries, or separate multiple file masks with a p|pe.
41
You can use the special value `auto` to attempt automatic detection.
41
-
42
+- `max-kbps` throttle output speed. Default is Infinity.
43
+- `max-kbps-per-ip` throttle out speed on a per-ip basis. Default is Infinity.
44
+
45
## Virtual File System (VFS)
46
47
The virtual file system is a tree of files and folders, collectively called *nodes*.
config.yaml
+3
-1
@@ -1,4 +1,6 @@
1
-port: 80
1
+#port: 80
2
+#max-kbps: 1000
3
+#max-kbps-per-ip: 500
4
mime:
5
"*.jpg|*.png|*.mp3|*.txt": auto
6
vfs:
package-lock.json
+93
@@ -19,8 +19,10 @@
19
"koa-compress": "^5.1.0",
20
"koa-mount": "^4.0.0",
21
"koa-session": "^6.2.0",
22
+ "limiter": "^2.1.0",
23
"lodash": "^4.17.21",
24
"minimist": "^1.2.5",
25
+ "stream-throttle": "^0.1.3",
26
"tssrp6a": "^3.0.0",
27
"yaml": "^2.0.0-10"
28
},
@@ -42,6 +44,7 @@
44
"@types/minimist": "^1.2.2",
45
"@types/mocha": "^9.0.0",
46
"@types/node": "^16.11.12",
47
+ "@types/stream-throttle": "^0.1.1",
48
"axios": "^0.24.0",
49
"mocha": "^9.1.3",
50
"nm-prune": "^5.0.0",
@@ -733,6 +736,15 @@
736
"@types/node": "*"
737
}
738
},
739
+ "node_modules/@types/stream-throttle": {
740
+ "version": "0.1.1",
741
+ "resolved": "https://registry.npmjs.org/@types/stream-throttle/-/stream-throttle-0.1.1.tgz",
742
+ "integrity": "sha512-blC1VrTJBPET4IDEpRO05L0ks7cqyyc4XBRnQVroOLkcE8iaG6MYEQ/RSU7CQ3uIGk4r7wEH0W3ifbQqCVJVAA==",
743
+ "dev": true,
744
+ "dependencies": {
745
+ "@types/node": "*"
746
+ }
747
+ },
748
"node_modules/@ungap/promise-all-settled": {
749
"version": "1.1.2",
750
"resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
@@ -1230,6 +1242,11 @@
1242
"node": ">=0.1.90"
1243
}
1244
},
1245
+ "node_modules/commander": {
1246
+ "version": "2.20.3",
1247
+ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
1248
+ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
1249
+ },
1250
"node_modules/compressible": {
1251
"version": "2.0.18",
1252
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
@@ -2239,6 +2256,11 @@
2256
"graceful-fs": "^4.1.6"
2257
}
2258
},
2259
+ "node_modules/just-performance": {
2260
+ "version": "4.3.0",
2261
+ "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz",
2262
+ "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q=="
2263
+ },
2264
"node_modules/keygrip": {
2265
"version": "1.1.0",
2266
"resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz",
@@ -2417,6 +2439,14 @@
2439
"node": ">=8"
2440
}
2441
},
2442
+ "node_modules/limiter": {
2443
+ "version": "2.1.0",
2444
+ "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz",
2445
+ "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==",
2446
+ "dependencies": {
2447
+ "just-performance": "4.3.0"
2448
+ }
2449
+ },
2450
"node_modules/lines-and-columns": {
2451
"version": "1.2.4",
2452
"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -3684,6 +3714,26 @@
3714
"node": ">= 0.6"
3715
}
3716
},
3717
+ "node_modules/stream-throttle": {
3718
+ "version": "0.1.3",
3719
+ "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz",
3720
+ "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=",
3721
+ "dependencies": {
3722
+ "commander": "^2.2.0",
3723
+ "limiter": "^1.0.5"
3724
+ },
3725
+ "bin": {
3726
+ "throttleproxy": "bin/throttleproxy.js"
3727
+ },
3728
+ "engines": {
3729
+ "node": ">= 0.10.0"
3730
+ }
3731
+ },
3732
+ "node_modules/stream-throttle/node_modules/limiter": {
3733
+ "version": "1.1.5",
3734
+ "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz",
3735
+ "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA=="
3736
+ },
3737
"node_modules/string-width": {
3738
"version": "4.2.3",
3739
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
@@ -4746,6 +4796,15 @@
4796
"@types/node": "*"
4797
}
4798
},
4799
+ "@types/stream-throttle": {
4800
+ "version": "0.1.1",
4801
+ "resolved": "https://registry.npmjs.org/@types/stream-throttle/-/stream-throttle-0.1.1.tgz",
4802
+ "integrity": "sha512-blC1VrTJBPET4IDEpRO05L0ks7cqyyc4XBRnQVroOLkcE8iaG6MYEQ/RSU7CQ3uIGk4r7wEH0W3ifbQqCVJVAA==",
4803
+ "dev": true,
4804
+ "requires": {
4805
+ "@types/node": "*"
4806
+ }
4807
+ },
4808
"@ungap/promise-all-settled": {
4809
"version": "1.1.2",
4810
"resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
@@ -5106,6 +5165,11 @@
5165
"resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
5166
"integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs="
5167
},
5168
+ "commander": {
5169
+ "version": "2.20.3",
5170
+ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
5171
+ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
5172
+ },
5173
"compressible": {
5174
"version": "2.0.18",
5175
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
@@ -5857,6 +5921,11 @@
5921
"graceful-fs": "^4.1.6"
5922
}
5923
},
5924
+ "just-performance": {
5925
+ "version": "4.3.0",
5926
+ "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz",
5927
+ "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q=="
5928
+ },
5929
"keygrip": {
5930
"version": "1.1.0",
5931
"resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz",
@@ -6006,6 +6075,14 @@
6075
"package-json": "^6.3.0"
6076
}
6077
},
6078
+ "limiter": {
6079
+ "version": "2.1.0",
6080
+ "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz",
6081
+ "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==",
6082
+ "requires": {
6083
+ "just-performance": "4.3.0"
6084
+ }
6085
+ },
6086
"lines-and-columns": {
6087
"version": "1.2.4",
6088
"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -6935,6 +7012,22 @@
7012
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
7013
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
7014
},
7015
+ "stream-throttle": {
7016
+ "version": "0.1.3",
7017
+ "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz",
7018
+ "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=",
7019
+ "requires": {
7020
+ "commander": "^2.2.0",
7021
+ "limiter": "^1.0.5"
7022
+ },
7023
+ "dependencies": {
7024
+ "limiter": {
7025
+ "version": "1.1.5",
7026
+ "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz",
7027
+ "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA=="
7028
+ }
7029
+ }
7030
+ },
7031
"string-width": {
7032
"version": "4.2.3",
7033
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
package.json
+1
@@ -40,6 +40,7 @@
40
"koa-compress": "^5.1.0",
41
"koa-mount": "^4.0.0",
42
"koa-session": "^6.2.0",
43
+ "limiter": "^2.1.0",
44
"lodash": "^4.17.21",
45
"minimist": "^1.2.5",
46
"tssrp6a": "^3.0.0",
src/ThrottledStream.ts
new
+85
@@ -0,0 +1,85 @@
1
+import { Transform, TransformCallback } from 'stream'
2
+import { TokenBucket } from 'limiter'
3
+
4
+// throttled stream
5
+export class ThrottledStream extends Transform {
6
+
7
+ private sent: number = 0
8
+ private lastSpeed: number = 0
9
+ private lastSpeedTime = Date.now()
10
+
11
+ constructor(private group: ThrottleGroup) {
12
+ super()
13
+ }
14
+
15
+ async _transform(chunk: any, encoding: BufferEncoding, done: TransformCallback) {
16
+ let pos = 0
17
+ while (1) {
18
+ const slice = chunk.slice(pos, pos + this.group.getMin() / 10)
19
+ const n = slice.length
20
+ if (!n) // we're done here
21
+ return done()
22
+ try {
23
+ await this.group.consume(n)
24
+ this.push(slice)
25
+ this.sent += n
26
+ pos += n
27
+ } catch (e) {
28
+ done(e as Error)
29
+ return
30
+ }
31
+ }
32
+ }
33
+
34
+ // @return kBs
35
+ getSpeed(): number {
36
+ const now = Date.now()
37
+ const past = now - this.lastSpeedTime
38
+ if (past >= 1000) { // recalculate?
39
+ this.lastSpeedTime = now
40
+ this.lastSpeed = this.sent / past
41
+ this.sent = 0
42
+ }
43
+ return this.lastSpeed
44
+ }
45
+}
46
+
47
+export class ThrottleGroup {
48
+
49
+ private bucket?: TokenBucket
50
+
51
+ constructor(public kBs: number, parent?: ThrottleGroup) {
52
+ this.updateLimit(kBs)
53
+ if (parent)
54
+ this.bucket!.parentBucket = parent.bucket
55
+ }
56
+
57
+ // @return kBs
58
+ getLimit() {
59
+ return this.bucket!.bucketSize / 1000
60
+ }
61
+
62
+ updateLimit(kBs: number) {
63
+ if (kBs < 0)
64
+ throw new Error('invalid bytesPerSecond')
65
+ kBs *= 1000
66
+ this.bucket = new TokenBucket({
67
+ bucketSize: kBs,
68
+ tokensPerInterval: kBs,
69
+ interval: 'second',
70
+ parentBucket: this.bucket?.parentBucket,
71
+ })
72
+ }
73
+
74
+ getMin() {
75
+ let b: TokenBucket | undefined = this.bucket
76
+ let ret = b!.bucketSize
77
+ while (b = b!.parentBucket)
78
+ ret = Math.min(ret, b.bucketSize)
79
+ return ret
80
+ }
81
+
82
+ consume(n: number) {
83
+ return this.bucket!.removeTokens(n)
84
+ }
85
+}
src/config.ts
+9
-1
@@ -6,9 +6,11 @@ export const CFG_ALLOW_CLEAR_TEXT_LOGIN = 'allow clear text login'
6
7
const PATH = 'config.yaml'
8
9
+let started = false
10
let state:Record<string,any> = {}
11
const emitter = new EventEmitter()
12
watchLoad(argv.config || process.env.hfs_config || PATH, data => {
13
+ started = true
14
for (const k in data)
15
check(k)
16
for (const k in { ...state, ...configProps })
@@ -44,11 +46,17 @@ export function defineConfig(k:string, definition:ConfigProps) {
46
export function subscribeConfig({ k, ...definition }:{ k:string } & ConfigProps, cb:(v:any, was?:any)=>void) {
47
if (definition)
48
defineConfig(k, definition)
47
- const { caster } = configProps[k] ?? {}
49
+ const { caster, defaultValue } = configProps[k] ?? {}
50
const a = argv[k]
51
if (a !== undefined)
52
return cb(caster ? caster(a) : a)
53
emitter.on('new.'+k, cb)
54
+ if (!started) return
55
+ let v = state[k]
56
+ if (v === undefined)
57
+ v = defaultValue
58
+ if (v !== undefined)
59
+ cb(v)
60
}
61
62
export function getConfig(k:string) {
src/index.ts
+2
@@ -17,6 +17,7 @@ import { frontEndApis } from './frontEndApis'
17
import { log } from './log'
18
import './plugins'
19
import { pluginsMiddleware } from './plugins'
20
+import { throttler } from './throttler'
21
22
const BUILD_TIMESTAMP = ""
23
@@ -33,6 +34,7 @@ app.use(session({
34
}, app))
35
app.use(log())
36
app.use(pluginsMiddleware())
37
+app.use(throttler())
38
39
// serve apis
40
app.use(mount(API_URI, new Koa()
src/misc.ts
+5
@@ -55,3 +55,8 @@ export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterat
55
yield res as Exclude<OUT,undefined>
56
}
57
}
58
+
59
+export function getOrSet<T>(o:any, k:string, creator:()=>T): T {
60
+ return k in o ? o[k]
61
+ : (o[k] = creator())
62
+}
src/throttler.ts
new
+34
@@ -0,0 +1,34 @@
1
+import { Readable } from 'stream'
2
+import Koa from 'koa'
3
+import { ThrottledStream, ThrottleGroup } from './ThrottledStream'
4
+import { subscribeConfig } from './config'
5
+import { getOrSet } from './misc'
6
+
7
+const mainThrottleGroup = new ThrottleGroup(Infinity)
8
+
9
+subscribeConfig({ k:'max-kbps', defaultValue:Infinity }, v =>
10
+ mainThrottleGroup.updateLimit(v))
11
+
12
+interface GroupThrottler { count:number, throttler:ThrottledStream }
13
+const ip2group: Record<string,GroupThrottler> = {}
14
+
15
+export function throttler(): Koa.Middleware {
16
+ return async (ctx, next) => {
17
+ await next()
18
+ const { body } = ctx
19
+ if (!body || !(body instanceof Readable))
20
+ return
21
+ const ipGroup = getOrSet(ip2group, ctx.ip, ()=> {
22
+ const tg = new ThrottleGroup(Infinity, mainThrottleGroup)
23
+ subscribeConfig({ k:'max-kbps-per-ip', defaultValue:Infinity }, v =>
24
+ tg.updateLimit(v))
25
+ return { tg, count:0 }
26
+ })
27
+ const ts = new ThrottledStream(ipGroup.tg)
28
+ ts.on('close', ()=> {
29
+ if (!--ipGroup.count) // any left?
30
+ delete ip2group[ctx.ip]
31
+ })
32
+ ctx.body = ctx.body.pipe(ts)
33
+ }
34
+}
todo.md
+2
-1
@@ -1,10 +1,11 @@
1
# To do
2
+- fix: deleteModule is unloading even shared modules
3
+- fix: icons are not being cached
4
- fix: plugins folder is not found
5
- anti-csrf
6
- upload
7
- search and login dialogs should push to history so that mobile can use back button to close them
8
- node.comment
7
-- config: max speed (total/per-ip)
9
- config: max connections (total/per-ip)
10
- user.ignoreLimits
11
- user.redirect