admin/options: max_downloads*
Massimo Melina committed
Dec 19, 2023 at 12:06 UTC
9437f8d19bbc38b029233ae6c195cc78647e52a1
8 files changed
+109
-26
admin/src/OptionsPage.ts
+15
-3
@@ -6,8 +6,10 @@ import { apiCall, useApiEx } from './api'
6
import { state, useSnapState } from './state'
7
import { Link as RouterLink } from 'react-router-dom'
8
import { CardMembership, EditNote, Refresh, Warning } from '@mui/icons-material'
9
-import { Dict, iconTooltip, InLink, LinkBtn, MAX_TILE_SIZE, modifiedSx, REPO_URL, ipLocalHost,
10
- wait, wikiLink, with_, try_, ipForUrl, SORT_BY_OPTIONS, THEME_OPTIONS, useBreakpoint } from './misc'
9
+import {
10
+ Dict, iconTooltip, InLink, LinkBtn, MAX_TILE_SIZE, modifiedSx, REPO_URL, ipLocalHost,
11
+ wait, wikiLink, with_, try_, ipForUrl, SORT_BY_OPTIONS, THEME_OPTIONS, useBreakpoint, CFG
12
+} from './misc'
13
import { Form, BoolField, NumberField, SelectField, FieldProps, Field, StringField } from '@hfs/mui-grid-form';
14
import { ArrayField } from './ArrayField'
15
import FileField from './FileField'
@@ -63,6 +65,13 @@ export default function OptionsPage() {
65
placeholder: "no limit",
66
sm: 6,
67
}
68
+ const maxDownloadsDefaults = {
69
+ comp: NumberField,
70
+ min: 0,
71
+ placeholder: "no limit",
72
+ toField: (x: any) => x || '',
73
+ sm: 4,
74
+ }
75
const httpsEnabled = values.https_port >= 0
76
return h(Form, {
77
sx: { maxWidth: '60em' },
@@ -127,7 +136,10 @@ export default function OptionsPage() {
136
helperText: "Access Admin-panel without entering credentials"
137
},
138
{ k: 'max_kbps', ...maxSpeedDefaults, label: "Limit output", helperText: "Doesn't apply to localhost" },
130
- { k: 'max_kbps_per_ip', ...maxSpeedDefaults, label: "Limit output per-ip" },
139
+ { k: 'max_kbps_per_ip', ...maxSpeedDefaults, label: "Limit output per-IP" },
140
+ { k : CFG.max_downloads, ...maxDownloadsDefaults, helperText: "Number of simultaneous downloads" },
141
+ { k : CFG.max_downloads_per_ip, ...maxDownloadsDefaults, label: "Max downloads per-IP" },
142
+ { k : CFG.max_downloads_per_account, ...maxDownloadsDefaults, label: "Max downloads per-account", helperText: "Overrides other limits" },
143
{ k: 'dont_overwrite_uploading', comp: BoolField, sm: 12, md: 6, label: "Don't overwrite uploading",
144
helperText: "Files will be numbered to avoid overwriting" },
145
{ k: 'delete_unfinished_uploads_after', comp: NumberField, md: 3, min : 0, unit: "seconds", placeholder: "Never",
config.md
+3
@@ -90,6 +90,9 @@ Configuration can be done in several ways
90
- `server_code` javascript code that works similarly to [a plugin](dev-plugins.md).
91
- `tiles_size` starting value for frontend's tiles size. Default is 0.
92
- `update_to_beta` includes beta versions searching for updates. Default is false.
93
+- `max_downloads` limit the number of concurrent downloads on the whole server. Default is unlimited.
94
+- `max_downloads_per_ip` limit the number of concurrent downloads for the same IP address. Default is unlimited.
95
+- `max_downloads_per_account` limit the number of concurrent downloads for each account. This is enforced only for connections that are logged in, and will override other similar settings. Default is unlimited.
96
- `create-admin` special entry to quickly create an admin account. The value will be set as password. As soon as the account is created, this entry is removed.
97
98
#### Virtual File System (VFS)
src/cross-const.ts
+2
@@ -23,6 +23,7 @@ export const HTTP_PAYLOAD_TOO_LARGE = 413
23
export const HTTP_RANGE_NOT_SATISFIABLE = 416
24
export const HTTP_FOOL = 418
25
export const HTTP_FAILED_DEPENDENCY = 424
26
+export const HTTP_TOO_MANY_REQUESTS = 429
27
export const HTTP_SERVER_ERROR = 500
28
export const HTTP_SERVICE_UNAVAILABLE = 503
29
@@ -31,5 +32,6 @@ export const HTTP_MESSAGES: Record<number, string> = {
32
[HTTP_FORBIDDEN]: "Forbidden",
33
[HTTP_NOT_FOUND]: "Not found",
34
[HTTP_SERVER_ERROR]: "Server error",
35
+ [HTTP_TOO_MANY_REQUESTS]: "Too many requests",
36
}
37
src/cross.ts
+1
-1
@@ -23,7 +23,7 @@ export const FRONTEND_OPTIONS = {
23
export const SORT_BY_OPTIONS = ['name', 'extension', 'size', 'time']
24
export const THEME_OPTIONS = { auto: '', light: 'light', dark: 'dark' }
25
export const CFG = constMap(['geo_enable', 'geo_allow', 'geo_list', 'geo_allow_unknown',
26
- 'roots', 'roots_mandatory'])
26
+ 'max_downloads', 'max_downloads_per_ip', 'max_downloads_per_account', 'roots', 'roots_mandatory'])
27
export const LIST = { add: '+', remove: '-', update: '=', props: 'props', ready: 'ready', error: 'e' }
28
export type Dict<T=any> = Record<string, T>
29
export type Falsy = false | null | undefined | '' | 0
src/errorPages.ts
+2
-2
@@ -1,9 +1,9 @@
1
import Koa from 'koa'
2
import { getLangData } from './lang'
3
import { getSection } from './customHtml'
4
-import { HTTP_FORBIDDEN, HTTP_MESSAGES, HTTP_NOT_FOUND } from './cross'
4
+import { HTTP_FORBIDDEN, HTTP_MESSAGES, HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS } from './cross'
5
6
-const declaredErrorPages = [HTTP_NOT_FOUND, HTTP_FORBIDDEN].map(String)
6
+const declaredErrorPages = [HTTP_NOT_FOUND, HTTP_FORBIDDEN, HTTP_TOO_MANY_REQUESTS].map(String)
7
8
export function getErrorSections() {
9
return declaredErrorPages
src/serveFile.ts
+40
-5
@@ -3,19 +3,24 @@
3
import Koa from 'koa'
4
import { createReadStream, stat } from 'fs'
5
import { HTTP_BAD_REQUEST, HTTP_FORBIDDEN, HTTP_METHOD_NOT_ALLOWED, HTTP_NO_CONTENT, HTTP_NOT_FOUND, HTTP_NOT_MODIFIED,
6
- HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_RANGE_NOT_SATISFIABLE, MIME_AUTO } from './const'
6
+ HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_RANGE_NOT_SATISFIABLE, HTTP_TOO_MANY_REQUESTS, MIME_AUTO } from './const'
7
import { getNodeName, VfsNode } from './vfs'
8
import mimetypes from 'mime-types'
9
import { defineConfig } from './config'
10
-import { Dict, makeMatcher, matches } from './misc'
10
+import { CFG, Dict, makeMatcher, matches } from './misc'
11
import _ from 'lodash'
12
import { basename } from 'path'
13
import { promisify } from 'util'
14
import { updateConnection } from './connections'
15
+import { getCurrentUsername } from './auth'
16
+import { sendErrorPage } from './errorPages'
17
18
const allowedReferer = defineConfig('allowed_referer', '')
19
+const limitDownloads = downloadLimiter(defineConfig(CFG.max_downloads, 0), () => true)
20
+const limitDownloadsPerIp = downloadLimiter(defineConfig(CFG.max_downloads_per_ip, 0), ctx => ctx.ip)
21
+const limitDownloadsPerAccount = downloadLimiter(defineConfig(CFG.max_downloads_per_account, 0), ctx => getCurrentUsername(ctx) || undefined)
22
18
-export function serveFileNode(ctx: Koa.Context, node: VfsNode) {
23
+export async function serveFileNode(ctx: Koa.Context, node: VfsNode) {
24
const { source, mime } = node
25
const name = getNodeName(node)
26
const mimeString = typeof mime === 'string' ? mime
@@ -30,7 +35,10 @@ export function serveFileNode(ctx: Koa.Context, node: VfsNode) {
35
ctx.vfsNode = node // useful to tell service files from files shared by the user
36
if ('dl' in ctx.query) // please, download
37
ctx.attachment(name)
33
- return serveFile(ctx, source||'', mimeString)
38
+ await serveFile(ctx, source||'', mimeString)
39
+
40
+ if (await limitDownloadsPerAccount(ctx) === undefined) // returning false will not execute other limits
41
+ await limitDownloads(ctx) || await limitDownloadsPerIp(ctx)
42
43
function host() {
44
const s = ctx.get('host')
@@ -124,4 +132,31 @@ declare module "koa" {
132
interface DefaultState {
133
includesLastByte?: boolean
134
}
127
-}
\ No newline at end of file
135
+}
136
+function downloadLimiter<T>(configMax: { get: () => number | undefined }, cbKey: (ctx: Koa.Context) => T | undefined) {
137
+ const map = new Map<T, number>()
138
+ return (ctx: Koa.Context) => {
139
+ if (!ctx.body) return // no file sent, cache hit
140
+ const k = cbKey(ctx)
141
+ if (k === undefined) return // undefined = skip limit
142
+ const max = configMax.get()
143
+ const now = map.get(k) || 0
144
+ if (max && now >= max)
145
+ return tooMany()
146
+ map.set(k, now + 1)
147
+ ctx.req.on('close', () => {
148
+ const n = map.get(k)!
149
+ if (n > 1)
150
+ map.set(k, n - 1)
151
+ else
152
+ map.delete(k)
153
+ })
154
+ return false // limit is enforced but passed
155
+
156
+ async function tooMany() {
157
+ ctx.set('retry-after', '3600')
158
+ await sendErrorPage(ctx, HTTP_TOO_MANY_REQUESTS)
159
+ return true
160
+ }
161
+ }
162
+}
tests/config.yaml
+3
-1
@@ -128,4 +128,6 @@ accounts:
128
disabled: true
129
admins:
130
admin: true
131
-version: 0.50.4
131
+version: 0.51.0-alpha3
132
+max_downloads_per_account: 2
133
+max_downloads: 1
tests/test.ts
+43
-14
@@ -2,9 +2,11 @@ import { srpClientSequence } from '../src/srp'
2
import { createReadStream } from 'fs'
3
import { dirname, join } from 'path'
4
import _ from 'lodash'
5
-import { findDefined, tryJson } from '../src/cross'
5
+import { findDefined, randomId, tryJson, wait } from '../src/cross'
6
import { httpStream, stream2string, XRequestOptions } from '../src/util-http'
7
-import { rm } from 'fs/promises'
7
+import { ThrottledStream, ThrottleGroup } from '../src/ThrottledStream'
8
+import { rm, writeFile } from 'fs/promises'
9
+import { Readable } from 'stream'
10
/*
11
import { PORT, srv } from '../src'
12
@@ -17,8 +19,10 @@ const username = 'rejetto'
19
const password = 'password'
20
const API = '/~/api/'
21
const BASE_URL = 'http://localhost:81'
20
-const UPLOAD_ROOT = '/for-admins/upload'
21
-const UPLOAD_DEST = UPLOAD_ROOT + '/temp/gpl.png'
22
+const UPLOAD_ROOT = '/for-admins/upload/'
23
+const UPLOAD_DEST = UPLOAD_ROOT + 'temp/gpl.png'
24
+const BIG_CONTENT = _.repeat(randomId(10), 200_000) // 2MB, big enough to saturate buffers
25
+const throttle = BIG_CONTENT.length /1000 /0.5 // KB, finish in 0.5s, quick but still overlapping downloads
26
27
describe('basics', () => {
28
//before(async () => appStarted)
@@ -95,7 +99,7 @@ describe('basics', () => {
99
headers: { Referer: 'https://some-website.com/try-to-trick/x.com/' }
100
}))
101
98
- testUpload('upload.need account', UPLOAD_DEST, 401)
102
+ it('upload.need account', reqUpload( UPLOAD_DEST, 401))
103
it('create_folder', reqApi('create_folder', { uri: UPLOAD_ROOT, name: 'temp' }, 401))
104
it('delete.no perm', reqApi('delete', { uri: '/for-admins' }, 403))
105
it('delete.need account', reqApi('delete', { uri: '/for-admins/upload' }, 401))
@@ -110,19 +114,31 @@ describe('accounts', () => {
114
it('accounts.remove', reqApi('del_account', { username }, 200))
115
})
116
117
+describe('limits', () => {
118
+ const fn = 'tests/big'
119
+ before(() => writeFile(fn, BIG_CONTENT))
120
+ it('max_dl', () => testMaxDl('/' + fn, 1, 2))
121
+ after(() => rm(fn))
122
+})
123
+
124
describe('after-login', () => {
125
before(() => login(username))
126
it('create_folder', reqApi('create_folder', { uri: UPLOAD_ROOT, name: 'temp' }, 200))
127
it('inherit.perm', reqList('/for-admins/', { inList:['alfa.txt'] }))
128
it('inherit.disabled', reqList('/for-disabled/', 401))
118
- testUpload('upload.never', '/random', 403)
119
- testUpload('upload.ok', UPLOAD_DEST, 200)
120
- testUpload('upload.crossing', UPLOAD_DEST.replace('temp', '../..'), 418)
129
+ it('upload.never', reqUpload('/random', 403))
130
+ it('upload.ok', reqUpload(UPLOAD_DEST, 200))
131
+ it('upload.crossing', reqUpload(UPLOAD_DEST.replace('temp', '../..'), 418))
132
const renameTo = 'z'
133
it('rename.ok', reqApi('rename', { uri: UPLOAD_DEST, dest: renameTo }, 200))
134
it('delete.miss renamed', reqApi('delete', { uri: UPLOAD_DEST }, 404))
135
it('delete.ok', reqApi('delete', { uri: dirname(UPLOAD_DEST) + '/' + renameTo }, 200))
136
it('delete.miss deleted', reqApi('delete', { uri: UPLOAD_DEST }, 404))
137
+ it('max_dl.account', async () => {
138
+ const uri = UPLOAD_ROOT + 'temp/big'
139
+ await reqUpload(uri, 200, BIG_CONTENT)()
140
+ await testMaxDl(uri, 2, 1)
141
+ })
142
after(() =>
143
rm(join(__dirname, 'temp'), { recursive: true}).catch(() => 0))
144
})
@@ -132,11 +148,22 @@ function login(usr: string, pwd=password) {
148
reqApi(cmd, params, (x,res)=> res.statusCode < 400)())
149
}
150
135
-function testUpload(name: string, dest: string, tester: Tester) {
136
- it(name, req(dest, tester, {
151
+function reqUpload(dest: string, tester: Tester, body?: Readable | string) {
152
+ return req(dest, tester, {
153
method: 'PUT',
138
- body: createReadStream(join(__dirname, 'page/gpl.png'))
139
- }))
154
+ body: body ?? createReadStream(join(__dirname, 'page/gpl.png'))
155
+ })
156
+}
157
+
158
+async function testMaxDl(uri: string, good: number, bad: number) {
159
+ let i = 0
160
+ const reqs = []
161
+ while (good--)
162
+ reqs.push( req(uri + '?' + (++i), 200, { throttle })() )
163
+ await wait(10) // ensure it the slots are taken
164
+ while (bad--)
165
+ reqs.push( req(uri + '?' + (++i), 429, { throttle })() )
166
+ await Promise.all(reqs)
167
}
168
169
type TesterFunction = ((data: any, fullResponse: any) => boolean)
@@ -157,7 +184,7 @@ type Tester = number
184
185
const jar = {}
186
160
-function req(url: string, test:Tester, requestOptions: XRequestOptions={}) {
187
+function req(url: string, test:Tester, requestOptions: XRequestOptions & { throttle?: number }={}) {
188
// passing 'path' keeps it as it is, avoiding internal resolving
189
return () => httpStream(BASE_URL + url, { path: url, jar, ...requestOptions }).catch(e => {
190
if (e.code === "ECONNREFUSED")
@@ -171,7 +198,9 @@ function req(url: string, test:Tester, requestOptions: XRequestOptions={}) {
198
test = { re:test }
199
if (typeof test === 'number')
200
test = { status: test }
174
- const data = await stream2string(res)
201
+ const { throttle } = requestOptions
202
+ const stream = throttle ? res.pipe(new ThrottledStream(new ThrottleGroup(throttle))) : res
203
+ const data = await stream2string(stream)
204
const obj = tryJson(data)
205
if (typeof test === 'object') {
206
const { status, mime, re, inList, outList, length, permInList } = test