upload: faster with many small files #485
Massimo Melina committed
Mar 26, 2024 at 21:30 UTC
5eb133cdacf3d4bc88abb9a4e98b16bbb7abe985
3 files changed
+29
-8
frontend/src/upload.ts
+12
-5
@@ -1,9 +1,11 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
-import { createElement as h, DragEvent, Fragment, useMemo, CSSProperties } from 'react'
3
+import { createElement as h, DragEvent, Fragment, useMemo, CSSProperties, useState, useEffect } from 'react'
4
import { Flex, FlexV, iconBtn, Select } from './components'
5
-import { basename, closeDialog, formatBytes, formatPerc, hIcon, useIsMobile, newDialog, prefix, selectFiles, working,
6
- HTTP_CONFLICT, HTTP_PAYLOAD_TOO_LARGE, formatSpeed, dirname, getHFS, onlyTruthy, with_ } from './misc'
5
+import {
6
+ basename, closeDialog, formatBytes, formatPerc, hIcon, useIsMobile, newDialog, prefix, selectFiles, working,
7
+ HTTP_CONFLICT, HTTP_PAYLOAD_TOO_LARGE, formatSpeed, dirname, getHFS, onlyTruthy, with_, cpuSpeedIndex
8
+} from './misc'
9
import _ from 'lodash'
10
import { proxy, ref, subscribe, useSnapshot } from 'valtio'
11
import { alertDialog, confirmDialog, promptDialog, toast } from './dialog'
@@ -220,9 +222,13 @@ function path(f: File) {
222
function FilesList({ entries, actions }: { entries: ToUpload[], actions: { [icon:string]: null | ((rec :ToUpload) => any) } }) {
223
const { uploading, progress } = useSnapshot(uploadState)
224
const snapEntries = useSnapshot(entries)
225
+ const [all, setAll] = useState(false)
226
+ useEffect(() => setAll(false), [entries.length])
227
+ const MAX = all ? Infinity : _.round(_.clamp(100 * cpuSpeedIndex, 10, 100))
228
+ const rest = Math.max(0, snapEntries.length - MAX)
229
return !snapEntries.length ? null : h('table', { className: 'upload-list', width: '100%' },
230
h('tbody', {},
225
- snapEntries.map((e, i) => {
231
+ snapEntries.slice(0, MAX).map((e, i) => {
232
const working = e === uploading
233
return h(Fragment, { key: i },
234
h('tr', {},
@@ -237,7 +243,8 @@ function FilesList({ entries, actions }: { entries: ToUpload[], actions: { [icon
243
),
244
e.comment && h('tr', {}, h('td', { colSpan: 3 }, h('div', { className: 'entry-comment' }, e.comment)) )
245
)
240
- })
246
+ }),
247
+ rest > 0 && h('tr', {}, h('td', { colSpan: 99 }, h('a', { href: '#', onClick: () => setAll(true) }, t('more_items', { rest }, "{rest} more items"))))
248
)
249
)
250
}
shared/index.ts
+11
-2
@@ -8,16 +8,25 @@ export * from './dialogs'
8
export * from './md'
9
export * from '../src/srp'
10
export * from '../src/cross'
11
-
11
// code in this file is shared among frontends, but not backend
12
14
-(window as any)._ = _
13
+;(window as any)._ = _
14
+
15
+// roughly 0.7 on m1 max
16
+export const cpuSpeedIndex = (() => {
17
+ let ms = performance.now()
18
+ _.range(1E5).map(x => ++x)
19
+ ms = performance.now() - ms
20
+ return 1 / ms
21
+})()
22
+
23
24
const HFS = getHFS()
25
Object.assign(HFS, {
26
getPluginKey: () => getScriptAttr('plugin'),
27
getPluginPublic: () => getScriptAttr('src')?.match(/^.*\//)?.[0],
28
getPluginConfig: () => HFS.plugins[HFS.getPluginKey()] || {},
29
+ cpuSpeedIndex,
30
})
31
32
function getScriptAttr(k: string) {
src/upload.ts
+6
-1
@@ -37,6 +37,7 @@ function setUploadMeta(path: string, ctx: Koa.Context) {
37
}
38
39
// stay sync because we use this function with formidable()
40
+const cache: any = {}
41
export function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
42
if (dirTraversal(path))
43
return fail(HTTP_FOOL)
@@ -48,7 +49,11 @@ export function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
49
const reqSize = Number(ctx.headers["content-length"])
50
if (reqSize)
51
try {
51
- const { free } = getDiskSpaceSync(dir)
52
+ if (!Object.hasOwn(cache, dir)) {
53
+ cache[dir] = getDiskSpaceSync(dir)
54
+ setTimeout(() => delete cache[dir], 3_000) // invalidate shortly
55
+ }
56
+ const { free } = cache[dir]
57
if (typeof free !== 'number' || isNaN(free))
58
throw ''
59
if (reqSize > free - (min || 0))