upload: automatic resume
Massimo Melina committed
Jan 26, 2025 at 11:12 UTC
a7742fe3b796de513a553bf0045694fc244df64b
19 files changed
+186
-96
admin/src/OptionsPage.ts
+1
-2
@@ -213,8 +213,7 @@ export default function OptionsPage() {
213
{ k : CFG.split_uploads, comp: NumberField, unit: 'MB', md: 2, step: .1,
214
fromField: x => x * 1E6, toField: x => x ? x / 1E6 : null,
215
placeholder: "disabled", label: "Split uploads in chunks", helperText: "Overcome proxy limits" },
216
- { k: 'delete_unfinished_uploads_after', comp: NumberField, md: 3, min : 0, unit: "seconds", placeholder: "Never",
217
- helperText: "Leave empty to never delete" },
216
+ { k: 'delete_unfinished_uploads_after', comp: NumberField, md: 3, min : 0, unit: "seconds", required: true },
217
{ k: 'min_available_mb', comp: NumberField, md: 3, min : 0, unit: "MBytes", placeholder: "None",
218
label: "Min. available disk space", helperText: "Reject uploads that don't comply" },
219
frontend/package.json
+3
-1
@@ -16,12 +16,14 @@
16
"tssrp6a": "^3.0.0",
17
"usehooks-ts": "^2.6.0",
18
"valtio": "^1.13.0",
19
- "web-vitals": "^2.1.4"
19
+ "web-vitals": "^2.1.4",
20
+ "xxhashjs": "^0.2.2"
21
},
22
"devDependencies": {
23
"@types/lodash": "^4.14.178",
24
"@types/react": "^18.3.14",
25
"@types/react-dom": "^18.3.2",
26
+ "@types/xxhashjs": "^0.2.4",
27
"@vitejs/plugin-legacy": "^6.0.0",
28
"cross-env": "^7.0.3",
29
"sass": "^1.54.5",
frontend/src/FilterBar.ts
+2
-2
@@ -3,7 +3,7 @@ import { createElement as h, useEffect, useState } from 'react'
3
import { useDebounce } from 'usehooks-ts'
4
import { Checkbox, CustomCode } from './components'
5
import { usePath } from './useFetchList'
6
-import { getHFS, with_ } from './misc'
6
+import { onHfsEvent, with_ } from './misc'
7
import i18n from './i18n'
8
const { useI18N } = i18n
9
@@ -15,7 +15,7 @@ export function FilterBar() {
15
const {t} = useI18N()
16
17
state.patternFilter = useDebounce(showFilter ? filter : '', 300)
18
- useEffect(() => getHFS().onEvent('entryToggleSelection', () => setAll(false)), [])
18
+ useEffect(() => onHfsEvent('entryToggleSelection', () => setAll(false)), [])
19
20
const tabIndex = showFilter ? undefined : -1
21
return h('div', { id: 'filter-bar', style: { display: showFilter ? undefined : 'none' } },
frontend/src/index.scss
+1
@@ -481,6 +481,7 @@ button .icon + .label {
481
z-index: 3; // necessary to not be covered by icon-buttons
482
box-shadow: 0 0 .3em .3em var(--bg);
483
}
484
+.upload-hashing::before,
485
.upload-progress::before { content: var(--separator); }
486
.entry-size::after { content: var(--separator) }
487
frontend/src/misc.ts
+22
-20
@@ -64,6 +64,27 @@ export function hfsEvent(name: string, params?:Dict) {
64
})
65
}
66
67
+export function onHfsEvent(name: string, cb: (params:any, extra: { output: any[], setOrder: Callback<number>, preventDefault: Callback }, output: any[]) => any) {
68
+ const key = 'hfs.' + name
69
+ document.addEventListener(key, wrapper)
70
+ return () => document.removeEventListener(key, wrapper)
71
+
72
+ function wrapper(ev: Event) {
73
+ const { params, output, order } = (ev as CustomEvent).detail
74
+ let thisOrder
75
+ const res = cb(params, {
76
+ output,
77
+ setOrder(x) { thisOrder = x },
78
+ preventDefault: () => ev.preventDefault()
79
+ }, output) // legacy pre-0.54, third parameter used by file-icons plugin
80
+ if (res !== undefined && Array.isArray(output)) {
81
+ output.push(res)
82
+ if (thisOrder)
83
+ order[output.length - 1] = thisOrder
84
+ }
85
+ }
86
+}
87
+
88
export function formatTimestamp(x: number | string | Date, options?: Intl.DateTimeFormatOptions) {
89
const cached = getOrSet(formatTimestamp as any, 'langs', () => {
90
const ret = getLangs()
@@ -79,6 +100,7 @@ Object.assign(getHFS(), {
100
fileShowComponents: { Video, Audio },
101
misc: { ...cross, ...shared },
102
emit: hfsEvent,
103
+ onEvent: onHfsEvent,
104
watchState(k: string, cb: (v: any) => void) {
105
const up = k.split('upload.')[1]
106
return subscribeKey(up ? uploadState : state as any, up || k, cb, true)
@@ -87,26 +109,6 @@ Object.assign(getHFS(), {
109
return apiCall(cross.PLUGIN_CUSTOM_REST_PREFIX + name, ...rest)
110
},
111
html: (html: string) => h(Html, {}, html),
90
- onEvent(name: string, cb: (params:any, extra: { output: any[], setOrder: Callback<number>, preventDefault: Callback }, output: any[]) => any) {
91
- const key = 'hfs.' + name
92
- document.addEventListener(key, wrapper)
93
- return () => document.removeEventListener(key, wrapper)
94
-
95
- function wrapper(ev: Event) {
96
- const { params, output, order } = (ev as CustomEvent).detail
97
- let thisOrder
98
- const res = cb(params, {
99
- output,
100
- setOrder(x) { thisOrder = x },
101
- preventDefault: () => ev.preventDefault()
102
- }, output) // legacy pre-0.54, third parameter used by file-icons plugin
103
- if (res !== undefined && Array.isArray(output)) {
104
- output.push(res)
105
- if (thisOrder)
106
- order[output.length - 1] = thisOrder
107
- }
108
- }
109
- }
112
})
113
114
export function operationSuccessful() {
frontend/src/upload.ts
+2
-1
@@ -160,7 +160,7 @@ export function showUpload() {
160
}
161
162
function FileList({ entries, actions }: { entries: ToUpload[], actions: { [icon:string]: null | ((rec :ToUpload) => any) } }) {
163
- const { uploading, progress, partial } = useSnapshot(uploadState)
163
+ const { uploading, progress, partial, hashing } = useSnapshot(uploadState)
164
const snapEntries = useSnapshot(entries)
165
const [all, setAll] = useState(false)
166
useEffect(() => setAll(false), [entries.length])
@@ -179,6 +179,7 @@ function FileList({ entries, actions }: { entries: ToUpload[], actions: { [icon:
179
h('td', {},
180
h('span', { className: working ? 'ani-working' : undefined }, e.name || getFilePath(entries[i].file)),
181
working && h('span', { className: 'upload-progress', title }, formatBytes(partial)),
182
+ working && hashing && h('span', { className: 'upload-hashing' }, t`Considering resume`, ' (', formatPerc(hashing), ')'),
183
working && h('progress', { className: 'upload-progress-bar', title, value: progress, max: 1 }),
184
),
185
),
frontend/src/uploadQueue.ts
+75
-24
@@ -1,18 +1,17 @@
1
import {
2
- buildUrlQueryString, dirname, formatBytes, formatPerc, getHFS,
3
- HTTP_CONFLICT, HTTP_MESSAGES, HTTP_PAYLOAD_TOO_LARGE, UPLOAD_RESUMABLE, UPLOAD_STATUS,
4
- pathEncode, pendingPromise, prefix, randomId, tryJson, with_, wait
2
+ HTTP_CONFLICT, HTTP_MESSAGES, HTTP_PAYLOAD_TOO_LARGE, UPLOAD_RESUMABLE, UPLOAD_REQUEST_STATUS, UPLOAD_RESUMABLE_HASH,
3
+ buildUrlQueryString, dirname, getHFS, pathEncode, pendingPromise, prefix, randomId, tryJson, with_, wait
4
} from '@hfs/shared'
5
import { state } from './state'
6
import { getNotifications } from '@hfs/shared/api'
8
-import { subscribeKey } from 'valtio/utils'
9
-import { alertDialog, confirmDialog, toast } from './dialog'
7
+import { alertDialog, toast } from './dialog'
8
import { reloadList } from './useFetchList'
9
import { proxy, ref, snapshot, subscribe } from 'valtio'
10
import { createElement as h } from 'react'
11
import _ from 'lodash'
12
import { UploadStatus } from './upload'
13
import i18n from './i18n'
14
+import { hfsEvent, onHfsEvent } from './misc'
15
const { t } = i18n
16
17
export interface ToUpload { file: File, comment?: string, name?: string, to?: string, error?: string }
@@ -25,6 +24,7 @@ export const uploadState = proxy<{
24
qs: { to: string, entries: ToUpload[] }[]
25
paused: boolean
26
uploading?: ToUpload
27
+ hashing?: number
28
progress: number // percentage
29
partial: number // relative to uploading file. This is how much we have done of the current queue.
30
speed: number
@@ -78,7 +78,7 @@ let req: XMLHttpRequest | undefined
78
let overrideStatus = 0
79
let notificationChannel = ''
80
let notificationSource: EventSource | undefined
81
-let closeLastDialog: undefined | ((() => void) & { path?: string })
81
+let closeLastDialog: undefined | (() => void)
82
83
let reloadOnClose = false
84
export function resetReloadOnClose() {
@@ -88,6 +88,7 @@ export function resetReloadOnClose() {
88
}
89
90
export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
91
+ console.debug('start upload', getFilePath(toUpload.file))
92
let resuming = false
93
let preserveTempFile = undefined
94
overrideStatus = 0
@@ -153,6 +154,7 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
154
const partial = splitSize && offset + splitSize < fullSize
155
req.open('PUT', to + pathEncode(uploadPath) + buildUrlQueryString({
156
notificationChannel,
157
+ giveBack: toUpload.file.lastModified,
158
...partial && { partial: 'y' },
159
...offset && { resume: offset, preserveTempFile },
160
...toUpload.comment && { comment: toUpload.comment },
@@ -167,42 +169,45 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
169
async function subscribeNotifications() {
170
if (notificationChannel) return
171
notificationChannel = 'upload-' + randomId()
172
+ const PREFIX = 'resume hash/'
173
notificationSource = await getNotifications(notificationChannel, async (name, data) => {
174
const {uploading} = uploadState
175
if (!uploading) return
176
+ if (name === UPLOAD_RESUMABLE_HASH)
177
+ return hfsEvent(PREFIX + data.path, data.hash)
178
if (name === UPLOAD_RESUMABLE) {
179
waitSecondChunk.resolve()
180
const path = getFilePath(uploading.file)
176
- const size = data?.[path] //TODO use toUpload?
181
+ if (path !== data.path) return // is it about current file?
182
+ const {size} = data //TODO use toUpload?
183
if (!size) {
178
- if (!closeLastDialog?.path || closeLastDialog?.path === path) {// previous resumable is gone
179
- closeLastDialog?.()
180
- preserveTempFile = undefined
181
- }
184
+ preserveTempFile = undefined
185
return
186
}
187
preserveTempFile = true // this is affecting only split-uploads, because is undefined on first chunk (or no chunking)
188
if (size > toUpload.file.size) return
189
+ if (data.giveBack) {
190
+ // lastModified doesn't necessarily mean the file has changed, but it seems ok for the time being
191
+ if (data.giveBack !== String(toUpload.file.lastModified)) // query params are always string
192
+ return console.debug('upload timestamp changed')
193
+ console.debug('upload unchanged')
194
+ }
195
+ else { // timestamp may miss if the file is left by old version, or HFS was killed
196
+ const hashFromServer = new Promise<any>(res => onHfsEvent(PREFIX + path, res))
197
+ const hashed = await calcHash(uploading.file, size) // therefore, we attempt a check using the hash
198
+ if (!hashed) return // too late, we are working on another file
199
+ if (hashed !== await hashFromServer) return console.debug('upload hash mismatch')
200
+ console.debug('upload hash is matching')
201
+ }
202
closeLastDialog?.()
187
- const cancelSub = subscribeKey(uploadState, 'partial', v =>
188
- v >= size && closeLastDialog?.() ) // dismiss dialog as soon as we pass the threshold
189
- const msg = t('confirm_resume', "Resume upload?") + ` (${formatPerc(size/toUpload.file.size)} = ${formatBytes(size)})`
190
- const {expires} = data
191
- const timeout = typeof expires !== 'number' ? 0
192
- : (Number(new Date(expires)) - Date.now()) / 1000
193
- const dialog = confirmDialog(msg, { timeout })
194
- closeLastDialog = Object.assign(dialog.close, { path })
195
- const confirmed = await dialog
196
- cancelSub()
197
- if (!confirmed) return
198
- if (uploading !== uploadState.uploading) return // too late
203
resuming = true
204
+ console.debug('resuming upload', size.toLocaleString())
205
preserveTempFile = undefined
206
abortCurrentUpload()
207
await wait(500) // be sure the server had the time to react to the abort() and unlocked the file, or our next request will fail
208
return startUpload(toUpload, to, size)
209
}
205
- if (name === UPLOAD_STATUS) {
210
+ if (name === UPLOAD_REQUEST_STATUS) {
211
overrideStatus = data?.[getFilePath(uploading.file)]
212
if (overrideStatus >= 400)
213
abortCurrentUpload()
@@ -302,3 +307,49 @@ export function resetCounters() {
307
skipped: [],
308
})
309
}
310
+
311
+async function calcHash(file: File, limit=Infinity) {
312
+ const hash = await hasher()
313
+ const t = Date.now()
314
+ const reader = file.stream().getReader()
315
+ let left = limit
316
+ const updateUI = _.debounce(() => uploadState.hashing = (limit - left) / limit, 100, { maxWait: 500 })
317
+ try {
318
+ while (left > 0) {
319
+ if (uploadState.uploading?.file !== file) return // upload aborted
320
+ const res = await reader.read()
321
+ if (res.done) break
322
+ const chunk = res.value.slice(0, left)
323
+ hash.update(chunk)
324
+ left -= chunk.length
325
+ updateUI()
326
+ await wait(1) // cooperative: without this, the browser may freeze
327
+ }
328
+ }
329
+ finally {
330
+ updateUI.flush()
331
+ uploadState.hashing = undefined
332
+ }
333
+ const ret = hash.digest().toString(16)
334
+ console.debug('hash calculated in', Date.now() - t, 'ms', ret)
335
+ return ret
336
+
337
+ async function hasher() {
338
+ /* using this lib because it's much faster, works on legacy browsers, and we don't need it to be cryptographic. Sure 32bit isn't much.
339
+ Benchmark on 2GB:
340
+ 18.5s aws-crypto/sha256-browser
341
+ 43.6s js-sha512
342
+ 73.3s sha512@hash-wasm
343
+ 8.2s xxhash-wasm/64
344
+ 8.2s xxhash-wasm/32
345
+ 41s xxhashjs/64
346
+ 9.1s xxhashjs/32
347
+ */
348
+ //if (BigInt !== Number && BigInt) return (await (await import('xxhash-wasm')).default()).create64() // at 32bit, a 9% difference is not worth having 2 libs, but 64bit is terrible without wasm
349
+ const ret = (await import('xxhashjs')).h32()
350
+ const original = ret.update
351
+ ret.update = (x: Uint8Array) => original.call(ret, x.buffer) // xxhashjs only works with ArrayBuffer, not UInt8Array
352
+ return ret
353
+ }
354
+}
355
+
package.json
+1
@@ -97,6 +97,7 @@
97
"tssrp6a": "^3.0.0",
98
"unzip-stream": "^0.3.4",
99
"valtio": "^1.10.3",
100
+ "xxhash-wasm": "^1.1.0",
101
"yaml": "^2.0.0-10"
102
},
103
"devDependencies": {
src/comments.ts
+2
-2
@@ -1,7 +1,7 @@
1
import { defineConfig } from './config'
2
import { dirname, join } from 'path'
3
import { basename } from './cross'
4
-import { parseFile, parseFileCache } from './util-files'
4
+import { parseFileContent, parseFileCache } from './util-files'
5
import { createWriteStream } from 'fs'
6
import { singleWorkerFromBatchWorker } from './misc'
7
import _ from 'lodash'
@@ -52,7 +52,7 @@ export function areCommentsEnabled() {
52
const MULTILINE_SUFFIX = Buffer.from([4, 0xC2])
53
function readDescription(path: string) {
54
// decoding could also be done with native TextDecoder.decode, but we need iconv for the encoding anyway
55
- return parseFile(join(path, DESCRIPT_ION), raw => {
55
+ return parseFileContent(join(path, DESCRIPT_ION), raw => {
56
// for simplicity we "remove" the sequence MULTILINE_SUFFIX before iconv.decode messes it up
57
for (let i=0; i<raw.length; i++)
58
if (raw[i] === MULTILINE_SUFFIX[0] && raw[i+1] === MULTILINE_SUFFIX[1] && [undefined,13,10].includes(raw[i+2]))
src/cross-const.ts
+2
-1
@@ -9,7 +9,8 @@ export const NBSP = '\xA0'
9
export const PLUGIN_CUSTOM_REST_PREFIX = '_'
10
export const HFS_REPO = 'rejetto/hfs'
11
export const UPLOAD_RESUMABLE = 'upload.resumable'
12
-export const UPLOAD_STATUS = 'upload.status'
12
+export const UPLOAD_RESUMABLE_HASH = 'upload.hash'
13
+export const UPLOAD_REQUEST_STATUS = 'upload.status'
14
export const PREVIOUS_TAG = 'previous'
15
16
export const HTTP_OK = 200
src/langs/hfs-lang-en.json
+2
-2
@@ -62,7 +62,6 @@
62
"send_files": "Send {n,plural,one{# file} other{# files}}, {size}",
63
"Clear": "Clear",
64
"failed_upload": "Couldn't upload {name}",
65
- "confirm_resume": "Resume upload?",
65
"file too large": "file too large",
66
"Enter folder name": "Enter folder name",
67
"Successfully created": "Successfully created",
@@ -183,6 +182,7 @@
182
"Creation": "Creation",
183
"creation": "creation",
184
"required_change_password": "You are required to change your password",
186
- "Wildcards": "Wildcards"
185
+ "Wildcards": "Wildcards",
186
+ "Considering resume": "Considering resume"
187
}
188
}
src/langs/hfs-lang-fi.json
+2
-1
@@ -187,6 +187,7 @@
187
"Creation": "Luonti",
188
"creation": "luonti",
189
"required_change_password": "Sinun täytyy vaihtaa salasanasi",
190
- "Wildcards": "Villimerkit"
190
+ "Wildcards": "Villimerkit",
191
+ "Considering resume": "Yritetään jatkaa"
192
}
193
}
src/langs/hfs-lang-it.json
+2
-2
@@ -60,7 +60,6 @@
60
"send_files": "Invia {n} file, {size}",
61
"Clear": "Azzera",
62
"failed_upload": "Upload fallito per {name}",
63
- "confirm_resume": "Vuoi riprendere questo upload?",
63
"file too large": "file troppo grande",
64
"Enter folder name": "Inserisci nome cartella",
65
"Successfully created": "Creazione riuscita",
@@ -175,6 +174,7 @@
174
"Creation": "Creazione",
175
"creation": "creazione",
176
"required_change_password": "È necessario cambiare la password",
178
- "Wildcards": "Caratteri jolly"
177
+ "Wildcards": "Caratteri jolly",
178
+ "Considering resume": "Ripristino upload"
179
}
180
}
src/misc.ts
+2
-2
@@ -132,7 +132,7 @@ export function apiAssertTypes(paramsByType: { [type:string]: { [name:string]: a
132
export function createStreamLimiter(limit: number) {
133
let got = 0
134
return new Transform({
135
- transform(chunk, enc, cb) {
135
+ transform(chunk, enc, done) {
136
const left = limit - got
137
got += chunk.length
138
if (left > 0) {
@@ -140,7 +140,7 @@ export function createStreamLimiter(limit: number) {
140
if (got >= limit)
141
this.end()
142
}
143
- cb()
143
+ done()
144
}
145
})
146
}
src/plugins.ts
+1
-1
@@ -4,7 +4,7 @@ import glob from 'fast-glob'
4
import { watchLoad } from './watchLoad'
5
import _ from 'lodash'
6
import {
7
- API_VERSION, APP_PATH, COMPATIBLE_API_VERSION, HTTP_NOT_FOUND, ICONS_URI, IS_WINDOWS, MIME_AUTO, PLUGINS_PUB_URI
7
+ API_VERSION, APP_PATH, COMPATIBLE_API_VERSION, HTTP_NOT_FOUND, IS_WINDOWS, MIME_AUTO, PLUGINS_PUB_URI
8
} from './const'
9
import * as Const from './const'
10
import Koa from 'koa'
src/serveGuiFiles.ts
+2
-2
@@ -12,7 +12,7 @@ import { refresh_session } from './api.auth'
12
import { ApiError } from './apiMiddleware'
13
import { join, extname } from 'path'
14
import {
15
- CFG, debounceAsync, formatBytes, FRONTEND_OPTIONS, isPrimitive, newObj, objSameKeys, onlyTruthy, parseFile
15
+ CFG, debounceAsync, formatBytes, FRONTEND_OPTIONS, isPrimitive, newObj, objSameKeys, onlyTruthy, parseFileContent
16
} from './misc'
17
import { favicon, title } from './adminApis'
18
import { customHtml, getAllSections, getSection } from './customHtml'
@@ -46,7 +46,7 @@ function serveStatic(uri: string): Koa.Middleware {
46
return ctx.status = HTTP_METHOD_NOT_ALLOWED
47
const serveApp = shouldServeApp(ctx)
48
const fullPath = join(__dirname, '..', DEV_STATIC, folder, serveApp ? '/index.html': ctx.path)
49
- const content = await parseFile(fullPath,
49
+ const content = await parseFileContent(fullPath,
50
raw => serveApp || !raw.length ? raw : adjustBundlerLinks(ctx, uri, raw) )
51
.catch(() => null)
52
if (content === null)
src/upload.ts
+56
-21
@@ -2,13 +2,13 @@ import { getNodeByName, statusCodeForMissingPerm, VfsNode } from './vfs'
2
import Koa from 'koa'
3
import {
4
HTTP_CONFLICT, HTTP_FOOL, HTTP_PAYLOAD_TOO_LARGE, HTTP_RANGE_NOT_SATISFIABLE, HTTP_BAD_REQUEST,
5
- UPLOAD_RESUMABLE, UPLOAD_STATUS,
5
+ UPLOAD_RESUMABLE, UPLOAD_REQUEST_STATUS, UPLOAD_RESUMABLE_HASH,
6
} from './const'
7
import { basename, dirname, extname, join } from 'path'
8
import fs from 'fs'
9
import {
10
- Callback, dirTraversal, loadFileAttr, pendingPromise, storeFileAttr, try_, createStreamLimiter, pathEncode,
11
- enforceFinal, Timeout
10
+ dirTraversal, loadFileAttr, pendingPromise, storeFileAttr, try_, createStreamLimiter, pathEncode,
11
+ enforceFinal, Timeout, with_, parseFile
12
} from './misc'
13
import { notifyClient } from './frontEndApis'
14
import { defineConfig } from './config'
@@ -22,12 +22,13 @@ import events from './events'
22
import { rename, rm } from 'fs/promises'
23
import { expiringCache } from './expiringCache'
24
import { onProcessExit } from './first'
25
+import { once, Transform } from 'stream'
26
27
export const deleteUnfinishedUploadsAfter = defineConfig<undefined|number>('delete_unfinished_uploads_after', 86_400)
28
export const minAvailableMb = defineConfig('min_available_mb', 100)
29
export const dontOverwriteUploading = defineConfig('dont_overwrite_uploading', true)
30
30
-const waitingToBeDeleted: Record<string, { timeout: Timeout, expires: number }> = {}
31
+const waitingToBeDeleted: Record<string, { timeout: Timeout, expires: number, mtimeMs: any, giveBack: any }> = {}
32
onProcessExit(() => {
33
if (!Object.keys(waitingToBeDeleted).length) return
34
console.log("removing unfinished uploads")
@@ -49,9 +50,29 @@ function setUploadMeta(path: string, ctx: Koa.Context) {
50
})
51
}
52
52
-// stay sync because we use this function with formidable()
53
+async function calcHash(fn: string, limit=Infinity) {
54
+ const hash = await makeXXHash()
55
+ const stream = new Transform({
56
+ transform(chunk, enc, done) {
57
+ hash.update(chunk)
58
+ done()
59
+ }
60
+ })
61
+ fs.createReadStream(fn, { end: limit - 1 }).pipe(stream)
62
+ console.debug('hashing', fn)
63
+ await once(stream, 'finish')
64
+ console.debug('hashed', fn)
65
+ return hash.digest().toString(16)
66
+}
67
+
68
+async function makeXXHash(seed?: string) {
69
+ const lib = await import('xxhash-wasm')
70
+ return (await lib.default()).create32(seed ? parseInt(seed, 16) : undefined)
71
+}
72
+
73
const diskSpaceCache = expiringCache<ReturnType<typeof getDiskSpaceSync>>(3_000) // invalidate shortly
74
const uploadingFiles = new Set()
75
+// stay sync because we use this function with formidable()
76
export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx: Koa.Context) {
77
let fullPath = ''
78
if (dirTraversal(path))
@@ -110,7 +131,8 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
131
let tempName = splitAndPreserving ? altTempName : firstTempName
132
const stats = try_(() => fs.statSync(tempName))
133
const resumableSize = stats?.size || 0 // we use size to even when user has not required resume, yet, to notify frontend of the possibility
113
- let resumableTempName = resumableSize > 0 && tempName
134
+ const firstResumableStats = tempName === firstTempName ? stats : try_(() => fs.statSync(firstTempName))
135
+ let resumableTempName = resumableSize > 0 ? tempName : undefined
136
if (resumableTempName)
137
tempName = altTempName
138
// checks for resume feature
@@ -118,9 +140,21 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
140
if (resume > resumableSize)
141
return fail(HTTP_RANGE_NOT_SATISFIABLE)
142
// warn frontend about resume possibility
143
+ let resumeInfo = resumableTempName ? waitingToBeDeleted[resumableTempName] : undefined
144
+ if (resumeInfo?.mtimeMs && resumeInfo?.mtimeMs !== try_(() => fs.statSync(resumableTempName!).mtimeMs)) // outdated?
145
+ resumeInfo = undefined
146
if (!resume)
122
- notifyClient(ctx, UPLOAD_RESUMABLE,
123
- resumableTempName ? { [path]: resumableSize, expires: waitingToBeDeleted[path]?.expires } : { [path]: 0 } )
147
+ with_(resumableTempName, async x => {
148
+ notifyClient(ctx, UPLOAD_RESUMABLE, !x ? { path } : {
149
+ path,
150
+ size: resumableSize,
151
+ // a resumable file exists without a record? then we record it (delayedDelete), plus we provide a hash ASAP, since there's no previous giveBack to compare with
152
+ ...resumeInfo || _.omit(delayedDelete(path, deleteUnfinishedUploadsAfter.get() || 0), 'giveBack'), // giveBack makes sense only if coming from resumeObject
153
+ timeout: undefined
154
+ })
155
+ if (x && !resumeInfo)
156
+ notifyClient(ctx, UPLOAD_RESUMABLE_HASH, { path, hash: await parseFile(x!, calcHash) }) // negligible memory leak
157
+ })
158
let isWritingSecondFile = tempName === altTempName
159
// append if resuming
160
const resuming = resume && resumableTempName
@@ -176,8 +210,10 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
210
try {
211
await rename(tempName, dest)
212
cancelDeletion(tempName) // not necessary, as deletion's failure is silent, but still
179
- if (isWritingSecondFile) // we've been using altTempName, but now we're done, so we can delete firstTempName
180
- await delayedDelete(firstTempName, 0) // wait, so the client can count on the temp-file being gone
213
+ if (isWritingSecondFile) { // we've been using altTempName, but now we're done, so we can delete firstTempName
214
+ cancelDeletion(firstTempName)
215
+ await rm(firstTempName) // wait, so the client can count on the temp-file being gone
216
+ }
217
ctx.state.uploadDestinationPath = dest
218
setUploadMeta(dest, ctx)
219
if (ctx.query.comment)
@@ -220,12 +256,13 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
256
257
function checkIfNewUploadBecameLargerThanResumable() {
258
const currentSize = fileStream.bytesWritten + resume
223
- if (isWritingSecondFile && currentSize > resumableSize)
259
+ if (isWritingSecondFile && currentSize > firstResumableStats?.size!)
260
try { // better be sync here, as we don't want the upload to finish in the middle of the rename
261
fs.renameSync(tempName, firstTempName) // try to rename $upload2 to $upload, overwriting
262
tempName = firstTempName
227
- isWritingSecondFile = resumableTempName = false
228
- notifyClient(ctx, UPLOAD_RESUMABLE, { [path]: 0 }) // no longer resumable
263
+ isWritingSecondFile = false
264
+ resumableTempName = undefined
265
+ notifyClient(ctx, UPLOAD_RESUMABLE, { path }) // no longer resumable
266
}
267
catch{}
268
}
@@ -243,17 +280,15 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
280
return false
281
}
282
246
- function delayedDelete(path: string, secs: number, cb?: Callback) {
247
- if (!secs) {
248
- cancelDeletion(path)
249
- return rm(path)
250
- }
283
+ function delayedDelete(path: string, secs: number) {
284
clearTimeout(waitingToBeDeleted[path]?.timeout)
252
- waitingToBeDeleted[path] = {
285
+ return waitingToBeDeleted[path] = {
286
+ giveBack: ctx.query.giveBack,
287
+ mtimeMs: try_(() => fs.statSync(path).mtimeMs),
288
expires: Date.now() + secs * 1000,
289
timeout: setTimeout(() => {
290
delete waitingToBeDeleted[path]
256
- fs.rm(path, () => cb?.())
291
+ void rm(path)
292
}, secs * 1000)
293
}
294
}
@@ -274,7 +309,7 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
309
ctx.status = status
310
if (msg)
311
ctx.body = msg
277
- notifyClient(ctx, UPLOAD_STATUS, { [path]: ctx.status }) // allow browsers to detect failure while still sending body
312
+ notifyClient(ctx, UPLOAD_REQUEST_STATUS, { [path]: ctx.status }) // allow browsers to detect failure while still sending body
313
}
314
}
315
src/util-files.ts
+7
-4
@@ -123,15 +123,18 @@ export function exists(path: string) {
123
return access(path).then(() => true, () => false)
124
}
125
126
-// read and parse a file, caching unless timestamp has changed
126
+// parse a file, caching unless timestamp has changed
127
export const parseFileCache = new Map<string, { ts: Date, parsed: unknown }>()
128
-export async function parseFile<T>(path: string, parse: (raw: Buffer) => T) {
128
+export async function parseFile<T>(path: string, parse: (path: string) => T) {
129
const { mtime: ts } = await stat(path)
130
const cached = parseFileCache.get(path)
131
if (cached && Number(ts) === Number(cached.ts))
132
return cached.parsed as T
133
- const raw = await readFile(path)
134
- const parsed = parse(raw)
133
+ const parsed = parse(path)
134
parseFileCache.set(path, { ts, parsed })
135
return parsed
136
}
137
+
138
+export async function parseFileContent<T>(path: string, parse: (raw: Buffer) => T) {
139
+ return parseFile(path, () => readFile(path).then(parse))
140
+}
tests/test.ts
+1
-8
@@ -227,13 +227,6 @@ describe('after-login', () => {
227
it('reupload', reqUpload(UPLOAD_DEST, 200))
228
it('delete.method', req(UPLOAD_DEST, 200, { method: 'DELETE' }))
229
it('delete.miss deleted', reqApi('delete', { uri: UPLOAD_DEST }, 404))
230
- it('upload.size', async () => {
231
- const fn = 'temp/size'
232
- await reqUpload(UPLOAD_ROOT + fn, 200, BIG_CONTENT)()
233
- const { size } = statSync(ROOT + fn)
234
- if (size !== BIG_CONTENT.length)
235
- throw Error(`wrote ${size}`)
236
- })
230
it('upload.too much', async () => {
231
const fn = 'temp/tooMuch'
232
const wrongSize = BIG_CONTENT.length / 2
@@ -268,7 +261,7 @@ function reqUpload(dest: string, tester: Tester, body?: string | Readable, size?
261
if (!stats)
262
throw Error("uploaded file not found: " + fn)
263
if (size !== stats.size)
271
- throw Error("uploaded file wrong size: " + fn)
264
+ throw Error(`uploaded file wrong size: ${fn} = ${stats.size.toLocaleString()} expected ${size?.toLocaleString()}`)
265
return true
266
}
267
}