upload: removed 30-second mechanism for resume
Massimo Melina committed
Jan 25, 2025 at 20:40 UTC
25042d8a2768ff635356e7c30260ba3b9ff8b08e
5 files changed
+71
-45
frontend/src/dialog.ts
+1
-1
@@ -164,7 +164,7 @@ export function confirmDialog(msg: ReactElement | string, options: ConfirmOption
164
function Content() {
165
const [sec,setSec] = useState(Math.ceil(timeout||0))
166
useInterval(() => setSec(x => Math.max(0, x-1)), 1000)
167
- const missingText = timeout!>0 && ` (${sec})`
167
+ const missingText = timeout && timeout > 0 && timeout < 60 && ` (${sec})` || ''
168
useEffect(() => {
169
if (timeout && !sec)
170
dialog.close(timeoutConfirm)
frontend/src/uploadQueue.ts
+14
-10
@@ -78,7 +78,7 @@ let req: XMLHttpRequest | undefined
78
let overrideStatus = 0
79
let notificationChannel = ''
80
let notificationSource: EventSource | undefined
81
-let closeLastDialog: undefined | (() => void)
81
+let closeLastDialog: undefined | ((() => void) & { path?: string })
82
83
let reloadOnClose = false
84
export function resetReloadOnClose() {
@@ -172,22 +172,26 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
172
if (!uploading) return
173
if (name === UPLOAD_RESUMABLE) {
174
waitSecondChunk.resolve()
175
- const size = data?.[getFilePath(uploading.file)] //TODO use toUpload?
176
- if (!size) return
175
+ const path = getFilePath(uploading.file)
176
+ const size = data?.[path] //TODO use toUpload?
177
+ if (!size) {
178
+ if (!closeLastDialog?.path || closeLastDialog?.path === path) {// previous resumable is gone
179
+ closeLastDialog?.()
180
+ preserveTempFile = undefined
181
+ }
182
+ return
183
+ }
184
preserveTempFile = true // this is affecting only split-uploads, because is undefined on first chunk (or no chunking)
185
if (size > toUpload.file.size) return
179
- const {expires} = data
180
- let timeout = typeof expires !== 'number' ? 0
181
- : (Number(new Date(expires)) - Date.now())
182
- if (timeout)
183
- setTimeout(() => preserveTempFile = undefined, timeout) // the resumable is gone
186
closeLastDialog?.()
187
const cancelSub = subscribeKey(uploadState, 'partial', v =>
188
v >= size && closeLastDialog?.() ) // dismiss dialog as soon as we pass the threshold
189
const msg = t('confirm_resume', "Resume upload?") + ` (${formatPerc(size/toUpload.file.size)} = ${formatBytes(size)})`
188
- timeout /= 1000 // needs seconds
190
+ const {expires} = data
191
+ const timeout = typeof expires !== 'number' ? 0
192
+ : (Number(new Date(expires)) - Date.now()) / 1000
193
const dialog = confirmDialog(msg, { timeout })
190
- closeLastDialog = dialog.close
194
+ closeLastDialog = Object.assign(dialog.close, { path })
195
const confirmed = await dialog
196
cancelSub()
197
if (!confirmed) return
src/cross.ts
+1
@@ -38,6 +38,7 @@ type Truthy<T> = T extends false | '' | 0 | null | undefined | void ? never : T
38
export type Callback<IN=void, OUT=void> = (x:IN) => OUT
39
export type Promisable<T> = T | Promise<T>
40
export type Functionable<T> = T | ((...args: any[]) => T)
41
+export type Timeout = ReturnType<typeof setTimeout>
42
export interface VfsPerms {
43
can_see?: Who
44
can_read?: Who
src/upload.ts
+42
-32
@@ -8,7 +8,7 @@ import { basename, dirname, extname, join } from 'path'
8
import fs from 'fs'
9
import {
10
Callback, dirTraversal, loadFileAttr, pendingPromise, storeFileAttr, try_, createStreamLimiter, pathEncode,
11
- enforceFinal
11
+ enforceFinal, Timeout
12
} from './misc'
13
import { notifyClient } from './frontEndApis'
14
import { defineConfig } from './config'
@@ -27,7 +27,7 @@ export const deleteUnfinishedUploadsAfter = defineConfig<undefined|number>('dele
27
export const minAvailableMb = defineConfig('min_available_mb', 100)
28
export const dontOverwriteUploading = defineConfig('dont_overwrite_uploading', true)
29
30
-const waitingToBeDeleted: Record<string, ReturnType<typeof setTimeout>> = {}
30
+const waitingToBeDeleted: Record<string, { timeout: Timeout, expires: number }> = {}
31
onProcessExit(() => {
32
if (!Object.keys(waitingToBeDeleted).length) return
33
console.log("removing unfinished uploads")
@@ -105,12 +105,12 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
105
// use temporary name while uploading
106
const keepName = basename(fullPath).slice(-200)
107
const firstTempName = join(dir, 'hfs$upload-' + keepName)
108
- const altTempName = join(dir, 'hfs$upload2-' + keepName)
108
+ const altTempName = join(dir, 'hfs$upload2-' + keepName) // this file makes sense only while smaller than firstTempName
109
const splitAndPreserving = ctx.query.preserveTempFile // frontend knows about existing temp that can be resumed, but it is not resuming that, but instead it is continuing split-uploading on alternative temp file
110
let tempName = splitAndPreserving ? altTempName : firstTempName
111
const stats = try_(() => fs.statSync(tempName))
112
const resumableSize = stats?.size || 0 // we use size to even when user has not required resume, yet, to notify frontend of the possibility
113
- const resumableTempName = resumableSize > 0 && tempName
113
+ let resumableTempName = resumableSize > 0 && tempName
114
if (resumableTempName)
115
tempName = altTempName
116
// checks for resume feature
@@ -118,25 +118,16 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
118
if (resume > resumableSize)
119
return fail(HTTP_RANGE_NOT_SATISFIABLE)
120
// warn frontend about resume possibility
121
- let resumableLost = false
122
- if (!resume && !resumableTempName)
123
- notifyClient(ctx, UPLOAD_RESUMABLE, { [path]: 0 })
124
- if (!resume && resumableTempName) {
125
- const timeout = 30
126
- notifyClient(ctx, UPLOAD_RESUMABLE, { [path]: resumableSize, expires: Date.now() + timeout * 1000 })
127
- delayedDelete(resumableTempName, timeout, () => // if user resumes, this upload is interrupted, and next upload will cancel this delayedDelete
128
- fs.rename(tempName, resumableTempName, err => { // try to rename $upload2 to $upload, overwriting
129
- if (err) return
130
- tempName = resumableTempName
131
- resumableLost = true
132
- }) )
133
- }
121
+ if (!resume)
122
+ notifyClient(ctx, UPLOAD_RESUMABLE,
123
+ resumableTempName ? { [path]: resumableSize, expires: waitingToBeDeleted[path]?.expires } : { [path]: 0 } )
124
+ let isWritingSecondFile = tempName === altTempName
125
// append if resuming
126
const resuming = resume && resumableTempName
127
if (!resuming)
128
resume = 0
129
const writeStream = createStreamLimiter(reqSize ?? Infinity)
139
- if (resuming && !splitAndPreserving) {
130
+ if (resume && resumableTempName && !splitAndPreserving) {
131
fs.rm(tempName, () => {})
132
tempName = resumableTempName
133
}
@@ -147,7 +138,7 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
138
const resEvent = events.emit('uploadStart', obj)
139
if (resEvent?.isDefaultPrevented()) return
140
150
- const fileStream = resuming ? fs.createWriteStream(resumableTempName, { flags: 'r+', start: resume })
141
+ const fileStream = resume && resumableTempName ? fs.createWriteStream(resumableTempName, { flags: 'r+', start: resume })
142
: fs.createWriteStream(tempName)
143
writeStream.on('error', e => {
144
releaseFile()
@@ -156,14 +147,16 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
147
writeStream.pipe(fileStream)
148
Object.assign(obj, { fileStream })
149
trackProgress()
150
+ // the file stream doesn't have an event for data being written, so we use 'data' of its feeder, which happens before, so we postpone a bit, trying have a fresher view
151
+ writeStream.on('data', () => setTimeout(checkIfNewUploadBecameLargerThanResumable))
152
153
const lockMiddleware = pendingPromise<string>() // outside we need to know when all operations stopped
154
writeStream.once('close', async () => {
155
try {
156
await new Promise(res => fileStream.close(res)) // this only seem to be necessary on Windows
157
if (ctx.isAborted()) {
165
- if (resumableTempName && !resumableLost && !resuming) // we don't want to be left with 2 temp files
166
- return rm(tempName)
158
+ if (isWritingSecondFile) // we don't want to be left with 2 temp files
159
+ return rm(altTempName).catch(console.warn)
160
const sec = deleteUnfinishedUploadsAfter.get()
161
return _.isNumber(sec) && delayedDelete(tempName, sec)
162
}
@@ -171,7 +164,7 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
164
let dest = fullPath
165
if (dontOverwriteUploading.get() && !await overwriteAnyway() && fs.existsSync(dest)) {
166
if (overwriteRequestedButForbidden) {
174
- await rm(tempName)
167
+ await rm(tempName).catch(console.warn)
168
return fail()
169
}
170
const ext = extname(dest)
@@ -183,14 +176,12 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
176
try {
177
await rename(tempName, dest)
178
cancelDeletion(tempName) // not necessary, as deletion's failure is silent, but still
186
- if (splitAndPreserving) // we've been using altTempName, but now we're done, so we can delete firstTempName
187
- delayedDelete(firstTempName, 0)
179
+ if (isWritingSecondFile) // we've been using altTempName, but now we're done, so we can delete firstTempName
180
+ await delayedDelete(firstTempName, 0) // wait, so the client can count on the temp-file being gone
181
ctx.state.uploadDestinationPath = dest
182
setUploadMeta(dest, ctx)
183
if (ctx.query.comment)
184
void setCommentFor(dest, String(ctx.query.comment))
192
- if (resumableTempName && !resuming) // this happens if user decided to not resume and the new upload finished before delayedDelete
193
- rm(resumableTempName).catch(console.warn)
185
obj.uri = enforceFinal('/', baseUri) + pathEncode(basename(dest))
186
events.emit('uploadFinished', obj)
187
if (resEvent) for (const cb of resEvent)
@@ -226,6 +217,18 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
217
}, 1000)
218
writeStream.once('close', () => clearInterval(h) )
219
}
220
+
221
+ function checkIfNewUploadBecameLargerThanResumable() {
222
+ const currentSize = fileStream.bytesWritten + resume
223
+ if (isWritingSecondFile && currentSize > resumableSize)
224
+ try { // better be sync here, as we don't want the upload to finish in the middle of the rename
225
+ fs.renameSync(tempName, firstTempName) // try to rename $upload2 to $upload, overwriting
226
+ tempName = firstTempName
227
+ isWritingSecondFile = resumableTempName = false
228
+ notifyClient(ctx, UPLOAD_RESUMABLE, { [path]: 0 }) // no longer resumable
229
+ }
230
+ catch{}
231
+ }
232
}
233
catch (e: any) {
234
releaseFile()
@@ -241,15 +244,22 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
244
}
245
246
function delayedDelete(path: string, secs: number, cb?: Callback) {
244
- clearTimeout(waitingToBeDeleted[path])
245
- waitingToBeDeleted[path] = setTimeout(() => {
246
- delete waitingToBeDeleted[path]
247
- fs.rm(path, () => cb?.())
248
- }, secs * 1000)
247
+ if (!secs) {
248
+ cancelDeletion(path)
249
+ return rm(path)
250
+ }
251
+ clearTimeout(waitingToBeDeleted[path]?.timeout)
252
+ waitingToBeDeleted[path] = {
253
+ expires: Date.now() + secs * 1000,
254
+ timeout: setTimeout(() => {
255
+ delete waitingToBeDeleted[path]
256
+ fs.rm(path, () => cb?.())
257
+ }, secs * 1000)
258
+ }
259
}
260
261
function cancelDeletion(path: string) {
252
- clearTimeout(waitingToBeDeleted[path])
262
+ clearTimeout(waitingToBeDeleted[path]?.timeout)
263
delete waitingToBeDeleted[path]
264
}
265
tests/test.ts
+13
-2
@@ -207,9 +207,18 @@ describe('after-login', () => {
207
await makeAbortedRequest(timeFirstRequest * .5) // upload less than r1
208
if (size !== getTempSize()) // shouldn't change, as r2 is smaller
209
throw Error("modified temp file")
210
+ await makeAbortedRequest(timeFirstRequest * 1.5) // upload more than r1
211
+ if (!(size < getTempSize()!)) // should be increased
212
+ throw Error("temp file not enlarged")
213
await reqUpload(UPLOAD_DEST, 200, makeReadableThatTakes(0))() // quickly complete the upload, and check for final size
214
if (getTempSize())
215
throw Error("temp file should be cleared")
216
+ // test resume
217
+ await makeAbortedRequest(timeFirstRequest)
218
+ const partial = getTempSize()
219
+ if (!partial)
220
+ throw Error("partial file missing")
221
+ await reqUpload(UPLOAD_DEST, 200, Readable.from(BIG_CONTENT.slice(partial)), BIG_CONTENT.length, partial)()
222
})
223
const renameTo = 'z'
224
it('rename.ok', reqApi('rename', { uri: UPLOAD_DEST, dest: renameTo }, 200))
@@ -246,7 +255,9 @@ function login(usr: string, pwd=password) {
255
reqApi(cmd, params, (x,res)=> res.statusCode < 400)())
256
}
257
249
-function reqUpload(dest: string, tester: Tester, body?: string | Readable, size?: number) {
258
+function reqUpload(dest: string, tester: Tester, body?: string | Readable, size?: number, resume?: number) {
259
+ if (resume)
260
+ dest += '?resume=' + resume
261
size ??= (body as any)?.length ?? statSync(SAMPLE_FILE_PATH).size // it's ok that Readable.length is undefined
262
if (tester === 200)
263
tester = {
@@ -263,7 +274,7 @@ function reqUpload(dest: string, tester: Tester, body?: string | Readable, size?
274
}
275
return req(dest, tester, {
276
method: 'PUT',
266
- headers: { 'content-length': size },
277
+ headers: { 'content-length': size === undefined ? size : size - (resume||0) },
278
body: body ?? createReadStream(SAMPLE_FILE_PATH)
279
})
280
}