upload: allow resume in case of disconnection
Massimo Melina committed
May 10, 2025 at 18:42 UTC
a251ec09c25c15b8451a3d7701b5b77a28a1f378
2 files changed
+29
-20
frontend/src/uploadQueue.ts
+13
-10
@@ -89,8 +89,8 @@ export function resetReloadOnClose() {
89
return true
90
}
91
92
-export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
93
- console.debug('start upload', getFilePath(toUpload.file), resume)
92
+export async function startUpload(toUpload: ToUpload, to: string, startingResume=0) {
93
+ console.debug('start upload', getFilePath(toUpload.file), startingResume)
94
let resuming = false
95
let preserveTempFile = undefined
96
overrideStatus = 0
@@ -101,9 +101,12 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
101
const waitSecondChunk = pendingPromise() // to avoid race condition in case the notification arrives after the first chunk is finished
102
const splitSize = getHFS().splitUploads
103
const fullSize = toUpload.file.size
104
- let offset = resume
104
+ let offset = startingResume
105
+ let splitResume = startingResume // keep track of "split" advancements
106
+ let lastWrittenReceived = 0
107
let stopLooping = false
108
do { // at least one iteration, even for empty files
109
+ offset = Math.max(splitResume, lastWrittenReceived)
110
const req = currentReq = new XMLHttpRequest()
111
req.timeout = 10_000
112
const finished = pendingPromise()
@@ -138,8 +141,8 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
141
else if (!status) // request failed at a network level, so try again, but not too often
142
return await wait(2000)
143
else {
141
- offset += splitSize || Infinity
142
- if (offset < fullSize) return // go on with the next chunk
144
+ splitResume += splitSize || Infinity
145
+ if (splitResume < fullSize) return // go on with the next chunk
146
waitSecondChunk.resolve() // finished, there's no second chunk
147
uploadState.done.push({ ...toUpload, res: tryJson(req.responseText) })
148
uploadState.doneByte += toUpload!.file.size
@@ -174,7 +177,7 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
177
}), true)
178
req.send(toUpload.file.slice(offset, splitSize ? offset + splitSize : undefined))
179
await finished
177
- if (!resume && notificationSource?.readyState === OPEN) // wait only if notifications are currently available
180
+ if (!startingResume && notificationSource?.readyState === OPEN) // wait only if notifications are currently available
181
await waitSecondChunk
182
} while (!stopLooping && offset < fullSize)
183
@@ -198,11 +201,11 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
201
waitSecondChunk.resolve()
202
const path = getFilePath(uploading.file)
203
if (path !== data.path) return // is it about current file?
204
+ if (data.written)
205
+ return lastWrittenReceived = data.written
206
const {size} = data //TODO use toUpload?
202
- if (!size) {
203
- preserveTempFile = undefined
204
- return
205
- }
207
+ if (!size)
208
+ return preserveTempFile = undefined
209
preserveTempFile = true // this is affecting only split-uploads, because is undefined on first chunk (or no chunking)
210
if (size > toUpload.file.size) return
211
if (data.giveBack) {
src/upload.ts
+16
-10
@@ -123,6 +123,7 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
123
uploadingFiles.add(fullPath)
124
let overwriteRequestedButForbidden = false
125
try {
126
+ const sendCurrentSize = _.debounce(() => notifyClient(ctx, UPLOAD_RESUMABLE, { path, written: getCurrentSize() }), 1000, { maxWait: 1000 })
127
// if upload creates a folder, then add meta to it too
128
if (!dir.endsWith(':\\') && fs.mkdirSync(dir, { recursive: true }))
129
setUploadMeta(dir, ctx)
@@ -153,21 +154,21 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
154
size: resumableSize,
155
// a resumable file exists without a record? then we record it (delayedDelete), plus we provide a hash ASAP, since there's no previous giveBack to compare with
156
...resumeInfo || _.omit(delayedDelete(path, deleteUnfinishedUploadsAfter.get() || 0), 'giveBack'), // giveBack makes sense only if coming from resumeObject
156
- timeout: undefined
157
+ timeout: undefined // this entry is here to remove the property copied in the previous line
158
})
159
if (x && !resumeInfo)
160
notifyClient(ctx, UPLOAD_RESUMABLE_HASH, { path, hash: await parseFile(x!, calcHash) }) // negligible memory leak
161
})
161
- let isWritingSecondFile = tempName === altTempName
162
// append if resuming
163
const resuming = resume && resumableTempName
164
if (!resuming)
165
resume = 0
166
const writeStream = createStreamLimiter(contentLength ?? Infinity)
167
- if (resume && resumableTempName && !splitAndPreserving) {
168
- fs.rm(tempName, () => {})
167
+ if (resume && resumableTempName && !splitAndPreserving) { // we want to resume the firstTempName, actually
168
+ fs.rm(altTempName, () => {})
169
tempName = resumableTempName
170
}
171
+ let isWritingSecondFile = tempName === altTempName
172
cancelDeletion(tempName)
173
const fullSize = stillToWrite + resume
174
ctx.state.uploadDestinationPath = tempName
@@ -246,10 +247,6 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
247
})
248
return Object.assign(obj.writeStream, { lockMiddleware })
249
249
- function bytesGot() {
250
- return fileStream.bytesWritten + fileStream.writableLength
251
- }
252
-
250
function trackProgress() {
251
let lastGot = 0
252
let lastGotTime = 0
@@ -267,9 +264,18 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
264
writeStream.once('close', () => clearInterval(h) )
265
}
266
267
+ function getCurrentSize() {
268
+ return bytesGot() + resume
269
+ }
270
+
271
+ function bytesGot() {
272
+ return fileStream.bytesWritten + fileStream.writableLength
273
+ }
274
+
275
function checkIfNewUploadBecameLargerThanResumable() {
271
- const currentSize = bytesGot() + resume
272
- if (isWritingSecondFile && currentSize > firstResumableStats?.size!)
276
+ if (!isWritingSecondFile)
277
+ return sendCurrentSize() // keep the client updated in case it needs to resume on disconnection
278
+ if (getCurrentSize() > firstResumableStats?.size!)
279
try { // better be sync here, as we don't want the upload to finish in the middle of the rename
280
fs.renameSync(tempName, firstTempName) // try to rename $upload2 to $upload, overwriting
281
tempName = firstTempName