fix: admin/logs: wrong partial upload size
Massimo Melina committed
May 4, 2025 at 10:33 UTC
820d526ab6f9cb73ab9d1bac0f2a8e0d6a2866cb
4 files changed
+28
-20
admin/src/LogsPage.ts
+3
-2
@@ -340,10 +340,11 @@ export function LogFile({ file, footerSide, hidden, limit, filter, ...rest }: Lo
340
if (extra?.ua && !showAgent)
341
setShowAgent(true)
342
if (row.uri) {
343
- const partial = stringAfter('?', row.uri).includes('partial=')
343
const upload = row.method === 'PUT' || extra?.ul
344
+ const partial = upload && stringAfter('?', row.uri).includes('partial=')
345
if (upload)
346
- row.length = extra?.size ?? 0
346
+ row.length = (extra?.size ?? 0)
347
+ + (!partial && Number(row.uri.match(/\?.*resume=(\d+)/)?.[1]) || 0) // show full size for full uploads
348
row.notes = extra?.dl ? "full download " + (extra.speed ? formatSpeed(extra.speed, { sep: ' ' }) : '') // 'dl' here is not the '?dl' of the url, and has a different meaning
349
: upload ? `${partial ? "partial " : ""} upload ${extra.speed ? formatSpeed(extra.speed, { sep: ' ' }) : ''}`
350
: row.status === HTTP_UNAUTHORIZED && row.uri?.startsWith(API_URL + 'loginSrp') ? "login failed" + prefix(':\n', extra?.u)
frontend/src/uploadQueue.ts
+1
-1
@@ -155,7 +155,7 @@ export async function startUpload(toUpload: ToUpload, to: string, resume=0) {
155
req.open('PUT', to + pathEncode(uploadPath) + buildUrlQueryString({
156
notificationChannel,
157
giveBack: toUpload.file.lastModified,
158
- ...partial && { partial: fullSize - offset },
158
+ ...partial && { partial: fullSize - offset }, // how much space we need
159
...offset && { resume: offset, preserveTempFile },
160
...toUpload.comment && { comment: toUpload.comment },
161
...with_(state.uploadOnExisting, x => x !== 'rename' && { existing: x }), // rename is the default
src/log.ts
+3
-3
@@ -123,10 +123,10 @@ export const logMw: Koa.Middleware = async (ctx, next) => {
123
ctx.logExtra(ctx.vfsNode && {
124
speed: Math.round(length / duration),
125
...ctx.state.includesLastByte && ctx.res.finished && { dl: 1 }
126
- } || ctx.state.uploadPath && {
126
+ } || ctx.state.uploadSize !== undefined && {
127
ul: ctx.state.uploads,
128
- size: ctx.state.opTotal,
129
- speed: Math.round((ctx.state.opTotal! - (ctx.state.opOffset || 0)) / duration)
128
+ size: ctx.state.uploadSize,
129
+ speed: Math.round(ctx.state.uploadSize / duration),
130
})
131
if (conn?.country)
132
ctx.logExtra({ country: conn.country })
src/upload.ts
+21
-14
@@ -90,9 +90,10 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
90
fullPath = join(base.source!, path)
91
const dir = dirname(fullPath)
92
const min = minAvailableMb.get() * (1 << 20)
93
- const reqSize = Number(ctx.headers["content-length"])
94
- const fullSize = Math.max(reqSize, Number(ctx.query.partial) || 0)
95
- if (isNaN(fullSize)) {
93
+ const contentLength = Number(ctx.headers["content-length"])
94
+ const isPartial = ctx.query.partial !== undefined // while the presence of "partial" conveys the upload is split...
95
+ const stillToWrite = Math.max(contentLength, Number(ctx.query.partial) || 0) // ...the number is used to tell how much space we need (fullSize - offset)
96
+ if (isNaN(stillToWrite)) {
97
if (min)
98
return fail(HTTP_BAD_REQUEST, 'content-length mandatory')
99
}
@@ -108,7 +109,7 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
109
const { free } = res
110
if (typeof free !== 'number' || isNaN(free))
111
throw ''
111
- if (fullSize > free - (min || 0))
112
+ if (stillToWrite > free - (min || 0))
113
return fail(HTTP_INSUFFICIENT_STORAGE)
114
}
115
catch(e: any) { // warn, but let it through
@@ -162,12 +163,13 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
163
const resuming = resume && resumableTempName
164
if (!resuming)
165
resume = 0
165
- const writeStream = createStreamLimiter(reqSize ?? Infinity)
166
+ const writeStream = createStreamLimiter(contentLength ?? Infinity)
167
if (resume && resumableTempName && !splitAndPreserving) {
168
fs.rm(tempName, () => {})
169
tempName = resumableTempName
170
}
171
cancelDeletion(tempName)
172
+ const fullSize = stillToWrite + resume
173
ctx.state.uploadDestinationPath = tempName
174
// allow plugins to mess with the write-stream, because the read-stream can be complicated in case of multipart
175
const obj = { ctx, writeStream, uri: '' }
@@ -183,20 +185,21 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
185
writeStream.pipe(fileStream)
186
Object.assign(obj, { fileStream })
187
trackProgress()
186
- // the file stream doesn't have an event for data being written, so we use 'data' of its feeder, which happens before, so we postpone a bit, trying have a fresher view
188
+ // the file stream doesn't have an event for data being written, so we use 'data' of its feeder, which happens before, so we postpone a bit, trying to have a fresher number
189
writeStream.on('data', () => setTimeout(checkIfNewUploadBecameLargerThanResumable))
190
189
- const lockMiddleware = pendingPromise<string>() // outside we need to know when all operations stopped
191
+ const lockMiddleware = pendingPromise<string>() // expose when all operations stopped
192
writeStream.once('close', async () => {
193
try {
192
- await new Promise(res => fileStream.close(res)) // this only seem to be necessary on Windows
194
+ ctx.state.uploadSize = bytesGot() // in case content-length is not specified
195
+ await new Promise(res => fileStream.close(res)) // this only seems necessary on Windows
196
if (ctx.isAborted()) {
197
if (isWritingSecondFile) // we don't want to be left with 2 temp files
198
return rm(altTempName).catch(console.warn)
199
const sec = deleteUnfinishedUploadsAfter.get()
200
return _.isNumber(sec) && delayedDelete(tempName, sec)
201
}
199
- if (ctx.query.partial) // this upload is partial, and we are supposed to leave the upload as unfinished, with the temp name
202
+ if (isPartial) // we are supposed to leave the upload as unfinished, with the temp name
203
return ctx.status = HTTP_NO_CONTENT // lockMiddleware contains an empty string, so we must take care of the status
204
let dest = fullPath
205
if (dontOverwriteUploading.get() && !await overwriteAnyway() && fs.existsSync(dest)) {
@@ -243,26 +246,29 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
246
})
247
return Object.assign(obj.writeStream, { lockMiddleware })
248
249
+ function bytesGot() {
250
+ return fileStream.bytesWritten + fileStream.writableLength
251
+ }
252
+
253
function trackProgress() {
254
let lastGot = 0
255
let lastGotTime = 0
249
- const opTotal = fullSize + resume
250
- Object.assign(ctx.state, { opTotal, opOffset: resume / opTotal, opProgress: 0 })
256
+ Object.assign(ctx.state, { opTotal: fullSize, opOffset: resume / fullSize, opProgress: 0 })
257
const conn = updateConnectionForCtx(ctx)
258
if (!conn) return
259
const h = setInterval(() => {
260
const now = Date.now()
255
- const got = fileStream.bytesWritten
261
+ const got = bytesGot()
262
const inSpeed = roundSpeed((got - lastGot) / (now - lastGotTime))
263
lastGot = got
264
lastGotTime = now
259
- updateConnection(conn, { inSpeed, got }, { opProgress: (resume + got) / opTotal })
265
+ updateConnection(conn, { inSpeed, got }, { opProgress: (resume + got) / fullSize })
266
}, 1000)
267
writeStream.once('close', () => clearInterval(h) )
268
}
269
270
function checkIfNewUploadBecameLargerThanResumable() {
265
- const currentSize = fileStream.bytesWritten + resume
271
+ const currentSize = bytesGot() + resume
272
if (isWritingSecondFile && currentSize > firstResumableStats?.size!)
273
try { // better be sync here, as we don't want the upload to finish in the middle of the rename
274
fs.renameSync(tempName, firstTempName) // try to rename $upload2 to $upload, overwriting
@@ -322,5 +328,6 @@ export function uploadWriter(base: VfsNode, baseUri: string, path: string, ctx:
328
declare module "koa" {
329
interface DefaultState {
330
uploadDestinationPath?: string
331
+ uploadSize?: number
332
}
333
}
\ No newline at end of file