new config split_uploads
Massimo Melina committed
Sep 18, 2024 at 23:07 UTC
4c17f4f94d83db20628544a31a60e08b70095e66
5 files changed
+58
-37
admin/src/OptionsPage.ts
+5
-3
@@ -192,11 +192,13 @@ export default function OptionsPage() {
192
helperText: "The icon associated to your website" },
193
194
h(Section, { title: "Uploads" }),
195
- { k: 'dont_overwrite_uploading', comp: BoolField, sm: 4, md: 6, label: "Don't overwrite uploading",
195
+ { k: 'dont_overwrite_uploading', comp: BoolField, md: 4, label: "Uploads don't overwrite",
196
helperText: "Files will be numbered to avoid overwriting" },
197
- { k: 'delete_unfinished_uploads_after', comp: NumberField, sm: 4, md: 3, min : 0, unit: "seconds", placeholder: "Never",
197
+ { k : CFG.split_uploads, comp: NumberField, unit: 'MB', md: 2, fromField: x => x * 1E6, toField: x => x ? x / 1E6 : null, min: 0, step: .1,
198
+ placeholder: "disabled", label: "Split uploads in chunks", helperText: "Overcome proxy limits" },
199
+ { k: 'delete_unfinished_uploads_after', comp: NumberField, md: 3, min : 0, unit: "seconds", placeholder: "Never",
200
helperText: "Leave empty to never delete" },
199
- { k: 'min_available_mb', comp: NumberField, sm: 4, md: 3, min : 0, unit: "MBytes", placeholder: "None",
201
+ { k: 'min_available_mb', comp: NumberField, md: 3, min : 0, unit: "MBytes", placeholder: "None",
202
label: "Min. available disk space", helperText: "Reject uploads that don't comply" },
203
204
h(Section, { title: "Others" }),
frontend/src/upload.ts
+49
-33
@@ -5,7 +5,7 @@ import { Btn, Flex, FlexV, iconBtn, Select } from './components'
5
import {
6
basename, closeDialog, formatBytes, formatPerc, hIcon, useIsMobile, newDialog, prefix, selectFiles, working,
7
HTTP_CONFLICT, HTTP_PAYLOAD_TOO_LARGE, formatSpeed, dirname, getHFS, onlyTruthy, with_, cpuSpeedIndex,
8
- buildUrlQueryString, randomId, HTTP_MESSAGES, pathEncode,
8
+ buildUrlQueryString, randomId, HTTP_MESSAGES, pathEncode, pendingPromise,
9
} from './misc'
10
import _ from 'lodash'
11
import { INTERNAL_Snapshot, proxy, ref, snapshot, subscribe, useSnapshot } from 'valtio'
@@ -310,38 +310,54 @@ async function startUpload(toUpload: ToUpload, to: string, resume=0) {
310
overrideStatus = 0
311
uploadState.uploading = toUpload
312
await subscribeNotifications()
313
- req = new XMLHttpRequest()
314
- req.onloadend = () => {
315
- if (req?.readyState !== 4) return
316
- const status = overrideStatus || req.status
317
- closeLast?.()
318
- if (!status || status === HTTP_CONFLICT) // 0 = user-aborted, HTTP_CONFLICT = skipped because existing
319
- uploadState.skipped.push(toUpload)
320
- else if (status >= 400)
321
- error(status)
322
- else
323
- done()
324
- if (!resuming)
325
- next()
326
- }
327
- req.onerror = () => error(0)
328
- let lastProgress = 0
329
- req.upload.onprogress = (e:any) => {
330
- uploadState.partial = e.loaded + resume
331
- uploadState.progress = uploadState.partial / (e.total + resume)
332
- bytesSent += e.loaded - lastProgress
333
- lastProgress = e.loaded
334
- }
335
- let uploadPath = path(toUpload.file)
336
- if (toUpload.name)
337
- uploadPath = prefix('', dirname(uploadPath), '/') + toUpload.name
338
- req.open('PUT', to + pathEncode(uploadPath) + buildUrlQueryString({
339
- notificationChannel,
340
- ...resume && { resume: String(resume) },
341
- ...toUpload.comment && { comment: toUpload.comment },
342
- ...with_(state.uploadOnExisting, x => x !== 'rename' && { existing: x }), // rename is the default
343
- }), true)
344
- req.send(toUpload.file.slice(resume))
313
+ const splitSize = getHFS().splitUploads || Infinity
314
+ const fullSize = toUpload.file.size
315
+ let offset = resume
316
+ do { // at least one iteration, even for empty files
317
+ req = new XMLHttpRequest()
318
+ const finished = pendingPromise()
319
+ req.onloadend = () => {
320
+ finished.resolve()
321
+ if (req?.readyState !== 4) return
322
+ const status = overrideStatus || req.status
323
+ closeLast?.()
324
+ if (!status || status === HTTP_CONFLICT) // 0 = user-aborted, HTTP_CONFLICT = skipped because existing
325
+ uploadState.skipped.push(toUpload)
326
+ else if (status >= 400)
327
+ error(status)
328
+ else {
329
+ offset += splitSize
330
+ if (offset < fullSize) return // continue looping
331
+ done()
332
+ }
333
+ if (!resuming)
334
+ next()
335
+ offset = fullSize // stop looping
336
+ }
337
+ req.onerror = () => {
338
+ error(0)
339
+ finished.resolve()
340
+ }
341
+ let lastProgress = 0
342
+ req.upload.onprogress = (e:any) => {
343
+ uploadState.partial = e.loaded + offset
344
+ uploadState.progress = uploadState.partial / fullSize
345
+ bytesSent += e.loaded - lastProgress
346
+ lastProgress = e.loaded
347
+ }
348
+ let uploadPath = path(toUpload.file)
349
+ if (toUpload.name)
350
+ uploadPath = prefix('', dirname(uploadPath), '/') + toUpload.name
351
+ req.open('PUT', to + pathEncode(uploadPath) + buildUrlQueryString({
352
+ notificationChannel,
353
+ ...offset + splitSize < fullSize && { partial: 'y' },
354
+ ...offset && { resume: String(offset) },
355
+ ...toUpload.comment && { comment: toUpload.comment },
356
+ ...with_(state.uploadOnExisting, x => x !== 'rename' && { existing: x }), // rename is the default
357
+ }), true)
358
+ req.send(toUpload.file.slice(offset, offset + splitSize))
359
+ await finished
360
+ } while (offset < fullSize)
361
362
async function subscribeNotifications() {
363
if (notificationChannel) return
src/cross.ts
+1
-1
@@ -25,7 +25,7 @@ export const SORT_BY_OPTIONS = ['name', 'extension', 'size', 'time']
25
export const THEME_OPTIONS = { auto: '', light: 'light', dark: 'dark' }
26
export const CFG = constMap(['geo_enable', 'geo_allow', 'geo_list', 'geo_allow_unknown', 'dynamic_dns_url',
27
'log', 'error_log', 'log_rotation', 'dont_log_net', 'log_gui', 'log_api', 'log_ua', 'log_spam', 'track_ips',
28
- 'max_downloads', 'max_downloads_per_ip', 'max_downloads_per_account', 'roots', 'force_address'])
28
+ 'max_downloads', 'max_downloads_per_ip', 'max_downloads_per_account', 'roots', 'force_address', 'split_uploads'])
29
export const LIST = { add: '+', remove: '-', update: '=', props: 'props', ready: 'ready', error: 'e' }
30
export type Dict<T=any> = Record<string, T>
31
export type Falsy = false | null | undefined | '' | 0
src/serveGuiFiles.ts
+2
@@ -18,6 +18,7 @@ import { defineConfig, getConfig } from './config'
18
import { getLangData } from './lang'
19
import { dontOverwriteUploading } from './upload'
20
21
+const splitUploads = defineConfig(CFG.split_uploads, 0)
22
export const logGui = defineConfig(CFG.log_gui, false)
23
_.each(FRONTEND_OPTIONS, (v,k) => defineConfig(k, v)) // define default values
24
@@ -106,6 +107,7 @@ async function treatIndex(ctx: Koa.Context, filesUri: string, body: string) {
107
loadScripts: Object.fromEntries(mapPlugins((p, id) => [id, p.frontend_js?.map(f => f.includes('//') ? f : pub + id + '/' + f)])),
108
prefixUrl: ctx.state.revProxyPath,
109
dontOverwriteUploading: dontOverwriteUploading.get(),
110
+ splitUploads: splitUploads.get(),
111
forceTheme: mapPlugins(p => _.isString(p.isTheme) ? p.isTheme : undefined).find(Boolean),
112
customHtml: _.omit(getAllSections(), ['top', 'bottom', 'htmlHead', 'style']), // exclude the sections we already apply in this phase
113
...newObj(FRONTEND_OPTIONS, (v, k) => getConfig(k)),
src/upload.ts
+1
@@ -145,6 +145,7 @@ export function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
145
const sec = deleteUnfinishedUploadsAfter.get()
146
return _.isNumber(sec) && delayedDelete(tempName, sec)
147
}
148
+ if (ctx.query.partial) return // this upload is partial, and we are supposed to leave the upload as unfinished, with the temp name
149
let dest = fullPath
150
if (dontOverwriteUploading.get() && !await overwriteAnyway() && fs.existsSync(dest)) {
151
if (overwriteRequestedButForbidden) {