upload resume
Massimo Melina committed
Jan 22, 2023 at 15:49 UTC
28c036a128e1e8000f91011994d729cd523a6baf
5 files changed
+120
-46
admin/src/ConfigPage.ts
+1
-1
@@ -112,7 +112,7 @@ export default function ConfigPage() {
112
helperText: values.allowed_referer ? "Leave empty to allow any" : "Use this to avoid direct links from other websites", },
113
{ k: 'zip_calculate_size_for_seconds', comp: NumberField, sm: 6, md: 3, label: "Calculate ZIP size for", unit: "seconds",
114
helperText: "If time is not enough, the browser will not show download percentage" },
115
- { k: 'delete_unfinished_uploads_after', comp: NumberField, sm: 6, md: 3, min : 0, unit: "seconds",
115
+ { k: 'delete_unfinished_uploads_after', comp: NumberField, sm: 6, md: 3, min : 0, unit: "seconds", placeholder: "Never",
116
helperText: "Leave empty to never delete" },
117
{ k: 'custom_header', multiline: true, sm: 12, md: 6, sx: { '& textarea': { fontFamily: 'monospace' } },
118
helperText: "Any HTML code here will be used as header for the Frontend"
frontend/src/dialog.ts
+13
-5
@@ -1,9 +1,10 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
-import { createElement as h, ReactElement, ReactNode, useEffect, useRef } from 'react'
3
+import { createElement as h, ReactElement, ReactNode, useEffect, useRef, useState } from 'react'
4
import './dialog.css'
5
import { newDialog, closeDialog, DialogOptions } from '@hfs/shared/dialogs'
6
import _ from 'lodash'
7
+import { useInterval } from 'usehooks-ts'
8
export * from '@hfs/shared/dialogs'
9
10
interface PromptOptions extends Partial<DialogOptions> { def?:string, type?:string }
@@ -72,8 +73,8 @@ export async function alertDialog(msg: ReactElement | string | Error, type:Alert
73
}
74
}
75
75
-export interface ConfirmOptions { href?: string, afterButtons?: ReactNode }
76
-export async function confirmDialog(msg: ReactElement | string, { href, afterButtons }: ConfirmOptions={}) : Promise<boolean> {
76
+export interface ConfirmOptions { href?: string, afterButtons?: ReactNode, timeout?: number, timeoutConfirm?: boolean }
77
+export async function confirmDialog(msg: ReactElement | string, { href, afterButtons, timeout, timeoutConfirm=false }: ConfirmOptions={}) : Promise<boolean> {
78
if (typeof msg === 'string')
79
msg = h('p', {}, msg)
80
return new Promise(resolve => newDialog({
@@ -84,6 +85,13 @@ export async function confirmDialog(msg: ReactElement | string, { href, afterBut
85
}) )
86
87
function Content() {
88
+ const [sec,setSec] = useState(Math.ceil(timeout||0))
89
+ useInterval(() => setSec(x => Math.max(0, x-1)), 1000)
90
+ const missingText = timeout!>0 && ` (${sec})`
91
+ useEffect(() => {
92
+ if (timeout && !sec)
93
+ closeDialog(timeoutConfirm)
94
+ }, [sec])
95
return h('div', {},
96
msg,
97
h('div', {
@@ -97,10 +105,10 @@ export async function confirmDialog(msg: ReactElement | string, { href, afterBut
105
h('a', {
106
href,
107
onClick() { closeDialog(true) },
100
- }, h('button', {}, "Confirm")),
108
+ }, h('button', {}, "Confirm", timeoutConfirm && missingText)),
109
h('button', {
110
onClick() { closeDialog(false) },
103
- }, "Don't"),
111
+ }, "Don't", !timeoutConfirm && missingText),
112
afterButtons,
113
)
114
)
frontend/src/upload.ts
+43
-18
@@ -5,7 +5,7 @@ import { Flex, FlexV } from './components'
5
import { formatBytes, hIcon, newDialog, prefix } from './misc'
6
import _ from 'lodash'
7
import { proxy, ref, subscribe, useSnapshot } from 'valtio'
8
-import { alertDialog } from './dialog'
8
+import { alertDialog, confirmDialog } from './dialog'
9
import { reloadList } from './useFetchList'
10
import { getNotification } from './api'
11
@@ -143,7 +143,7 @@ function FilesList({ files, remove }: { files: File[], remove: (f:File) => any }
143
h('td', {}, formatBytes(f.size)),
144
h('td', { className: working ? 'ani-working' : undefined },
145
path(f),
146
- working && h('span', { className: 'upload-progress' }, progress.toFixed(1), '%')
146
+ working && h('span', { className: 'upload-progress' }, formatPerc(progress))
147
),
148
)
149
})
@@ -155,6 +155,10 @@ function iconBtn(icon: string, onClick: () => any, { small=true, style={}, ...pr
155
return h('button', { onClick, ...props, ...small && { style: { padding: '.1em', ...style } } }, hIcon(icon))
156
}
157
158
+function formatPerc(p: number) {
159
+ return (p*100).toFixed(1) + '%'
160
+}
161
+
162
/// Manage upload queue
163
164
subscribe(uploadState, () => {
@@ -173,20 +177,11 @@ let overrideStatus = 0
177
let notificationChannel = ''
178
let notificationSource: EventSource
179
176
-async function startUpload(f: File, to: string) {
177
- if (!notificationChannel) {
178
- notificationChannel = 'upload-' + Math.random().toString(36).slice(2)
179
- notificationSource = await getNotification(notificationChannel, (name, data) => {
180
- if (name !== 'upload.status') return
181
- const {uploading} = uploadState
182
- if (!uploading) return
183
- overrideStatus = data?.[path(uploading)]
184
- if (overrideStatus >= 400)
185
- abortCurrentUpload()
186
- })
187
- }
180
+async function startUpload(f: File, to: string, resume=0) {
181
+ let resuming = false
182
overrideStatus = 0
183
uploadState.uploading = f
184
+ await subscribeNotifications()
185
req = new XMLHttpRequest()
186
req.onloadend = () => {
187
if (req?.readyState !== 4) return
@@ -196,14 +191,44 @@ async function startUpload(f: File, to: string) {
191
error()
192
else
193
done()
199
- next()
194
+ if (!resuming)
195
+ next()
196
}
201
- req.upload.onprogress = (e:any) => uploadState.progress = e.loaded / e.total * 100
202
- req.open("POST", to + '?notificationChannel=' + notificationChannel, true)
197
+ req.upload.onprogress = (e:any) =>
198
+ uploadState.progress = (e.loaded + resume) / (e.total + resume)
199
+ req.open('POST', to + '?' + new URLSearchParams({ notificationChannel, resume: String(resume) }), true)
200
const form = new FormData()
204
- form.append('file', f, path(f))
201
+ form.append('file', f.slice(resume), path(f))
202
req.send(form)
203
204
+ async function subscribeNotifications() {
205
+ if (!notificationChannel) {
206
+ notificationChannel = 'upload-' + Math.random().toString(36).slice(2)
207
+ notificationSource = await getNotification(notificationChannel, async (name, data) => {
208
+ const {uploading} = uploadState
209
+ if (!uploading) return
210
+ if (name === 'upload.resumable') {
211
+ const size = data?.[path(uploading)]
212
+ if (!size || size > f.size) return
213
+ const {expires} = data
214
+ const timeout = typeof expires !== 'number' ? 0
215
+ : (Number(new Date(expires)) - Date.now()) / 1000
216
+ if (!await confirmDialog(`Resume upload? (${formatPerc(size/f.size)} = ${formatBytes(size)})`, { timeout })) return
217
+ if (uploading !== uploadState.uploading) return // too late
218
+ resuming = true
219
+ abortCurrentUpload()
220
+ return startUpload(f, to, size)
221
+ }
222
+ if (name === 'upload.status') {
223
+ overrideStatus = data?.[path(uploading)]
224
+ if (overrideStatus >= 400)
225
+ abortCurrentUpload()
226
+ return
227
+ }
228
+ })
229
+ }
230
+ }
231
+
232
function error() {
233
if (!uploadState.errors++)
234
alertDialog("Upload error", 'error').then()
src/middlewares.ts
+54
-22
@@ -8,13 +8,11 @@ import {
8
BUILD_TIMESTAMP,
9
DEV,
10
SESSION_DURATION,
11
- HTTP_FORBIDDEN,
12
- HTTP_UNAUTHORIZED,
13
- HTTP_NOT_FOUND,
11
+ HTTP_FORBIDDEN, HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_RANGE_NOT_SATISFIABLE, HTTP_SERVER_ERROR,
12
} from './const'
13
import { FRONTEND_URI } from './const'
14
import { cantReadStatusCode, hasPermission, nodeIsDirectory, urlToNode, vfs, VfsNode } from './vfs'
17
-import { dirTraversal } from './misc'
15
+import { Callback, dirTraversal, try_ } from './misc'
16
import { zipStreamFromFolder } from './zip'
17
import { serveFileNode } from './serveFile'
18
import { serveGuiFiles } from './serveGuiFiles'
@@ -27,7 +25,7 @@ import basicAuth from 'basic-auth'
25
import { SRPClientSession, SRPParameters, SRPRoutines } from 'tssrp6a'
26
import { srpStep1 } from './api.auth'
27
import { basename, dirname, join } from 'path'
30
-import { createWriteStream, mkdirSync, rename, rm } from 'fs'
28
+import fs from 'fs'
29
import { pipeline } from 'stream/promises'
30
import formidable from 'formidable'
31
import { notifyClient } from './frontEndApis'
@@ -135,32 +133,66 @@ export const serveGuiAndSharedFiles: Koa.Middleware = async (ctx, next) => {
133
return serveFrontendFiles(ctx, next)
134
135
function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
138
- if (!base.source || !hasPermission(base, 'can_upload', ctx)) {
139
- ctx.status = base.can_upload === false ? HTTP_FORBIDDEN : HTTP_UNAUTHORIZED
140
- notifyClient(ctx, 'upload.status', { [path]: ctx.status }) // allow browsers to detect failure while still sending body
141
- return
142
- }
136
+ if (!base.source || !hasPermission(base, 'can_upload', ctx))
137
+ return fail(base.can_upload === false ? HTTP_FORBIDDEN : HTTP_UNAUTHORIZED)
138
const fullPath = join(base.source, path)
139
const dir = dirname(fullPath)
145
- mkdirSync(dir, { recursive: true })
146
- const tempName = join(dir, 'hfs$uploading-' + basename(fullPath).slice(-20))
147
- const ret = createWriteStream(tempName)
148
- clearTimeout(waitingToBeDeleted[tempName])
149
- delete waitingToBeDeleted[tempName]
140
+ fs.mkdirSync(dir, { recursive: true })
141
+ const keepName = basename(fullPath).slice(-200)
142
+ let tempName = join(dir, 'hfs$upload-' + keepName)
143
+ const resumable = fs.existsSync(tempName) && tempName
144
+ if (resumable)
145
+ tempName = join(dir, 'hfs$upload2-' + keepName)
146
+ const resume = Number(ctx.query.resume)
147
+ const size = resumable && try_(() => fs.statSync(resumable).size)
148
+ if (size === undefined) // stat failed
149
+ return fail(HTTP_SERVER_ERROR)
150
+ if (resume > size)
151
+ return fail(HTTP_RANGE_NOT_SATISFIABLE)
152
+ if (!resume && resumable) {
153
+ const timeout = 30
154
+ notifyClient(ctx, 'upload.resumable', { [path]: size, expires: Date.now() + timeout * 1000 })
155
+ delayedDelete(resumable, timeout, () =>
156
+ fs.rename(tempName, resumable, err => {
157
+ if (!err)
158
+ tempName = resumable
159
+ }) )
160
+ }
161
+ const resuming = resume && resumable
162
+ const ret = resuming ? fs.createWriteStream(resumable, { flags: 'r+', start: resume })
163
+ : fs.createWriteStream(tempName)
164
+ if (resuming) {
165
+ fs.rm(tempName, () => {})
166
+ tempName = resumable
167
+ }
168
+ cancelDeletion(tempName)
169
ret.on('close', () => {
170
if (!ctx.req.aborted)
152
- return rename(tempName, fullPath, err =>
171
+ return fs.rename(tempName, fullPath, err =>
172
err && console.error("couldn't rename temp to", fullPath, String(err)))
173
const sec = deleteUnfinishedUploadsAfter.get()
174
if (typeof sec !== 'number') return
156
- waitingToBeDeleted[tempName] = setTimeout(deleteNow, sec * 1000)
157
-
158
- function deleteNow() {
159
- delete waitingToBeDeleted[tempName]
160
- rm(tempName, () => {})
161
- }
175
+ delayedDelete(tempName, sec)
176
})
177
return ret
178
+
179
+ function delayedDelete(path: string, secs: number, cb?: Callback) {
180
+ clearTimeout(waitingToBeDeleted[path])
181
+ waitingToBeDeleted[path] = setTimeout(() => {
182
+ delete waitingToBeDeleted[path]
183
+ fs.rm(path, () => cb?.())
184
+ }, secs * 1000)
185
+ }
186
+
187
+ function cancelDeletion(path: string) {
188
+ clearTimeout(waitingToBeDeleted[path])
189
+ delete waitingToBeDeleted[path]
190
+ }
191
+
192
+ function fail(status: number) {
193
+ ctx.status = status
194
+ notifyClient(ctx, 'upload.status', { [path]: ctx.status }) // allow browsers to detect failure while still sending body
195
+ }
196
}
197
198
}
src/misc.ts
+9
@@ -165,3 +165,12 @@ export async function stream2string(stream: Readable): Promise<string> {
165
})
166
})
167
}
168
+
169
+export function try_(cb: () => any, onException?: (e:any) => any) {
170
+ try {
171
+ return cb()
172
+ }
173
+ catch(e) {
174
+ return onException?.(e)
175
+ }
176
+}
\ No newline at end of file