| 1 | import { |
| 2 | HTTP_CONFLICT, HTTP_MESSAGES, HTTP_PAYLOAD_TOO_LARGE, HTTP_RANGE_NOT_SATISFIABLE, HTTP_INSUFFICIENT_STORAGE, |
| 3 | HTTP_PRECONDITION_FAILED, UPLOAD_TEMP_HASH, MTIME_CHECK, |
| 4 | buildUrlQueryString, getHFS, pathEncode, pendingPromise, prefix, randomId, tryJson, wait, with_, |
| 5 | } from '@hfs/shared' |
| 6 | import { state } from './state' |
| 7 | import { alertDialog, toast } from './dialog' |
| 8 | import { reloadList } from './useFetchList' |
| 9 | import { proxy, ref, snapshot, subscribe } from 'valtio' |
| 10 | import { createElement as h } from 'react' |
| 11 | import _ from 'lodash' |
| 12 | import { UploadStatus } from './upload' |
| 13 | import i18n from './i18n' |
| 14 | const { t } = i18n |
| 15 | |
| 16 | export interface ToUpload { file: File, comment?: string, path: string, to?: string, error?: string } |
| 17 | export const uploadState = proxy<{ |
| 18 | done: (ToUpload & { response?: any })[] // res will contain the response from the server, |
| 19 | doneByte: number |
| 20 | errors: ToUpload[] |
| 21 | interrupted: ToUpload[] |
| 22 | adding: ToUpload[] |
| 23 | qs: { to: string, entries: ToUpload[] }[] |
| 24 | paused: boolean |
| 25 | uploading?: ToUpload |
| 26 | hashing?: number |
| 27 | progress: number // percentage |
| 28 | partial: number // relative to uploading file. This is how much we have done of the current queue. |
| 29 | speed: number |
| 30 | eta: number |
| 31 | uploadDialogIsOpen: boolean |
| 32 | }>({ |
| 33 | uploadDialogIsOpen: false, |
| 34 | eta: 0, |
| 35 | speed: 0, |
| 36 | partial: 0, |
| 37 | progress: 0, |
| 38 | paused: false, |
| 39 | qs: [], |
| 40 | adding: [], |
| 41 | interrupted: [], |
| 42 | errors: [], |
| 43 | doneByte: 0, |
| 44 | done: [], |
| 45 | }) |
| 46 | |
| 47 | window.onbeforeunload = ev => { |
| 48 | if (!uploadState.qs.length) return |
| 49 | ev.preventDefault() |
| 50 | return ev.returnValue = t("Uploading") // modern browsers ignore this message |
| 51 | } |
| 52 | |
| 53 | let stuckSince = Infinity |
| 54 | // keep track of speed |
| 55 | let bytesSentTimestamp = Date.now() |
| 56 | let bytesSent = 0 |
| 57 | const recentSpeedSamples: number[] = [] |
| 58 | setInterval(() => { |
| 59 | const now = Date.now() |
| 60 | const passed = (now - bytesSentTimestamp) / 1000 |
| 61 | const speed = bytesSent / passed |
| 62 | if (currentReq && now - stuckSince >= 10_000) { // this will normally cause the upload to be retried after long time of no progress |
| 63 | currentReq.abort() |
| 64 | console.debug('upload stuck, aborting') |
| 65 | } |
| 66 | bytesSent = 0 // reset counter |
| 67 | bytesSentTimestamp = now |
| 68 | |
| 69 | // keep track of ETA |
| 70 | const qBytes = _.sumBy(uploadState.qs, q => _.sumBy(q.entries, x => x.file.size)) |
| 71 | const left = (qBytes - uploadState.partial) |
| 72 | const doSample = uploadState.uploading |
| 73 | if (doSample) |
| 74 | recentSpeedSamples.push(speed) |
| 75 | if (!doSample || recentSpeedSamples.length > 5) // max 5 samples, 10 seconds |
| 76 | recentSpeedSamples.shift() |
| 77 | if (uploadState.paused) |
| 78 | recentSpeedSamples.length = 0 |
| 79 | uploadState.speed = _.mean(recentSpeedSamples) |
| 80 | uploadState.eta = left / uploadState.speed || 0 |
| 81 | }, 2_000) |
| 82 | |
| 83 | let currentReq: XMLHttpRequest | undefined |
| 84 | const id = randomId() |
| 85 | let userAborted = false |
| 86 | let closeLastDialog: undefined | (() => void) |
| 87 | |
| 88 | let reloadOnClose = false |
| 89 | export function resetReloadOnClose() { |
| 90 | if (!reloadOnClose) return |
| 91 | reloadOnClose = false |
| 92 | return true |
| 93 | } |
| 94 | |
| 95 | export async function startUpload(toUpload: ToUpload, to: string, resume=0) { |
| 96 | console.debug('start upload', toUpload.path, resume) |
| 97 | uploadState.uploading = toUpload |
| 98 | uploadState.progress = 0 |
| 99 | userAborted = false |
| 100 | let strictResume = true // ask to reject our request if a better resume is available |
| 101 | const splitSize = getHFS().splitUploads |
| 102 | const fullSize = toUpload.file.size |
| 103 | const uriPath = to + pathEncode(toUpload.path) |
| 104 | let stopLooping = false // allow callbacks to stop the loop |
| 105 | do { // at least one iteration, even for empty files |
| 106 | let req = currentReq = new XMLHttpRequest() |
| 107 | const requestIsOver = pendingPromise() |
| 108 | stuckSince = Date.now() |
| 109 | // beware of 'abort' event: it isn't triggered if connection isn't established yet |
| 110 | req.onloadend = async () => { // loadend = fired for both success and error. Safari doesn't always fire this on disconnections, leaving readyState = 3. The problem is mitigated by the abort-when-stuck mechanism above. |
| 111 | try { |
| 112 | currentReq = undefined |
| 113 | if (uploadState.paused) |
| 114 | return stopLooping = true |
| 115 | strictResume = true // reset at each request |
| 116 | if (!userAborted && !req.status) { // we were disconnected, possibly with a status that we couldn't read, so we give it another chance without the body |
| 117 | /* Browsers are unreliable when it comes to read the status before the request is fully sent. |
| 118 | - chrome139 works for most of the cases. It seems it doesn't when the disconnection happens a bit late (like for file system errors). |
| 119 | - safari18 is inconsistent and it seems random. |
| 120 | - firefox141 basically never works. |
| 121 | */ |
| 122 | req = new XMLHttpRequest() |
| 123 | req.open('PUT', uriPath + queryString + '&simulating=' + body.size, false) // not async this time |
| 124 | try { req.send() } |
| 125 | catch(e) { console.log(e) } |
| 126 | await wait(500) // on a fast connection (localhost) firefox is aborting next request (without the delay), reporting NS_BINDING_ABORTED. Still don't know why |
| 127 | } |
| 128 | const { status } = req |
| 129 | if (status === HTTP_RANGE_NOT_SATISFIABLE) |
| 130 | return stopLooping = true |
| 131 | if (status === HTTP_PRECONDITION_FAILED) { // resume available |
| 132 | const size = Number(req.getResponseHeader('x-size')) |
| 133 | if (req.getResponseHeader(MTIME_CHECK)) { // the only value for this header is not-available, so we fallback to the hash check |
| 134 | const hashFromServer = fetch(uriPath + '?get=' + UPLOAD_TEMP_HASH).then(r => r.text()) |
| 135 | const hashed = await calcHash(toUpload.file, size) // therefore, we attempt a check using the hash |
| 136 | if (hashed !== await hashFromServer) { |
| 137 | strictResume = false |
| 138 | return console.debug('upload hash mismatch') |
| 139 | } |
| 140 | console.debug('upload hash is matching') |
| 141 | } |
| 142 | resume = size |
| 143 | return console.debug('resuming upload', size.toLocaleString()) |
| 144 | } |
| 145 | if (userAborted || status === HTTP_CONFLICT) { // HTTP_CONFLICT = skipped because existing, or upload in progress |
| 146 | if (req.responseText === 'retry') // it's our previous request that didn't release the lock yet |
| 147 | return await wait(2000) // wait before resolving `finished` |
| 148 | toUpload.error = status ? t('upload_conflict', "already exists") : t`Interrupted` // the I is uppercase because we are just recycling an old string (with all its translations) |
| 149 | uploadState.interrupted.push(toUpload) |
| 150 | } |
| 151 | else if (status >= 400) |
| 152 | error(status) |
| 153 | else if (!status) // request failed at a network level, so try again same file (return), but not too often (wait) |
| 154 | return await wait(2000) // wait before resolving `finished` |
| 155 | else { |
| 156 | if (splitSize) { |
| 157 | resume += splitSize |
| 158 | if (resume < fullSize) return // go on with the next chunk |
| 159 | } |
| 160 | uploadState.done.push({ ...toUpload, response: tryJson(req.responseText) }) |
| 161 | uploadState.doneByte += toUpload!.file.size |
| 162 | reloadOnClose = true |
| 163 | } |
| 164 | stopLooping = true |
| 165 | requestIsOver.then(workNextFile) |
| 166 | } |
| 167 | finally { |
| 168 | requestIsOver.resolve() |
| 169 | } |
| 170 | } |
| 171 | let lastProgress = 0 |
| 172 | req.upload.onprogress = (e:any) => { |
| 173 | uploadState.partial = e.loaded + resume |
| 174 | uploadState.progress = uploadState.partial / fullSize |
| 175 | bytesSent += e.loaded - lastProgress |
| 176 | if (e.loaded > lastProgress) // some progress = not stuck |
| 177 | stuckSince = Date.now() |
| 178 | lastProgress = e.loaded |
| 179 | } |
| 180 | const partial = splitSize && resume + splitSize < fullSize |
| 181 | const queryString = buildUrlQueryString({ |
| 182 | id, |
| 183 | mtime: toUpload.file.lastModified, |
| 184 | resume: resume + (strictResume ? '!' : ''), |
| 185 | partial: partial ? fullSize - resume : undefined, // how much space we need |
| 186 | comment: toUpload.comment || undefined, |
| 187 | existing: with_(state.uploadOnExisting, x => x !== 'rename' ? x : undefined), // rename is the default |
| 188 | }) |
| 189 | req.open('PUT', uriPath + queryString, true) |
| 190 | const body = toUpload.file.slice(resume, splitSize ? resume + splitSize : undefined) |
| 191 | req.send(body) |
| 192 | await requestIsOver |
| 193 | } while (!stopLooping) |
| 194 | |
| 195 | function error(status: number) { |
| 196 | const ERRORS = { |
| 197 | [HTTP_PAYLOAD_TOO_LARGE]: t`file too large` + (getHFS().proxyDetected ? '\n– ' + t('proxy_413', "Check for this limit on the proxy server") : ''), |
| 198 | [HTTP_CONFLICT]: t('upload_conflict', "already exists"), |
| 199 | [HTTP_INSUFFICIENT_STORAGE]: t`insufficient storage`, |
| 200 | } |
| 201 | const specifier = (ERRORS as any)[status] || HTTP_MESSAGES[status] || status |
| 202 | toUpload.error = specifier |
| 203 | if (uploadState.errors.push(toUpload) > 1) return |
| 204 | const msg = t('failed_upload', { name: toUpload.path }, "Couldn't upload {name}") + prefix(': ', specifier) |
| 205 | closeLastDialog?.() |
| 206 | closeLastDialog = alertDialog(msg, 'error')?.close |
| 207 | } |
| 208 | |
| 209 | function workNextFile() { |
| 210 | uploadState.uploading = undefined |
| 211 | uploadState.partial = 0 |
| 212 | const { qs } = uploadState |
| 213 | if (!qs.length) return |
| 214 | qs[0].entries.shift() |
| 215 | if (!qs[0].entries.length) |
| 216 | qs.shift() |
| 217 | if (qs.length) return |
| 218 | setTimeout(reloadList, 500) // workaround: reloading too quickly can meet the new file still with its temp name |
| 219 | reloadOnClose = false |
| 220 | if (uploadState.uploadDialogIsOpen) return |
| 221 | // freeze and reset |
| 222 | const snap = snapshot(uploadState) |
| 223 | resetCounters() |
| 224 | const msg = h('div', {}, t(['upload_concluded', "Upload terminated"], "Upload concluded:"), |
| 225 | h(UploadStatus, { snapshot: snap, display: 'flex', flexDirection: 'column' }) ) |
| 226 | if (snap.errors.length || snap.interrupted.length) |
| 227 | alertDialog(msg, 'warning') |
| 228 | else |
| 229 | toast(msg, 'success') |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | export function abortCurrentUpload(userAskedForIt=false) { |
| 234 | userAborted = userAskedForIt |
| 235 | currentReq?.abort() |
| 236 | } |
| 237 | subscribe(uploadState, () => { |
| 238 | const [cur] = uploadState.qs |
| 239 | if (cur?.entries.length && !uploadState.uploading && !uploadState.paused) |
| 240 | void startUpload(cur.entries[0], cur.to) |
| 241 | }) |
| 242 | |
| 243 | export async function enqueueUpload(entries: ToUpload[], to=location.pathname) { |
| 244 | if (_.remove(entries, x => !simulateBrowserAccept(x.file)).length) |
| 245 | await alertDialog(t('upload_file_rejected', "Some files were not accepted"), 'warning') |
| 246 | |
| 247 | entries = _.uniqBy(entries, x => x.path) |
| 248 | if (!entries.length) return |
| 249 | entries = entries.map(x => ({ ...x, file: ref(x.file) })) // avoid valtio to mess with File object |
| 250 | const q = _.find(uploadState.qs, { to }) |
| 251 | if (!q) |
| 252 | return uploadState.qs.push({ to, entries }) |
| 253 | const missing = _.differenceBy(entries, q.entries, x => x.path) |
| 254 | q.entries.push(...missing.map(ref)) |
| 255 | } |
| 256 | |
| 257 | export function simulateBrowserAccept(f: File) { |
| 258 | const { props } = state |
| 259 | if (!props?.accept) return true |
| 260 | return normalizeAccept(props?.accept)!.split(/ *[|,] */).some(pattern => |
| 261 | pattern.startsWith('.') ? f.name.endsWith(pattern) |
| 262 | : f.type.match(pattern.replace('.','\\.').replace('*', '.*')) // '.' for .ext and '*' for 'image/*' |
| 263 | ) |
| 264 | } |
| 265 | |
| 266 | export function normalizeAccept(accept?: string) { |
| 267 | return accept?.replace(/\|/g, ',').replace(/ +/g, '') |
| 268 | } |
| 269 | |
| 270 | export function getFilePath(f: File) { |
| 271 | return (f.webkitRelativePath || f.name).replaceAll('//','/') |
| 272 | } |
| 273 | |
| 274 | export function resetCounters() { |
| 275 | Object.assign(uploadState, { |
| 276 | errors: [], |
| 277 | done: [], |
| 278 | doneByte: 0, |
| 279 | interrupted: [], |
| 280 | }) |
| 281 | } |
| 282 | |
| 283 | async function calcHash(file: File, limit=Infinity) { |
| 284 | const hash = await hasher() |
| 285 | const t = Date.now() |
| 286 | const reader = file.stream().getReader() |
| 287 | let left = limit |
| 288 | const updateUI = _.debounce(() => uploadState.hashing = (limit - left) / limit, 100, { maxWait: 500 }) |
| 289 | try { |
| 290 | while (left > 0) { |
| 291 | const res = await reader.read() |
| 292 | if (res.done) break |
| 293 | const chunk = res.value.slice(0, left) |
| 294 | hash.update(chunk.buffer) |
| 295 | left -= chunk.length |
| 296 | updateUI() |
| 297 | await wait(1) // cooperative: without this, the browser may freeze |
| 298 | } |
| 299 | } |
| 300 | finally { |
| 301 | updateUI.flush() |
| 302 | uploadState.hashing = undefined |
| 303 | } |
| 304 | const ret = hash.digest().toString(16) |
| 305 | console.debug('hash calculated in', Date.now() - t, 'ms', ret) |
| 306 | return ret |
| 307 | |
| 308 | async function hasher() { |
| 309 | /* using this lib because it's much faster, works on legacy browsers, and we don't need it to be cryptographic. Sure 32bit isn't much. |
| 310 | Benchmark on 2GB: |
| 311 | 18.5s aws-crypto/sha256-browser |
| 312 | 43.6s js-sha512 |
| 313 | 73.3s sha512@hash-wasm |
| 314 | 8.2s xxhash-wasm/64 |
| 315 | 8.2s xxhash-wasm/32 |
| 316 | 41s xxhashjs/64 |
| 317 | 9.1s xxhashjs/32 |
| 318 | */ |
| 319 | //if (BigInt !== Number && BigInt) return (await (await import('xxhash-wasm')).default()).create64() // at 32bit, a 9% difference is not worth having 2 libs, but 64bit is terrible without wasm |
| 320 | const ret = (await import('xxhashjs')).h32() |
| 321 | const original = ret.update |
| 322 | ret.update = (x: Buffer) => original.call(ret, x) // xxhashjs only works with ArrayBuffer, not UInt8Array |
| 323 | return ret |
| 324 | } |
| 325 | } |
| 326 |