try to estimate zip size so that download progress can be tracked by the client/browser
Massimo Melina committed
Jan 15, 2022 at 12:29 UTC
0f451905e25c1d735dbe4cc141f1d7a5dad8ae3a
4 files changed
+47
-11
config.yaml
+1
@@ -3,6 +3,7 @@
3
#max_kbps_per_ip: 500
4
#log:
5
#error_log:
6
+#zip-calculate-size-for-seconds: 1
7
mime:
8
"*.jpg|*.png|*.mp3|*.txt": auto
9
disable_plugins: [ 'theme-example', 'middleware-example', 'download-counter' ]
src/QuickZipStream.ts
+35
-5
@@ -21,6 +21,7 @@ export class QuickZipStream extends Readable {
21
private finished = false
22
private readonly centralDir: ({ size:number, crc:number, ts:Date, pathAsBuffer:Buffer, offset:number, version:number })[] = []
23
private dataWritten = 0
24
+ private prewalk?: ZipSource[]
25
26
constructor(private readonly walker: AsyncIterableIterator<ZipSource>) {
27
super({})
@@ -33,13 +34,41 @@ export class QuickZipStream extends Readable {
34
this.dataWritten += chunk.length
35
}
36
37
+ async calculateSize(howLong:number = 1000) {
38
+ this.prewalk = []
39
+ let timeIsOut = false
40
+ setTimeout(() => timeIsOut = true, howLong) // give it 1 second
41
+ while (!timeIsOut) {
42
+ const { value } = await this.walker.next()
43
+ if (!value) break
44
+ this.prewalk.push(value) // we keep same shape of the generator, so
45
+ }
46
+ if (timeIsOut)
47
+ return NaN
48
+ let size = 0
49
+ let centralDirSize = 0
50
+ for (const file of this.prewalk) {
51
+ const pathSize = Buffer.from(file.path, 'utf8').length
52
+ const sizeSize = size > ZIP64_LIMIT ? 8 : 4
53
+ const extraLength = (file.size > ZIP64_LIMIT ? 2 : 0) + (size > ZIP64_LIMIT ? 1 : 0)
54
+ const extraDataSize = extraLength && (2+2 + extraLength*8)
55
+ size += 4+2+2+4+4+4+4+2+2+ pathSize + file.size +4+4 +sizeSize*2
56
+ centralDirSize += 4+2+2+2+4+4+4+4+2+2+2+2+2+4+4 + pathSize + extraDataSize
57
+ }
58
+ const centralOffset = size
59
+ if (centralOffset > ZIP64_LIMIT)
60
+ centralDirSize += 4+8+2+2+4+4+8+8+8+8+4+4+8+4
61
+ centralDirSize += 4+4+2+2+4+4+2
62
+ return size + centralDirSize
63
+ }
64
+
65
async _read(): Promise<void> {
37
- if (this.workingFile || this.finished) return
38
- const { value } = await this.walker.next() as { value: ZipSource }
39
- if (!value)
66
+ if (this.workingFile || this.finished || this.destroyed) return
67
+ const file = this.prewalk?.shift() || (await this.walker.next()).value as ZipSource
68
+ if (!file)
69
return this.closeArchive()
70
++this.numberOfFiles
42
- let { path, data, size, ts } = value
71
+ let { path, data, size, ts } = file
72
const pathAsBuffer = Buffer.from(path, 'utf8')
73
const crc32 = await crc32provider
74
let crc: number | undefined = undefined
@@ -62,6 +91,8 @@ export class QuickZipStream extends Readable {
91
let total = 0
92
data.on('error', (err) => console.error(err))
93
data.on('data', chunk => {
94
+ if (this.destroyed)
95
+ return data.destroy()
96
this._push(chunk)
97
crc = crc32(chunk, crc)
98
total += chunk.length
@@ -69,7 +100,6 @@ export class QuickZipStream extends Readable {
100
this.workingFile = true
101
data.on('end', ()=>{
102
this.workingFile = false
72
- console.debug('zip', total, path)
103
this.centralDir.push({ size, crc:crc!, pathAsBuffer, ts, offset, version })
104
const sizeSize = size > ZIP64_LIMIT ? 8 : 4
105
this._push([
src/middlewares.ts
+5
-5
@@ -51,13 +51,13 @@ export const frontendAndSharedFiles: Koa.Middleware = async (ctx, next) => {
51
if (path.includes('..'))
52
ctx.throw(500)
53
if (ctx.body)
54
- return await next()
54
+ return next()
55
if (path.startsWith(FRONTEND_URI))
56
- return await serveFrontendPrefixed(ctx,next)
56
+ return serveFrontendPrefixed(ctx,next)
57
const decoded = decodeURI(path)
58
const node = await vfs.urlToNode(decoded, ctx)
59
if (!node)
60
- return await next()
60
+ return next()
61
const { source } = node
62
if (!source || await isDirectory(source)) {
63
const { get } = ctx.query
@@ -71,10 +71,10 @@ export const frontendAndSharedFiles: Koa.Middleware = async (ctx, next) => {
71
return serveFileNode(def)(ctx, next)
72
}
73
ctx.set({ server:'HFS '+BUILD_TIMESTAMP })
74
- return await serveFrontend(ctx, next)
74
+ return serveFrontend(ctx, next)
75
}
76
if (source)
77
return source.includes('//') ? mount(path,proxy(source,{}))(ctx,next)
78
: serveFileNode(node)(ctx,next)
79
- await next()
79
+ return next()
80
}
src/zip.ts
+6
-1
@@ -4,6 +4,7 @@ import { filterMapGenerator } from './misc'
4
import { QuickZipStream } from './QuickZipStream'
5
import { createReadStream } from 'fs'
6
import fs from 'fs/promises'
7
+import { getConfig } from './config'
8
9
export async function zipStreamFromFolder(node: VfsNode, ctx: Koa.Context) {
10
ctx.status = 200
@@ -22,5 +23,9 @@ export async function zipStreamFromFolder(node: VfsNode, ctx: Koa.Context) {
23
catch(e) {}
24
25
})
25
- ctx.body = new QuickZipStream(walker)
26
+ const zip = new QuickZipStream(walker)
27
+ const time = 1000 * (getConfig('zip-calculate-size-for-seconds') ?? 1)
28
+ ctx.response.length = await zip.calculateSize(time)
29
+ ctx.body = zip
30
+ ctx.req.on('close', ()=> zip.destroy())
31
}