better code
Massimo Melina committed
Aug 8, 2022 at 16:20 UTC
9e685780a129a02a14b08e456faa1b74c64d84ae
4 files changed
+40
-37
server/src/QuickZipStream.ts
+37
-30
@@ -5,7 +5,8 @@ import { Readable } from 'stream'
5
import { crc32 as crc32lib } from 'buffer-crc32'
6
import assert from 'assert'
7
8
-const ZIP64_LIMIT = 2**31 -1
8
+const ZIP64_SIZE_LIMIT = 0xffffffff
9
+const ZIP64_NUMBER_LIMIT = 0xffff
10
11
let crc32function: (input: string | Buffer, initialState?: number | undefined | null) => number
12
import('@node-rs/crc32').then(lib => crc32function = lib.crc32, () => {
@@ -25,9 +26,8 @@ interface ZipSource {
26
}
27
export class QuickZipStream extends Readable {
28
private workingFile: Readable | undefined
28
- private numberOfFiles: number = 0
29
private finished = false
30
- private readonly centralDir: ({ size:number, crc:number, ts:Date, pathAsBuffer:Buffer, offset:number, version:number, extAttr: number })[] = []
30
+ private readonly entries: ({ size:number, crc:number, ts:Date, pathAsBuffer:Buffer, offset:number, version:number, extAttr: number })[] = []
31
private dataWritten = 0
32
private consumedCalculating: ZipSource[] = []
33
private skip: number = 0
@@ -80,17 +80,21 @@ export class QuickZipStream extends Readable {
80
if (!value) break
81
this.consumedCalculating.push(value) // we keep same shape of the generator, so
82
}
83
+ // if we reach here, then we were able to consume all entries of the walker (in time)
84
let offset = 0
85
let centralDirSize = 0
86
for (const file of this.consumedCalculating) {
87
const pathSize = Buffer.from(file.path, 'utf8').length
87
- const extraLength = (file.size > ZIP64_LIMIT ? 2 : 0) + (offset > ZIP64_LIMIT ? 1 : 0)
88
+ const extraLength = (file.size > ZIP64_SIZE_LIMIT ? 2 : 0) + (offset > ZIP64_SIZE_LIMIT ? 1 : 0)
89
const extraDataSize = extraLength && (2+2 + extraLength*8)
90
offset += 4+2+2+2+ 4+4+4+4+ 2+2+ pathSize + file.size
91
centralDirSize += 4+2+2+2+2+ 4+4+4+4+ 2+2+2+2+2+ 4+4 + pathSize + extraDataSize
92
}
92
- const centralOffset = offset
93
- if (centralOffset > ZIP64_LIMIT)
93
+ const n = this.consumedCalculating.length
94
+ const centralDirOffset = offset
95
+ if (n >= ZIP64_NUMBER_LIMIT
96
+ || centralDirOffset >= ZIP64_SIZE_LIMIT
97
+ || centralDirSize >= ZIP64_SIZE_LIMIT)
98
centralDirSize += 4+8+2+2+4+4+8+8+8+8+4+4+8+4
99
centralDirSize += 4+4+2+2+4+4+2
100
return offset + centralDirSize
@@ -103,7 +107,6 @@ export class QuickZipStream extends Readable {
107
const file = this.consumedCalculating.shift() || (await this.walker.next()).value as ZipSource
108
if (!file)
109
return this.closeArchive()
106
- ++this.numberOfFiles
110
let { path, sourcePath, getData, size, ts, mode } = file
111
const pathAsBuffer = Buffer.from(path, 'utf8')
112
const offset = this.dataWritten
@@ -127,11 +130,11 @@ export class QuickZipStream extends Readable {
130
const cacheHit = Number(cache?.ts) === Number(ts)
131
let crc = cacheHit ? cache!.crc : crc32function('')
132
const extAttr = !mode ? 0 : (mode | 0x8000) * 0x10000 // it's like <<16 but doesn't overflow so easily
130
- const centralDirEntry = { size, crc, pathAsBuffer, ts, offset, version, extAttr }
133
+ const entry = { size, crc, pathAsBuffer, ts, offset, version, extAttr }
134
if (this.skip >= size && cacheHit) {
135
this.skip -= size
136
this.dataWritten += size
134
- this.centralDir.push(centralDirEntry)
137
+ this.entries.push(entry)
138
setTimeout(() => this.push('')) // this "signal" works only after _read() is done
139
return
140
}
@@ -139,10 +142,10 @@ export class QuickZipStream extends Readable {
142
data.on('error', (err) => console.error(err))
143
data.on('end', ()=>{
144
this.workingFile = undefined
142
- centralDirEntry.crc = crc
145
+ entry.crc = crc
146
if (sourcePath)
147
crcCache[sourcePath] = { ts, crc }
145
- this.centralDir.push(centralDirEntry)
148
+ this.entries.push(entry)
149
this.push('') // continue piping
150
})
151
this.workingFile = data
@@ -160,16 +163,16 @@ export class QuickZipStream extends Readable {
163
164
closeArchive() {
165
this.finished = true
163
- let centralOffset = this.dataWritten
164
- for (let { size, ts, crc, offset, pathAsBuffer, version, extAttr } of this.centralDir) {
166
+ let centralDirOffset = this.dataWritten
167
+ for (let { size, ts, crc, offset, pathAsBuffer, version, extAttr } of this.entries) {
168
const extra = []
166
- if (size > ZIP64_LIMIT) {
169
+ if (size > ZIP64_SIZE_LIMIT) {
170
extra.push(size, size)
168
- size = 0xffffffff
171
+ size = ZIP64_SIZE_LIMIT
172
}
170
- if (offset > ZIP64_LIMIT) {
173
+ if (offset > ZIP64_SIZE_LIMIT) {
174
extra.push(offset)
172
- offset = 0xffffffff
175
+ offset = ZIP64_SIZE_LIMIT
176
}
177
const extraData = buffer(!extra.length ? []
178
: [ 2,1, 2,8*extra.length, ...extra.map(x=> [8,x]).flat() ])
@@ -196,10 +199,12 @@ export class QuickZipStream extends Readable {
199
this.controlledPush(pathAsBuffer)
200
this.controlledPush(extraData)
201
}
199
- const n = this.centralDir.length
202
const after = this.dataWritten
201
- let centralSize = after-centralOffset
202
- if (centralOffset > ZIP64_LIMIT) {
203
+ let centralDirSize = after - centralDirOffset
204
+ let n = this.entries.length
205
+ if (n >= ZIP64_NUMBER_LIMIT
206
+ || centralDirOffset >= ZIP64_SIZE_LIMIT
207
+ || centralDirSize >= ZIP64_SIZE_LIMIT) {
208
this.controlledPush([
209
4, 0x06064b50, // end of central dir zip64
210
8, 44,
@@ -209,8 +214,8 @@ export class QuickZipStream extends Readable {
214
4, 0,
215
8, n,
216
8, n,
212
- 8, centralSize,
213
- 8, centralOffset,
217
+ 8, centralDirSize,
218
+ 8, centralDirOffset,
219
])
220
this.controlledPush([
221
4, 0x07064b50,
@@ -218,16 +223,18 @@ export class QuickZipStream extends Readable {
223
8, after,
224
4, 1,
225
])
221
- centralOffset = 0xFFFFFFFF
226
+ centralDirOffset = ZIP64_SIZE_LIMIT
227
+ centralDirSize = ZIP64_SIZE_LIMIT
228
+ n = ZIP64_NUMBER_LIMIT
229
}
230
this.controlledPush([
224
- 4,0x06054b50, // end of central directory signature
225
- 4,0, // disk-related stuff
226
- 2,this.numberOfFiles,
227
- 2,this.numberOfFiles,
228
- 4,centralSize,
229
- 4,centralOffset,
230
- 2,0, // comment length
231
+ 4, 0x06054b50, // end of central directory signature
232
+ 4, 0, // disk-related stuff
233
+ 2, n,
234
+ 2, n,
235
+ 4, centralDirSize,
236
+ 4, centralDirOffset,
237
+ 2, 0, // comment length
238
])
239
this.push(null) // EOF
240
}
server/src/serveFile.ts
-2
@@ -2,7 +2,6 @@
2
3
import Koa from 'koa'
4
import { createReadStream, stat } from 'fs'
5
-import fs from 'fs/promises'
5
import { FORBIDDEN, METHOD_NOT_ALLOWED, NO_CONTENT } from './const'
6
import { getNodeName, MIME_AUTO, VfsNode } from './vfs'
7
import mimetypes from 'mime-types'
@@ -11,7 +10,6 @@ import { isMatch } from 'micromatch'
10
import _ from 'lodash'
11
import path from 'path'
12
import { promisify } from 'util'
14
-import { updateConnection } from './connections'
13
14
const allowedReferer = defineConfig('allowed_referer', '')
15
server/src/vfs.ts
+3
-4
@@ -1,7 +1,7 @@
1
// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
import fs from 'fs/promises'
4
-import { basename } from 'path'
4
+import { basename, join } from 'path'
5
import { isMatch } from 'micromatch'
6
import { dirStream, dirTraversal, enforceFinal, getOrSet, isDirectory, typedKeys } from './misc'
7
import Koa from 'koa'
@@ -154,15 +154,14 @@ export async function* walkNode(parent:VfsNode, ctx: Koa.Context, depth:number=0
154
if (!source)
155
return
156
try {
157
- const base = enforceFinal('/', source)
158
- for await (const path of dirStream(base)) {
157
+ for await (const path of dirStream(source)) {
158
if (ctx.req.aborted)
159
return
160
let { rename } = parent
161
const renamed = rename?.[path]
162
yield* workItem({
163
name: (prefixPath || renamed) && prefixPath + (renamed || path),
165
- source: base + path,
164
+ source: join(source, path),
165
rename: renameUnderPath(rename, path),
166
})
167
}
server/src/zip.ts
-1
@@ -8,7 +8,6 @@ import { createReadStream } from 'fs'
8
import fs from 'fs/promises'
9
import { defineConfig } from './config'
10
import { dirname } from 'path'
11
-import { updateConnection } from './connections'
11
import { getRange } from './serveFile'
12
13
export async function zipStreamFromFolder(node: VfsNode, ctx: Koa.Context) {