fix: memory usage skyrocketing when zipping big files
Massimo Melina committed
Jun 26, 2022 at 11:43 UTC
b514c8334dcc0e2e46b439e4e17f27e6265bc098
1 file changed
+23
-17
server/src/QuickZipStream.ts
+23
-17
@@ -22,7 +22,7 @@ interface ZipSource {
22
mode?: number
23
}
24
export class QuickZipStream extends Readable {
25
- private workingFile = false
25
+ private workingFile: Readable | undefined
26
private numberOfFiles: number = 0
27
private finished = false
28
private readonly centralDir: ({ size:number, crc:number, ts:Date, pathAsBuffer:Buffer, offset:number, version:number, extAttr: number })[] = []
@@ -47,13 +47,15 @@ export class QuickZipStream extends Readable {
47
this.limit = end - start + 1
48
}
49
50
- _push(chunk: number[] | Buffer) {
50
+ controlledPush(chunk: number[] | Buffer) {
51
if (Array.isArray(chunk))
52
chunk = buffer(chunk)
53
this.dataWritten += chunk.length
54
if (this.skip) {
55
- if (this.skip >= chunk.length)
56
- return this.skip -= chunk.length
55
+ if (this.skip >= chunk.length) {
56
+ this.skip -= chunk.length
57
+ return true
58
+ }
59
chunk = chunk.subarray(this.skip)
60
this.skip = 0
61
}
@@ -61,9 +63,10 @@ export class QuickZipStream extends Readable {
63
if (lastBit)
64
chunk = chunk.subarray(0, this.limit)
65
64
- this.push(chunk)
66
+ const ret = this.push(chunk)
67
if (lastBit)
68
this.earlyClose()
69
+ return ret
70
}
71
72
async calculateSize(howLong:number = 1000) {
@@ -92,7 +95,9 @@ export class QuickZipStream extends Readable {
95
}
96
97
async _read() {
95
- if (this.workingFile || this.finished || this.destroyed) return
98
+ if (this.finished || this.destroyed) return
99
+ if (this.workingFile)
100
+ return this.workingFile.resume()
101
const file = this.consumedCalculating.shift() || (await this.walker.next()).value as ZipSource
102
if (!file)
103
return this.closeArchive()
@@ -101,7 +106,7 @@ export class QuickZipStream extends Readable {
106
const pathAsBuffer = Buffer.from(path, 'utf8')
107
const offset = this.dataWritten
108
let version = 20
104
- this._push([
109
+ this.controlledPush([
110
4, 0x04034b50,
111
2, version,
112
2, 0x08, // flags
@@ -113,7 +118,7 @@ export class QuickZipStream extends Readable {
118
2, pathAsBuffer.length,
119
2, 0, // extra length
120
])
116
- this._push(pathAsBuffer)
121
+ this.controlledPush(pathAsBuffer)
122
if (this.finished) return
123
124
const cache = sourcePath ? crcCache[sourcePath] : undefined
@@ -131,18 +136,19 @@ export class QuickZipStream extends Readable {
136
const data = getData()
137
data.on('error', (err) => console.error(err))
138
data.on('end', ()=>{
134
- this.workingFile = false
139
+ this.workingFile = undefined
140
centralDirEntry.crc = crc
141
if (sourcePath)
142
crcCache[sourcePath] = { ts, crc }
143
this.centralDir.push(centralDirEntry)
144
this.push('') // continue piping
145
})
141
- this.workingFile = true
146
+ this.workingFile = data
147
data.on('data', chunk => {
148
if (this.destroyed)
149
return data.destroy()
145
- this._push(chunk)
150
+ if (!this.controlledPush(chunk)) // destination buffer full
151
+ data.pause() // slow down
152
if (!cacheHit)
153
crc = crc32function(chunk, crc)
154
if (this.finished)
@@ -167,7 +173,7 @@ export class QuickZipStream extends Readable {
173
: [ 2,1, 2,8*extra.length, ...extra.map(x=> [8,x]).flat() ])
174
if (extraData.length && version < 45)
175
version = 45
170
- this._push([
176
+ this.controlledPush([
177
4, 0x02014b50, // central dir signature
178
2, version,
179
2, version,
@@ -185,14 +191,14 @@ export class QuickZipStream extends Readable {
191
4, extAttr,
192
4, offset,
193
])
188
- this._push(pathAsBuffer)
189
- this._push(extraData)
194
+ this.controlledPush(pathAsBuffer)
195
+ this.controlledPush(extraData)
196
}
197
const n = this.centralDir.length
198
const after = this.dataWritten
199
let centralSize = after-centralOffset
200
if (centralOffset > ZIP64_LIMIT) {
195
- this._push([
201
+ this.controlledPush([
202
4, 0x06064b50, // end of central dir zip64
203
8, 44,
204
2, 45,
@@ -204,7 +210,7 @@ export class QuickZipStream extends Readable {
210
8, centralSize,
211
8, centralOffset,
212
])
207
- this._push([
213
+ this.controlledPush([
214
4, 0x07064b50,
215
4, 0,
216
8, after,
@@ -212,7 +218,7 @@ export class QuickZipStream extends Readable {
218
])
219
centralOffset = 0xFFFFFFFF
220
}
215
- this._push([
221
+ this.controlledPush([
222
4,0x06054b50, // end of central directory signature
223
4,0, // disk-related stuff
224
2,this.numberOfFiles,