fix: upload: avoid writing more than declared
Massimo Melina committed
May 31, 2024 at 17:58 UTC
e5fed1bd46184fc386f7b98d5db2bf386aaac3a3
4 files changed
+51
-17
src/misc.ts
+17
-1
@@ -9,7 +9,7 @@ export * from './util-files'
9
export * from './fileAttr'
10
export * from './cross'
11
export * from './debounceAsync'
12
-import { Readable } from 'stream'
12
+import { Readable, Transform } from 'stream'
13
import { SocketAddress, BlockList } from 'node:net'
14
import { ApiError } from './apiMiddleware'
15
import { HTTP_BAD_REQUEST } from './const'
@@ -103,4 +103,20 @@ export function apiAssertTypes(paramsByType: { [type:string]: { [name:string]: a
103
for (const [name,val] of Object.entries(params))
104
if (! types.split('_').some(type => type === 'array' ? Array.isArray(val) : typeof val === type))
105
throw new ApiError(HTTP_BAD_REQUEST, 'bad ' + name)
106
+}
107
+
108
+export function createStreamLimiter(limit: number) {
109
+ let got = 0
110
+ return new Transform({
111
+ transform(chunk, enc, cb) {
112
+ const left = limit - got
113
+ got += chunk.length
114
+ if (left > 0) {
115
+ this.push(chunk.length >= left ? chunk.slice(0, left) : chunk)
116
+ if (got >= limit)
117
+ this.end()
118
+ }
119
+ cb()
120
+ }
121
+ })
122
}
\ No newline at end of file
src/upload.ts
+11
-10
@@ -4,7 +4,8 @@ import { HTTP_CONFLICT, HTTP_FOOL, HTTP_PAYLOAD_TOO_LARGE, HTTP_RANGE_NOT_SATISF
4
HTTP_BAD_REQUEST } from './const'
5
import { basename, dirname, extname, join } from 'path'
6
import fs from 'fs'
7
-import { Callback, dirTraversal, escapeHTML, loadFileAttr, pendingPromise, storeFileAttr, try_ } from './misc'
7
+import { Callback, dirTraversal, escapeHTML, loadFileAttr, pendingPromise, storeFileAttr, try_,
8
+ createStreamLimiter, } from './misc'
9
import { notifyClient } from './frontEndApis'
10
import { defineConfig } from './config'
11
import { getDiskSpaceSync } from './util-os'
@@ -105,23 +106,23 @@ export function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
106
const resuming = resume && resumable
107
if (!resuming)
108
resume = 0
108
- const writeStream = resuming ? fs.createWriteStream(resumable, { flags: 'r+', start: resume })
109
- : fs.createWriteStream(tempName)
109
+ const writeStream = createStreamLimiter(reqSize ?? Infinity)
110
if (resuming) {
111
fs.rm(tempName, () => {})
112
tempName = resumable
113
}
114
cancelDeletion(tempName)
115
ctx.state.uploadDestinationPath = tempName
116
- trackProgress()
116
// allow plugins to mess with the write-stream, because the read-stream can be complicated in case of multipart
117
const obj = { ctx, writeStream }
118
const resEvent = events.emit('uploadStart', obj)
120
- if (resEvent?.preventDefault())
121
- return writeStream.close(() => {
122
- try { fs.unlinkSync(writeStream.path) }
123
- catch {}
124
- })
119
+ if (resEvent?.preventDefault()) return
120
+
121
+ const fileStream = resuming ? fs.createWriteStream(resumable, { flags: 'r+', start: resume })
122
+ : fs.createWriteStream(tempName)
123
+ writeStream.pipe(fileStream)
124
+ Object.assign(obj, { fileStream })
125
+ trackProgress()
126
127
const lockMiddleware = pendingPromise() // outside we need to know when all operations stopped
128
writeStream.once('close', async () => {
@@ -182,7 +183,7 @@ export function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
183
if (!conn) return
184
const h = setInterval(() => {
185
const now = Date.now()
185
- const got = writeStream.bytesWritten
186
+ const got = fileStream.bytesWritten
187
const inSpeed = roundSpeed((got - lastGot) / (now - lastGotTime))
188
lastGot = got
189
lastGotTime = now
src/util-http.ts
+2
-2
@@ -34,11 +34,11 @@ export function httpStream(url: string, { body, jar, noRedirect, httpThrow, ...o
34
if (body) {
35
options.method ||= 'POST'
36
if (_.isPlainObject(body)) {
37
- options.headers['Content-Type'] ??= 'application/json'
37
+ options.headers['content-type'] ??= 'application/json'
38
body = JSON.stringify(body)
39
}
40
if (!(body instanceof Readable))
41
- options.headers['Content-Length'] ??= Buffer.byteLength(body)
41
+ options.headers['content-length'] ??= Buffer.byteLength(body)
42
}
43
if (jar)
44
options.headers.cookie = _.map(jar, (v,k) => `${k}=${v}; `).join('')
tests/test.ts
+21
-4
@@ -17,9 +17,11 @@ const appStarted = new Promise(resolve =>
17
const username = 'rejetto'
18
const password = 'password'
19
const API = '/~/api/'
20
+const ROOT = 'tests/'
21
const BASE_URL = 'http://localhost:81'
22
const UPLOAD_ROOT = '/for-admins/upload/'
22
-const UPLOAD_DEST = UPLOAD_ROOT + 'temp/gpl.png'
23
+const UPLOAD_RELATIVE = 'temp/gpl.png'
24
+const UPLOAD_DEST = UPLOAD_ROOT + UPLOAD_RELATIVE
25
const BIG_CONTENT = _.repeat(randomId(10), 200_000) // 2MB, big enough to saturate buffers
26
const throttle = BIG_CONTENT.length /1000 /0.5 // KB, finish in 0.5s, quick but still overlapping downloads
27
@@ -120,7 +122,7 @@ describe('accounts', () => {
122
})
123
124
describe('limits', () => {
123
- const fn = 'tests/big'
125
+ const fn = ROOT + 'big'
126
before(() => writeFile(fn, BIG_CONTENT))
127
it('max_dl', () => testMaxDl('/' + fn, 1, 2))
128
after(() => rm(fn))
@@ -139,6 +141,21 @@ describe('after-login', () => {
141
it('delete.miss renamed', reqApi('delete', { uri: UPLOAD_DEST }, 404))
142
it('delete.ok', reqApi('delete', { uri: dirname(UPLOAD_DEST) + '/' + renameTo }, 200))
143
it('delete.miss deleted', reqApi('delete', { uri: UPLOAD_DEST }, 404))
144
+ it('upload.size', async () => {
145
+ const fn = 'temp/size'
146
+ await reqUpload(UPLOAD_ROOT + fn, 200, BIG_CONTENT)()
147
+ const { size } = statSync(ROOT + fn)
148
+ if (size !== BIG_CONTENT.length)
149
+ throw Error(`wrote ${size}`)
150
+ })
151
+ it('upload.too much', async () => {
152
+ const fn = 'temp/tooMuch'
153
+ const wrongSize = BIG_CONTENT.length / 2
154
+ await reqUpload(UPLOAD_ROOT + fn, 200, BIG_CONTENT, wrongSize)()
155
+ const { size } = statSync(ROOT + fn)
156
+ if (size !== wrongSize)
157
+ throw Error(`wrote ${size}`)
158
+ })
159
it('max_dl.account', async () => {
160
const uri = UPLOAD_ROOT + 'temp/big'
161
await reqUpload(uri, 200, BIG_CONTENT)()
@@ -153,11 +170,11 @@ function login(usr: string, pwd=password) {
170
reqApi(cmd, params, (x,res)=> res.statusCode < 400)())
171
}
172
156
-function reqUpload(dest: string, tester: Tester, body?: string) {
173
+function reqUpload(dest: string, tester: Tester, body?: string, size?: number) {
174
const fn = join(__dirname, 'page/gpl.png')
175
return req(dest, tester, {
176
method: 'PUT',
160
- headers: { 'content-length': body?.length ?? statSync(fn).size },
177
+ headers: { 'content-length': size ?? body?.length ?? statSync(fn).size },
178
body: body ?? createReadStream(fn)
179
})
180
}