zip rotated logs
Massimo Melina committed
Mar 23, 2026 at 23:52 UTC
c4941d15b3eb8080e959036c0dc20ad383006446
3 files changed
+38
-4
dev-plugins.md
+6
@@ -772,6 +772,10 @@ This section is still partially documented, and you may need to have a look at t
772
- parameters: { ctx, length, user, ts, uri, extra }
773
- `error_log`
774
- parameters: { ctx, length, user, ts, uri, extra }
775
+- `logRotated` called as soon as the zipping is done and just before the original is deleted.
776
+ If you need to work on the original, please be async (return promise) so that HFS knows when you are done and will delay deletion accordingly.
777
+ - parameters: { path, zipPath }
778
+ - async supported
779
- `accountRenamed`
780
- parameters: { from, to }
781
- `pluginDownload`
@@ -1199,3 +1203,5 @@ If you want to override a text regardless of the language, use the special langu
1203
- backend events: dirEntry, request, alert
1204
- HFS.pathSeparator
1205
- config.type=show_html
1206
+- 13.1 (v3.2.0)
1207
+ - backend events: logRotated
\ No newline at end of file
package.json
+3
-1
@@ -109,7 +109,8 @@
109
"unzipper": "^0.12.3",
110
"valtio": "^1.13.2",
111
"xxhashjs": "^0.2.2",
112
- "yaml": "^2.8.1"
112
+ "yaml": "^2.8.1",
113
+ "yazl": "^3.3.1"
114
},
115
"devDependencies": {
116
"@playwright/test": "^1.55.1",
@@ -123,6 +124,7 @@
124
"@types/node-forge": "^1.3.14",
125
"@types/picomatch": "^4.0.2",
126
"@types/unzipper": "^0.10.11",
127
+ "@types/yazl": "^3.3.0",
128
"@yao-pkg/pkg": "6.14.2",
129
"cross-env": "^10.0.0",
130
"koa-better-http-proxy": "^0.2.10",
src/log.ts
+29
-3
@@ -10,7 +10,10 @@ import _ from 'lodash'
10
import { createFileWithPath, ensureParentFolder, statWithTimeout } from './util-files'
11
import { getCurrentUsername } from './auth'
12
import { DAY, makeNetMatcher, tryJson, Dict, Falsy, CFG, strinsert, repeat, formatTimestamp, HTTP_NOT_FOUND } from './misc'
13
-import { extname } from 'path'
13
+import { basename, extname } from 'path'
14
+import yazl from 'yazl'
15
+import { pipeline } from 'stream/promises'
16
+import { rm, unlink, utimes } from 'fs/promises'
17
import events from './events'
18
import { getConnection } from './connections'
19
import { app } from './index'
@@ -111,6 +114,7 @@ export const logMw: Koa.Middleware = async (ctx, next) => {
114
const newPath = strinsert(path, path.length - extname(path).length, suffix)
115
try { // other logging requests shouldn't happen while we are renaming. Since this is very infrequent we can tolerate solving this by making it sync.
116
renameSync(path, newPath)
117
+ void zipLogFile(newPath, last).catch(console.error)
118
}
119
catch(e: any) { // ok, rename failed, but this doesn't mean we ain't gonna log
120
console.error(e.message || String(e))
@@ -154,7 +158,7 @@ export const logMw: Koa.Middleware = async (ctx, next) => {
158
length?.toString() ?? '-',
159
_.isEmpty(extra) ? '' : JSON.stringify(JSON.stringify(extra)), // jsonize twice, as we need a field enclosed by double-quotes
160
))
157
- })
161
+ }).catch(e => console.error('log completion:', e.message || String(e)))
162
}
163
164
declare module "koa" {
@@ -176,6 +180,24 @@ events.once('app', () => { // wait for app to be set
180
}
181
})
182
183
+export async function zipLogFile(path: string, touch: Date) {
184
+ const zipPath = path + '.zip'
185
+ try {
186
+ const zip = new yazl.ZipFile()
187
+ const output = createWriteStream(zipPath)
188
+ zip.addFile(path, basename(path))
189
+ zip.end()
190
+ await pipeline(zip.outputStream, output)
191
+ }
192
+ catch (e) {
193
+ await rm(zipPath, { force: true }).catch(() => {})
194
+ throw e
195
+ }
196
+ await utimes(zipPath, touch, touch).catch(console.error)
197
+ await events.emitAsync('logRotated', { path, zipPath })
198
+ await unlink(path).catch(console.error)
199
+}
200
+
201
function doubleDigit(n: number) {
202
return n > 9 ? n : '0'+n
203
}
@@ -183,7 +205,11 @@ function doubleDigit(n: number) {
205
export async function getRotatedFiles() {
206
return Object.fromEntries(await Promise.all(loggers.map(async x => {
207
const mask = strinsert(x.path, x.path.length - extname(x.path).length, '-2*') // including 2, initial digit of the year, will only take rotated files and not "-error"
186
- return [x.name, (await glob(mask, { stats: true })).map(x => ({ path: x.path, size: x.stats?.size }))]
208
+ const list = await Promise.all(['', '.zip'].map(async postfix => // before 3.2 rotated logs were not zipped, and even today there's a very small (negligible) chance that the zipping fails
209
+ (await glob(mask + postfix, { stats: true }))
210
+ .map(x => ({ path: x.path, size: x.stats?.size }))
211
+ ))
212
+ return [x.name, list.flat()]
213
})))
214
}
215