admin/logs: much faster loading
Massimo Melina committed
Feb 9, 2024 at 19:50 UTC
7eb4db2dab8c5405a6ad7dfb37c85ee7ff500947
3 files changed
+81
-55
admin/src/LogsPage.ts
+61
-8
@@ -2,20 +2,22 @@
2
3
import { createElement as h, Fragment, ReactNode, useMemo, useState } from 'react';
4
import { Box, Tab, Tabs } from '@mui/material'
5
-import { API_URL, useApiList } from './api'
5
+import { API_URL, useApi, useApiList } from './api'
6
import { DataTable } from './DataTable'
7
-import { CFG, Dict, formatBytes, HTTP_UNAUTHORIZED, newDialog, prefix, shortenAgent, splitAt, tryJson, typedKeys, NBSP
8
-} from '@hfs/shared'
7
+import { CFG, Dict, formatBytes, HTTP_UNAUTHORIZED, newDialog, prefix, shortenAgent, splitAt, tryJson,
8
+ typedKeys, NBSP, _dbg } from '@hfs/shared'
9
import { logLabels } from './OptionsPage'
10
-import { NetmaskField, Flex, IconBtn, useBreakpoint, usePauseButton, useToggleButton, WildcardsSupported, Country,
11
- hTooltip } from './mui';
10
+import {
11
+ NetmaskField, Flex, IconBtn, useBreakpoint, usePauseButton, useToggleButton, WildcardsSupported, Country,
12
+ hTooltip, Btn
13
+} from './mui';
14
import { GridColDef } from '@mui/x-data-grid'
15
import _ from 'lodash'
14
-import { Download, Settings, SmartToy } from '@mui/icons-material'
16
+import { ClearAll, Download, Settings, SmartToy } from '@mui/icons-material'
17
import md from './md'
18
import { ConfigForm } from './ConfigForm'
19
import { BoolField, SelectField } from '@hfs/mui-grid-form'
18
-import { useDialogBarColors } from './dialog'
20
+import { toast, useDialogBarColors } from './dialog'
21
22
export default function LogsPage() {
23
const [tab, setTab] = useState(0)
@@ -87,7 +89,32 @@ function LogFile({ file, addToFooter }: { file: string, addToFooter?: ReactNode
89
sx: { rotate: v ? '0deg' : '180deg' },
90
disabled: file === 'console',
91
}), true)
90
- const { list, error, connecting } = useApiList('get_log', { file }, {
92
+ const [totalSize, setTotalSize] = useState(NaN)
93
+ const [limited, setLimited] = useState(true)
94
+ const [skipped, setSkipped] = useState(0)
95
+ const MAX = 2**20
96
+ useApi('get_log_file', { file, range: limited || !skipped ? -MAX : `0-${skipped}` }, {
97
+ skipParse: true, skipLog: true,
98
+ onResponse(res, body) {
99
+ const lines = body.split('\n')
100
+ if (limited) {
101
+ const size = Number(splitAt('/', res.headers.get('Content-Range') ||'')?.[1])
102
+ if (isNaN(size)) throw _dbg("shouldn't happen")
103
+ setTotalSize(size)
104
+ if (body.length >= size)
105
+ setLimited(false)
106
+ else
107
+ setSkipped(size! - body.length + lines.shift().length + 1)
108
+ }
109
+ else if (skipped) {
110
+ toast(`Entire log loaded, ${formatBytes(skipped)}`)
111
+ setSkipped(0)
112
+ }
113
+ const treated = lines.map(parseLogLine).filter(Boolean).reverse()
114
+ setList(x => [...x, ...treated])
115
+ }
116
+ })
117
+ const { list, setList, error, connecting } = useApiList('get_log', { file }, {
118
invert: true,
119
pause,
120
map(x) {
@@ -125,6 +152,15 @@ function LogFile({ file, addToFooter }: { file: string, addToFooter?: ReactNode
152
addToFooter: h(Box, {}, // 4 icons don't fit the tabs row on mobile
153
pauseButton,
154
showApiButton,
155
+ !connecting && skipped > 0 && h(Btn, {
156
+ icon: ClearAll,
157
+ variant: 'outlined',
158
+ sx: { ml: 1 },
159
+ labelFrom: 'md',
160
+ title: `Only ${formatBytes(MAX)} was loaded, for speed. Total size is ${formatBytes(totalSize)}`,
161
+ loading: !limited,
162
+ onClick: () => setLimited(false)
163
+ }, "Load whole log"),
164
addToFooter,
165
),
166
columns: file === 'console' ? [
@@ -264,3 +300,20 @@ const OSS = {
300
android: /Android/,
301
linux: /Linux/,
302
}
303
+
304
+function parseLogLine(line: string, id: number) {
305
+ const m = /^(.+?) (.+?) (.+?) \[(.{11}):(.{14})] "(\w+) ([^"]+) HTTP\/\d.\d" (\d+) (-|\d+) ?(.*)/.exec(line)
306
+ if (!m) return
307
+ const [, ip, , user, date, time, method, uri, status, length, extra] = m
308
+ return { // keep object format same as events emitted by the log module
309
+ id,
310
+ ip,
311
+ user: user === '-' ? undefined : user,
312
+ ts: new Date(date + ' ' + time),
313
+ method,
314
+ uri,
315
+ status: Number(status),
316
+ length: length === '-' ? undefined : Number(length),
317
+ extra: tryJson(tryJson(extra)) || undefined,
318
+ }
319
+}
\ No newline at end of file
shared/api.ts
+7
-3
@@ -16,6 +16,9 @@ interface ApiCallOptions {
16
timeout?: number | false
17
modal?: undefined | ((cmd: string, params?: Dict) => (() => unknown))
18
onResponse?: (res: Response, body: any) => any
19
+ method?: string
20
+ skipParse?: boolean
21
+ skipLog?: boolean
22
}
23
24
const defaultApiCallOptions: ApiCallOptions = {}
@@ -34,7 +37,7 @@ export function apiCall<T=any>(cmd: string, params?: Dict, options: ApiCallOptio
37
console.debug('API TIMEOUT', cmd, params??'')
38
}, ms)
39
return Object.assign(fetch(getPrefixUrl() + API_URL + cmd, {
37
- method: 'POST',
40
+ method: options.method || 'POST',
41
headers: { 'content-type': 'application/json', 'x-hfs-anti-csrf': '1' },
42
signal: controller.signal,
43
body: params && JSON.stringify(params),
@@ -42,10 +45,11 @@ export function apiCall<T=any>(cmd: string, params?: Dict, options: ApiCallOptio
45
stop?.()
46
let body: any = await res.text()
47
let data: any
45
- try { data = JSON.parse(body) }
48
+ try { data = options.skipParse ? undefined : JSON.parse(body) }
49
catch {}
50
const result = data ?? body
48
- console.debug(res.ok ? 'API' : 'API FAILED', cmd, params??'', '>>', result)
51
+ if (!options?.skipLog)
52
+ console.debug(res.ok ? 'API' : 'API FAILED', cmd, params??'', '>>', result)
53
await options.onResponse?.(res, result)
54
if (!res.ok)
55
throw new ApiError(res.status, data === undefined ? body : `Failed API ${cmd}: ${res.statusText}`, data)
src/api.log.ts
+13
-44
@@ -1,23 +1,25 @@
1
import { ApiHandlers } from './apiMiddleware'
2
import _ from 'lodash'
3
import { consoleLog } from './consoleLog'
4
-import { HTTP_NOT_ACCEPTABLE, HTTP_NOT_FOUND, tryJson, wait } from './cross'
4
+import { HTTP_NOT_ACCEPTABLE, HTTP_NOT_FOUND, wait } from './cross'
5
import events from './events'
6
import { loggers } from './log'
7
-import { createReadStream } from 'fs'
8
-import readline from 'readline'
7
import { onOff } from './misc'
8
import { SendListReadable } from './SendList'
9
import { serveFile } from './serveFile'
10
11
export default {
14
- async get_log_file({ file = 'log' }, ctx) {
12
+ async get_log_file({ file = 'log', range = '' }, ctx) {
13
const log = _.find(loggers, { name: file })
14
if (!log)
15
throw HTTP_NOT_FOUND
16
if (!log.path)
17
throw HTTP_NOT_ACCEPTABLE
18
ctx.attachment(log.path)
19
+ if (range)
20
+ ctx.request.header.range = `bytes=${range}`
21
+ if (ctx.method === 'POST') // this would cause method_not_allowed
22
+ ctx.method = 'GET'
23
await serveFile(ctx, log.path)
24
return null
25
},
@@ -36,49 +38,16 @@ export default {
38
events.on('console', x => list.add(x))
39
return
40
}
39
- const logger = loggers.find(l => l.name === file)
40
- if (!logger)
41
+ if (!_.find(loggers, { name: file }))
42
return list.error(HTTP_NOT_FOUND, true)
42
- const input = createReadStream(logger.path)
43
- input.on('error', async (e: any) => {
44
- if (e.code === 'ENOENT') // ignore ENOENT, consider it an empty log
45
- return list.ready()
46
- list.error(e.code || e.message)
47
- })
48
- input.on('end', () =>
49
- list.ready())
50
- input.on('ready', () => {
51
- readline.createInterface({ input }).on('line', line => {
52
- if (ctx.aborted)
53
- return input.close()
54
- const obj = parse(line)
55
- if (obj)
56
- list.add(obj)
57
- }).on('close', () => { // file is automatically closed, so we continue by events
58
- ctx.res.once('close', onOff(events, { // unsubscribe when connection is interrupted
59
- [logger.name](entry) {
60
- list.add(entry)
61
- }
62
- }))
63
- })
64
- })
43
+ list.ready()
44
+ ctx.res.once('close', onOff(events, { // unsubscribe when connection is interrupted
45
+ [file](entry) {
46
+ list.add(entry)
47
+ }
48
+ }))
49
}
50
})
51
68
- function parse(line: string) {
69
- const m = /^(.+?) (.+?) (.+?) \[(.{11}):(.{14})] "(\w+) ([^"]+) HTTP\/\d.\d" (\d+) (-|\d+) ?(.*)/.exec(line)
70
- if (!m) return
71
- const [, ip, , user, date, time, method, uri, status, length, extra] = m
72
- return { // keep object format same as events emitted by the log module
73
- ip,
74
- user: user === '-' ? undefined : user,
75
- ts: new Date(date + ' ' + time),
76
- method,
77
- uri,
78
- status: Number(status),
79
- length: length === '-' ? undefined : Number(length),
80
- extra: tryJson(tryJson(extra)) || undefined,
81
- }
82
- }
52
},
53
} satisfies ApiHandlers
\ No newline at end of file