fix: admin/fs: add/from disk could block for many seconds #648
Massimo Melina committed
Jun 24, 2024 at 15:09 UTC
787224a3d6c7bdc0737497b2a4e182b415d4acbf
5 files changed
+65
-47
src/api.vfs.ts
+3
-3
@@ -11,7 +11,7 @@ import {
11
VfsNodeAdminSend
12
} from './misc'
13
import { IS_WINDOWS, HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVER_ERROR, HTTP_CONFLICT, HTTP_NOT_ACCEPTABLE } from './const'
14
-import { getDiskSpaces, getDiskSpaceSync, getDrives } from './util-os'
14
+import { getDiskSpace, getDiskSpaces, getDrives } from './util-os'
15
import { getBaseUrlOrDefault, getServerStatus } from './listen'
16
import { promisify } from 'util'
17
import { execFile } from 'child_process'
@@ -195,8 +195,7 @@ const apis: ApiHandlers = {
195
}
196
return
197
}
198
- try { list.props(getDiskSpaceSync(path)) }
199
- catch {} // continue anyway
198
+ const sendPropsAsap = getDiskSpace(path).then(x => x && list.props(x))
199
try {
200
const matching = makeMatcher(fileMask)
201
path = isWindowsDrive(path) ? path + '\\' : resolve(path || '/')
@@ -217,6 +216,7 @@ const apis: ApiHandlers = {
216
})
217
} catch {} // just ignore entries we can't stat
218
}
219
+ await sendPropsAsap.catch(() => {})
220
list.close()
221
} catch (e: any) {
222
list.error(e.code || e.message || String(e), true)
src/update.ts
+2
-2
@@ -4,7 +4,7 @@ import { getRepoInfo } from './github'
4
import { argv, HFS_REPO, IS_BINARY, IS_WINDOWS, RUNNING_BETA } from './const'
5
import { dirname, join } from 'path'
6
import { spawn, spawnSync } from 'child_process'
7
-import { httpStream, unzip } from './misc'
7
+import { exists, httpStream, unzip } from './misc'
8
import { createReadStream, renameSync, unlinkSync } from 'fs'
9
import { pluginsWatcher } from './plugins'
10
import { access, chmod, stat } from 'fs/promises'
@@ -63,7 +63,7 @@ export async function getUpdates(strict=false) {
63
const LOCAL_UPDATE = 'hfs-update.zip' // update from file takes precedence over net
64
65
export function localUpdateAvailable() {
66
- return access(LOCAL_UPDATE).then(() => true, () => false)
66
+ return exists(LOCAL_UPDATE)
67
}
68
69
export async function updateSupported() {
src/upload.ts
+2
-1
@@ -61,7 +61,8 @@ export function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
61
else
62
try {
63
if (!Object.hasOwn(cache, dir)) {
64
- cache[dir] = getDiskSpaceSync(dir)
64
+ const c = cache[dir] = getDiskSpaceSync(dir)
65
+ if (!c) throw 'miss'
66
setTimeout(() => delete cache[dir], 3_000) // invalidate shortly
67
}
68
const { free } = cache[dir]
src/util-files.ts
+8
-4
@@ -1,6 +1,6 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
-import fs, { readFile, stat } from 'fs/promises'
3
+import { access, mkdir, readFile, stat } from 'fs/promises'
4
import { Promisable, try_, wait, isWindowsDrive } from './misc'
5
import { createWriteStream, mkdirSync, watch } from 'fs'
6
import { basename, dirname } from 'path'
@@ -12,12 +12,12 @@ import { once, Readable } from 'stream'
12
import unzipper from 'unzip-stream'
13
14
export async function isDirectory(path: string) {
15
- try { return (await fs.stat(path)).isDirectory() }
15
+ try { return (await stat(path)).isDirectory() }
16
catch {}
17
}
18
19
export async function readFileBusy(path: string): Promise<string> {
20
- return fs.readFile(path, 'utf8').catch(e => {
20
+ return readFile(path, 'utf8').catch(e => {
21
if ((e as any)?.code !== 'EBUSY')
22
throw e
23
console.debug('busy')
@@ -128,7 +128,7 @@ export async function prepareFolder(path: string, dirnameIt=true) {
128
path = dirname(path)
129
if (isWindowsDrive(path)) return
130
try {
131
- await fs.mkdir(path, { recursive: true })
131
+ await mkdir(path, { recursive: true })
132
return true
133
}
134
catch {
@@ -150,6 +150,10 @@ export function isValidFileName(name: string) {
150
return !/[/*?<>|\\]/.test(name) && !dirTraversal(name)
151
}
152
153
+export function exists(path: string) {
154
+ return access(path).then(() => true, () => false)
155
+}
156
+
157
// read and parse a file, caching unless timestamp has changed
158
export const parseFileCache = new Map<string, { ts: Date, parsed: unknown }>()
159
export async function parseFile<T>(path: string, parse: (raw: Buffer) => T) {
src/util-os.ts
+50
-37
@@ -1,58 +1,54 @@
1
import { dirname, resolve } from 'path'
2
import { existsSync } from 'fs'
3
-import { exec, execSync } from 'child_process'
4
-import { onlyTruthy, splitAt, try_ } from './misc'
3
+import { exec, ExecOptions, execSync } from 'child_process'
4
+import { exists, onlyTruthy, prefix, splitAt } from './misc'
5
import _ from 'lodash'
6
import { pid } from 'node:process'
7
import { promisify } from 'util'
8
import { IS_WINDOWS } from './const'
9
10
+const DF_TIMEOUT = 2000
11
+const LOGICALDISK_TIMEOUT = DF_TIMEOUT
12
+
13
export function getDiskSpaceSync(path: string) {
11
- const timeout = 1000
12
- if (IS_WINDOWS) {
13
- const drive = resolve(path).slice(0, 2).toUpperCase()
14
- const out = execSync('wmic logicaldisk get Size,FreeSpace,Name /format:list', { timeout }).toString().replace(/\r/g, '')
15
- const one = parseKeyValueObjects(out).find(x => x.Name === drive)
16
- if (!one)
17
- throw Error('miss')
18
- return { free: Number(one.FreeSpace), total: Number(one.Size) }
19
- }
14
+ if (IS_WINDOWS)
15
+ return parseLogicaldisk(execSync(makeLogicaldisk(path), { timeout: LOGICALDISK_TIMEOUT }).toString())[0]
16
while (path && !existsSync(path))
17
path = dirname(path)
22
- const out = try_(() => execSync(`df -k "${path}"`, { timeout }).toString(),
23
- err => { throw err.status === 1 ? Error('miss') : err.status === 127 ? Error('unsupported') : err })
24
- if (!out?.startsWith('Filesystem'))
25
- throw Error('unsupported')
26
- const one = out.split('\n')[1] as string
27
- const [used, free] = one.split(/\s+/).slice(2, 4).map(x => Number(x) * 1024) as [number, number]
28
- return { free, total: used + free }
18
+ try { return parseDfResult(execSync(`df -k "${path}"`, { timeout: DF_TIMEOUT }).toString())[0] }
19
+ catch(e: any) { throw parseDfResult(e) }
20
+}
21
+
22
+export async function getDiskSpace(path: string) {
23
+ if (IS_WINDOWS)
24
+ return parseLogicaldisk(await runCmd(makeLogicaldisk(path), [], { timeout: LOGICALDISK_TIMEOUT }))[0]
25
+ while (path && !await exists(path))
26
+ path = dirname(path)
27
+ return parseDfResult(await promisify(exec)(`df -k`, { timeout: DF_TIMEOUT }).then(x => x.stdout, e => e))[0]
28
}
29
30
export async function getDiskSpaces(): Promise<{ name: string, free: number, total: number, description?: string }[]> {
31
if (IS_WINDOWS) {
33
- const fields = ['Size','FreeSpace','Name','Description'] as const
34
- const out = await runCmd(`wmic logicaldisk get ${fields.join()} /format:list`)
35
- const objs = parseKeyValueObjects<typeof fields[number]>(out)
36
- return onlyTruthy(objs.map(x => x.Size && {
37
- total: Number(x.Size),
38
- free: Number(x.FreeSpace),
39
- name: x.Name,
40
- description: x.Description
41
- }))
32
+ const drives = await getDrives() // since network-drives can hang 'wmic' for many seconds checking disk space (issue#648), and a single timeout would make whole operation fail, so we fork the job on each drive
33
+ return onlyTruthy(await Promise.all(drives.map(getDiskSpace)))
34
+ }
35
+ return parseDfResult(await promisify(exec)(`df -k`, { timeout: DF_TIMEOUT }).then(x => x.stdout, e => e))
36
+ .filter(x => !/^\/(dev|System)\b/.test(x.name))
37
+
38
+}
39
+
40
+function parseDfResult(result: string | Error) {
41
+ if (result instanceof Error) {
42
+ const { status } = result as any
43
+ throw status === 1 ? Error('miss') : status === 127 ? Error('unsupported') : result
44
}
43
- const { stdout } = await promisify(exec)(`df -k`).catch(err => {
44
- throw err.status === 1 ? Error('miss')
45
- : err.status === 127 ? Error('unsupported')
46
- : err
47
- })
48
- const out = stdout.split('\n')
45
+ const out = result.split('\n')
46
if (!out.shift()?.startsWith('Filesystem'))
47
throw Error('unsupported')
48
return onlyTruthy(out.map(one => {
49
const bits = one.split(/\s+/)
50
if (bits[0] === 'tempfs') return
51
const name = bits.pop() || bits.shift() || ''
55
- if (/^\/(dev|System)\b/.test(name)) return
52
const [, used=0, free=0] = bits.map(x => Number(x) * 1024)
53
const total = used + free
54
return total && { free, total, name }
@@ -65,9 +61,9 @@ export async function getDrives() {
61
}
62
63
// execute win32 shell commands
68
-export async function runCmd(cmd: string, args: string[] = []) {
64
+export async function runCmd(cmd: string, args: string[] = [], options: ExecOptions = {}) {
65
const line = `@chcp 65001 >nul & cmd /c ${cmd} ${args.map(x => x.includes(' ') ? `"${x}"` : x).join(' ')}`
70
- const { stdout, stderr } = await promisify(exec)(line, { encoding: 'utf-8' })
66
+ const { stdout, stderr } = await promisify(exec)(line, { encoding: 'utf-8', ...options })
67
return (stderr || stdout).replace(/\r/g, '')
68
}
69
@@ -81,4 +77,21 @@ export const RUNNING_AS_SERVICE = IS_WINDOWS && getWindowsServicePids().then(x =
77
function parseKeyValueObjects<T extends string>(all: string, keySep='=', lineSep='\n', objectSep=/\n\n+/) {
78
return all.split(objectSep).map(obj =>
79
Object.fromEntries(obj.split(lineSep).map(kv => splitAt(keySep, kv))) ) as { [k in T]: string }[]
84
-}
\ No newline at end of file
80
+}
81
+
82
+const wmicFields = ['Size','FreeSpace','Name','Description'] as const
83
+
84
+function makeLogicaldisk(path='') {
85
+ const drive = resolve(path).slice(0, 2).toUpperCase()
86
+ return `wmic logicaldisk ${prefix(`where "DeviceID = '`, drive, `'"`)} get ${wmicFields.join()} /format:list`
87
+}
88
+
89
+function parseLogicaldisk(out: string) {
90
+ const objs = parseKeyValueObjects<typeof wmicFields[number]>(out.replace(/\r/g, ''))
91
+ return onlyTruthy(objs.map(x => x.Size && {
92
+ total: Number(x.Size),
93
+ free: Number(x.FreeSpace),
94
+ name: x.Name,
95
+ description: x.Description
96
+ }))
97
+}