| 1 | // This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt |
| 2 | |
| 3 | import { access, chmod, mkdir, readFile, stat } from 'fs/promises' |
| 4 | import { Promisable, try_, wait, isWindowsDrive, haveTimeout } from './cross' |
| 5 | import { defineConfig } from './config' |
| 6 | import { createWriteStream, mkdirSync, watch, ftruncate, Stats } from 'fs' |
| 7 | import { basename, dirname } from 'path' |
| 8 | import glob from 'fast-glob' |
| 9 | import { IS_WINDOWS } from './const' |
| 10 | import { finished } from 'stream/promises' |
| 11 | import { Readable } from 'stream' |
| 12 | import { getStatWorker } from './stat' |
| 13 | import unzipper from 'unzipper' |
| 14 | |
| 15 | const fileTimeout = defineConfig('file_timeout', 3, x => x * 1000) |
| 16 | // a smart (and a bit arbitrary) way to decide if we need the stat-workers functionality. Without it, we may be a bit faster. We'll see with experience if we need a dedicated configuration. |
| 17 | const disableStatWorkers = Number(process.env.UV_THREADPOOL_SIZE) >= 10 |
| 18 | |
| 19 | // some paths (mostly offline networked ones) can take 20+ seconds to fail |
| 20 | export async function statWithTimeout(path: string) { |
| 21 | // with just 4 requests we saturate the default UV_THREADPOOL_SIZE, blocking even local fs operations, so we move all UNC requests to worker threads |
| 22 | const workerKey = !disableStatWorkers && IS_WINDOWS && getUncHost(path) // pool requests by server name |
| 23 | const op = workerKey ? getStatWorker(workerKey)(path) : stat(path) |
| 24 | return haveTimeout(fileTimeout.compiled(), op) |
| 25 | } |
| 26 | |
| 27 | export function getUncHost(path: string) { |
| 28 | return /^\\\\([^\\]+)\\/.exec(path)?.[1] |
| 29 | } |
| 30 | |
| 31 | export async function isDirectory(path: string) { |
| 32 | try { return (await statWithTimeout(path)).isDirectory() } |
| 33 | catch {} |
| 34 | } |
| 35 | |
| 36 | export async function readFileWithBusyRetry(path: string): Promise<string> { |
| 37 | return readFile(path, 'utf8').catch(e => { |
| 38 | if ((e as any)?.code !== 'EBUSY') |
| 39 | throw e |
| 40 | console.debug('Busy') |
| 41 | return wait(100).then(()=> readFileWithBusyRetry(path)) |
| 42 | }) |
| 43 | } |
| 44 | |
| 45 | export function watchDir(dir: string, cb: ()=>void, atStart=false) { |
| 46 | let watcher: ReturnType<typeof watch> |
| 47 | let paused = false |
| 48 | try { |
| 49 | watcher = watch(dir, controlledCb) |
| 50 | } |
| 51 | catch { |
| 52 | // failing watching the content of the dir, we try to monitor its parent, but filtering events only for our target dir |
| 53 | const base = basename(dir) |
| 54 | try { |
| 55 | watcher = watch(dirname(dir), (event,name) => { |
| 56 | if (name !== base) return |
| 57 | try { |
| 58 | watcher.close() // if we succeed, we give up the parent watching |
| 59 | watcher = watch(dir, controlledCb) // attempt at passing to a more specific watching |
| 60 | } |
| 61 | catch {} |
| 62 | controlledCb() |
| 63 | }) |
| 64 | } |
| 65 | catch (e) { |
| 66 | console.debug(String(e)) |
| 67 | } |
| 68 | } |
| 69 | if (atStart) |
| 70 | controlledCb() |
| 71 | return { |
| 72 | working() { return Boolean(watcher) }, |
| 73 | stop() { watcher?.close() }, |
| 74 | pause() { paused = true }, |
| 75 | unpause() { paused = false }, |
| 76 | } |
| 77 | |
| 78 | function controlledCb() { |
| 79 | if (!paused) |
| 80 | cb() |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | export function hasDirTraversal(s?: string) { |
| 85 | return s && /(^|[/\\])\.\.($|[/\\])/.test(s) |
| 86 | } |
| 87 | |
| 88 | // apply this to paths that may contain \ as separator (not supported by fast-glob) and other special chars to be escaped (parenthesis) |
| 89 | export function escapeGlobPath(path: string) { |
| 90 | return glob.escapePath(path.replace(/\\/g, '/')) |
| 91 | } |
| 92 | |
| 93 | export async function unzip(stream: Readable, cb: (path: string) => Promisable<false | string>) { |
| 94 | const extracted = new Map<string, string>() |
| 95 | let chain: Promise<any> = Promise.resolve() |
| 96 | return new Promise((resolve, reject) => |
| 97 | stream.pipe(unzipper.Parse()) |
| 98 | .on('close', () => chain.then(resolve, reject)) |
| 99 | .on('error', reject) |
| 100 | .on('entry', (entry: any) => |
| 101 | chain = chain.then(async () => { |
| 102 | const { path, type } = entry |
| 103 | if (hasDirTraversal(path)) |
| 104 | return entry.autodrain().promise() |
| 105 | const dest = await try_(() => cb(path), e => console.warn(String(e))) |
| 106 | if (!dest || type !== 'File') |
| 107 | return entry.autodrain().promise() |
| 108 | extracted.set(path, dest) |
| 109 | console.debug('Unzip', dest) |
| 110 | // keep writes serialized so archive entries can't race while callers map paths asynchronously |
| 111 | const thisFile = entry.pipe(await createSafeWriteStream(dest)) |
| 112 | await finished(thisFile) |
| 113 | })) |
| 114 | // unix modes live in the central directory, so we reapply them after the file stream has been written |
| 115 | .on('entryInCentral', (entry: any) => |
| 116 | chain = chain.then(async () => { |
| 117 | if (entry.type !== 'File') return |
| 118 | const dest = extracted.get(entry.path) |
| 119 | if (dest && entry.unixAttrs) |
| 120 | await chmod(dest, entry.unixAttrs).catch(() => {}) |
| 121 | })) ) |
| 122 | } |
| 123 | |
| 124 | export async function ensureParentFolder(path: string, dirnameIt=true) { |
| 125 | if (dirnameIt) |
| 126 | path = dirname(path) |
| 127 | if (isWindowsDrive(path)) return |
| 128 | try { |
| 129 | await mkdir(path, { recursive: true }) |
| 130 | return true |
| 131 | } |
| 132 | catch { |
| 133 | return false |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | export function createFileWithPath(path: string, options?: Parameters<typeof createWriteStream>[1]) { |
| 138 | const folder = dirname(path) |
| 139 | if (!isWindowsDrive(folder)) // can't use prepareFolder because it's async |
| 140 | try { mkdirSync(folder, { recursive: true }) } |
| 141 | catch { |
| 142 | return |
| 143 | } |
| 144 | return createWriteStream(path, options) |
| 145 | } |
| 146 | |
| 147 | export async function createSafeWriteStream(path: string, options?: Parameters<typeof createWriteStream>[1]) { |
| 148 | await ensureParentFolder(path) |
| 149 | return new Promise<ReturnType<typeof createWriteStream>>((resolve, reject) => { |
| 150 | const first = createWriteStream(path, options) |
| 151 | .on('open', () => resolve(first)) |
| 152 | .on('error', (e: any) => { |
| 153 | if (!IS_WINDOWS || e.code !== 'EPERM') // Windows throws EPERM for hidden files with flags 'w' and 'a' |
| 154 | return reject(e) |
| 155 | if (typeof options === 'string') |
| 156 | options = { encoding: options } |
| 157 | else |
| 158 | options ||= {} |
| 159 | if (options.flags && options.flags !== 'w') // we only handle the 'w' case |
| 160 | return reject(e) |
| 161 | options.flags = 'r+' |
| 162 | const second = createWriteStream(path, options) |
| 163 | .on('open', fd => ftruncate(fd, 0, () => resolve(second))) |
| 164 | .on('error', reject) |
| 165 | }) |
| 166 | }) |
| 167 | } |
| 168 | |
| 169 | export function isValidFileName(name: string, acceptUnreadable=false) { |
| 170 | return name && name !== '.' && !(IS_WINDOWS ? !isValidWindowsFileName(name) : /\//.test(name)) && !hasDirTraversal(name) |
| 171 | && (acceptUnreadable || !/[\u0000-\u001F\u007F]/.test(name)) |
| 172 | } |
| 173 | |
| 174 | export function isValidWindowsFileName(name: string) { |
| 175 | return !/[/:"*?<>|\\]/.test(name) |
| 176 | && !/[. ]$/.test(name) |
| 177 | // windows treats these legacy DOS device names as device paths even when an extension is present |
| 178 | && !/^(?:con|prn|aux|nul|conin\$|conout\$|com[1-9\u00b9\u00b2\u00b3]|lpt[1-9\u00b9\u00b2\u00b3])(?:\..*)?$/i.test(name) |
| 179 | } |
| 180 | |
| 181 | export function exists(path: string) { |
| 182 | return access(path).then(() => true, () => false) |
| 183 | } |
| 184 | |
| 185 | // parse a file, caching unless timestamp has changed |
| 186 | export interface CachedFile<T> { stats: Stats, content: T } |
| 187 | export const parseFileCache = new Map<string, { stats: Stats, lastCheck: number, content: Promise<unknown> }>() |
| 188 | export async function loadFileCached<T>(path: string, loader: (path: string) => T, minInterval=0) { |
| 189 | const cached = parseFileCache.get(path) |
| 190 | const now = Date.now() |
| 191 | if (cached && now - cached.lastCheck < minInterval) |
| 192 | return { stats: cached.stats, content: await cached.content } as CachedFile<Awaited<T>> |
| 193 | const stats = await statWithTimeout(path).catch(e => { |
| 194 | if (e?.message === 'timeout' && cached) |
| 195 | return cached.stats // on timeout (e.g. thread pool saturated), serve cache if any |
| 196 | throw e |
| 197 | }) |
| 198 | if (cached) |
| 199 | cached.lastCheck = now |
| 200 | if (cached && Number(stats.mtime) === Number(cached.stats.mtime)) |
| 201 | return { stats, content: await cached.content } as CachedFile<Awaited<T>> |
| 202 | const content = Promise.resolve(loader(path)) |
| 203 | parseFileCache.set(path, { stats, content, lastCheck: now }) |
| 204 | return { stats, content: await content } as CachedFile<Awaited<T>> |
| 205 | } |
| 206 | |
| 207 | export async function parseFile<T>(path: string, parse: (raw: Buffer) => T, skipStatIfFresherThan=0) { |
| 208 | return loadFileCached(path, () => readFile(path).then(parse), skipStatIfFresherThan) |
| 209 | } |