optimization: list can be 40% faster on a folder with many files
Massimo Melina committed
May 10, 2026 at 01:05 UTC
d297bb5e87aaaff6dcaf6ca4b6b630253c377c62
4 files changed
+28
-19
src/comments.ts
+7
-4
@@ -6,6 +6,7 @@ import { loadFileAttr, singleWorkerFromBatchWorker, storeFileAttr } from './misc
6
import _ from 'lodash'
7
import iconv from 'iconv-lite'
8
import { unlink } from 'node:fs/promises'
9
+import { expiringCache } from './expiringCache'
10
11
export const DESCRIPT_ION = 'descript.ion'
12
export const DESCRIPT_ION_ALT = 'DESCRIPT.ION'
@@ -82,13 +83,15 @@ async function filePathHelper(folder: string) {
83
}
84
85
const MULTILINE_SUFFIX = Buffer.from([4, 0xC2])
86
+const pathCache = expiringCache<Promise<string>>(2_000)
87
+// this can be called many times when listing a folder, and we want to also not check too often as it can be expensive, especially on a networked drive
88
async function readDescriptIon(path: string) {
86
- // decoding could also be done with native TextDecoder.decode, but we need iconv for the encoding anyway
87
- return parseFile(await filePathHelper(path), raw => {
88
- // for simplicity we "remove" the sequence MULTILINE_SUFFIX before iconv.decode messes it up
89
+ return parseFile(await pathCache.try(path, filePathHelper), raw => {
90
+ // for simplicity, we "remove" the sequence MULTILINE_SUFFIX before iconv.decode messes it up
91
for (let i=0; i<raw.length; i++)
92
if (raw[i] === MULTILINE_SUFFIX[0] && raw[i+1] === MULTILINE_SUFFIX[1] && [undefined,13,10].includes(raw[i+2]))
93
raw[i] = raw[i+1] = 10
94
+ // decoding could also be done with native TextDecoder.decode, but we need iconv for the encoding anyway
95
const decoded = iconv.decode(raw, descriptIonEncoding.get())
96
const ret = new Map(decoded.split('\n').map(line => {
97
const quoted = line[0] === '"' ? 1 : 0
@@ -99,7 +102,7 @@ async function readDescriptIon(path: string) {
102
}))
103
ret.delete('')
104
return ret
102
- })
105
+ }, 2000)
106
}
107
108
descriptIonEncoding.sub(() => { // invalidate cache at encoding change
src/expiringCache.ts
+10
-8
@@ -3,25 +3,27 @@ export function expiringCache<T, K=string>(ttlMs: number) {
3
throw Error('invalid TTL')
4
const o = new Map<K,T>()
5
return Object.assign(o, {
6
- // creator can return undefined if the value should not be cached. `invalidate` is useful in case you have some custom logic for it, other than ttl.
7
- try(k: K, creator: (invalidate: () => void) => T): T {
6
+ invalidate,
7
+ // creator can return undefined if the value should not be cached
8
+ try(k: K, creator: (k: K) => T): T {
9
let ret = o.get(k)
10
if (ret === undefined) { // undefined = missing, as we don't accept this value in our cache
10
- ret = creator(invalidate)
11
+ ret = creator(k)
12
if (ret !== undefined) {
13
o.set(k, ret)
14
Promise.resolve(ret).then(v => {
15
if (v === undefined) // even in a promise, we'll consider undefined as a request to cancel the caching
15
- invalidate()
16
+ invalidate(k)
17
}, () => {}) // avoid js warning
17
- .finally(() => setTimeout(invalidate, ttlMs)) // wait for async (in case) before starting the timer
18
- }
19
- function invalidate() {
20
- o.delete(k)
18
+ .finally(() => setTimeout(() => invalidate(k), ttlMs)) // wait for async (in case) before starting the timer
19
}
20
}
21
return ret
22
},
23
})
24
+
25
+ function invalidate(k: K) {
26
+ o.delete(k)
27
+ }
28
}
29
src/serveGuiFiles.ts
+1
-2
@@ -42,8 +42,7 @@ function serveStatic(uri: string): Koa.Middleware {
42
return ctx.status = HTTP_METHOD_NOT_ALLOWED
43
const serveApp = shouldServeApp(ctx)
44
const fullPath = join(__dirname, '..', folder, serveApp ? '/index.html': ctx.path)
45
- const content = await parseFile(fullPath,
46
- raw => serveApp || !raw.length ? raw : adjustBundlerLinks(ctx, uri, raw) )
45
+ const content = await parseFile(fullPath, raw => serveApp || !raw.length ? raw : adjustBundlerLinks(ctx, uri, raw), 1000)
46
.catch(e => {
47
if (e?.code !== 'ENOENT') // not supposed to happen, and yet a user reported a strange behavior
48
console.error(`serveStatic/parseFile: ${String(e)}`)
src/util-files.ts
+10
-5
@@ -174,21 +174,26 @@ export function exists(path: string) {
174
}
175
176
// parse a file, caching unless timestamp has changed
177
-export const parseFileCache = new Map<string, { ts: Date, parsed: unknown }>()
178
-export async function loadFileCached<T>(path: string, loader: (path: string) => T) {
177
+export const parseFileCache = new Map<string, { ts: Date, lastCheck: number, parsed: unknown }>()
178
+export async function loadFileCached<T>(path: string, loader: (path: string) => T, minInterval=0) {
179
const cached = parseFileCache.get(path)
180
+ const now = Date.now()
181
+ if (cached && now - cached.lastCheck < minInterval)
182
+ return cached.parsed as T
183
const ts = await statWithTimeout(path).then(x => x.mtime, e => {
184
if (e?.message !== 'timeout')
185
throw e
186
return cached?.ts || new Date(0) // on timeout (e.g. thread pool saturated), serve cache if any, or attempt the loader
187
})
188
+ if (cached)
189
+ cached.lastCheck = now
190
if (cached && Number(ts) === Number(cached.ts))
191
return cached.parsed as T
192
const parsed = loader(path)
188
- parseFileCache.set(path, { ts, parsed })
193
+ parseFileCache.set(path, { ts, parsed, lastCheck: now })
194
return parsed
195
}
196
192
-export async function parseFile<T>(path: string, parse: (raw: Buffer) => T) {
193
- return loadFileCached(path, () => readFile(path).then(parse))
197
+export async function parseFile<T>(path: string, parse: (raw: Buffer) => T, skipStatIfFresherThan=0) {
198
+ return loadFileCached(path, () => readFile(path).then(parse), skipStatIfFresherThan)
199
}