WAY faster searching
Massimo Melina committed
Dec 29, 2022 at 15:30 UTC
b90004dd4b34c86adfba407cc1d124af4261c33b
3 files changed
+38
-39
src/api.vfs.ts
+1
-1
@@ -5,7 +5,7 @@ import _ from 'lodash'
5
import { stat } from 'fs/promises'
6
import { ApiError, ApiHandlers } from './apiMiddleware'
7
import { dirname, join, resolve } from 'path'
8
-import { dirStream, enforceFinal, isWindowsDrive, objSameKeys } from './misc'
8
+import { dirStream, isWindowsDrive, objSameKeys } from './misc'
9
import { exec } from 'child_process'
10
import { promisify } from 'util'
11
import { FORBIDDEN, IS_WINDOWS } from './const'
src/util-files.ts
+11
-10
@@ -78,23 +78,24 @@ export function adjustStaticPathForGlob(path: string) {
78
return glob.escapePath(path.replace(/\\/g, '/'))
79
}
80
81
-export async function* dirStream(path: string) {
82
- const stats = await fs.stat(path)
83
- if (!stats.isDirectory())
81
+export async function* dirStream(path: string, deep?: number) {
82
+ if (!await isDirectory(path))
83
throw Error('ENOTDIR')
85
- const dirStream = glob.stream('*', {
84
+ const dirStream = glob.stream(deep ? '**/*' : '*', {
85
cwd: path,
86
dot: true,
87
+ deep,
88
onlyFiles: false,
89
suppressErrors: true,
90
+ objectMode: true,
91
})
92
const skip = await getItemsToSkip(path)
92
- for await (let path of dirStream) {
93
- if (path instanceof Buffer)
94
- path = path.toString('utf8')
95
- if (skip?.includes(path))
96
- continue
97
- yield path
93
+ for await (const entry of dirStream) {
94
+ let { path, dirent } = entry as any
95
+ if (!dirent.isDirectory() && !dirent.isFile()) continue
96
+ path = String(path)
97
+ if (!skip?.includes(path))
98
+ yield path
99
}
100
101
async function getItemsToSkip(path: string) {
src/vfs.ts
+26
-28
@@ -140,7 +140,7 @@ export function hasPermission(node: VfsNode, perm: keyof VfsPerm, ctx: Koa.Conte
140
&& (perm !== 'can_see' || hasPermission(node, 'can_read', ctx)) // for can_see you must also can_read
141
}
142
143
-export async function* walkNode(parent:VfsNode, ctx: Koa.Context, depth:number=0, prefixPath:string=''): AsyncIterableIterator<VfsNode> {
143
+export async function* walkNode(parent:VfsNode, ctx?: Koa.Context, depth:number=0, prefixPath:string=''): AsyncIterableIterator<VfsNode> {
144
const { children, source } = parent
145
if (children)
146
for (let idx = 0; idx < children.length; idx++) {
@@ -148,20 +148,19 @@ export async function* walkNode(parent:VfsNode, ctx: Koa.Context, depth:number=0
148
yield* workItem({
149
...child,
150
name: prefixPath ? (prefixPath + getNodeName(child)) : child.name
151
- })
151
+ }, depth > 0 && await nodeIsDirectory(child).catch(() => false))
152
}
153
if (!source)
154
return
155
try {
156
- for await (const path of dirStream(source)) {
157
- if (ctx.req.aborted)
156
+ for await (const path of dirStream(source, depth)) {
157
+ if (ctx?.req.aborted)
158
return
159
- let { rename } = parent
160
- const renamed = rename?.[path]
159
+ const renamed = parent.rename?.[path]
160
yield* workItem({
162
- name: (prefixPath || renamed) && prefixPath + (renamed || path),
161
+ name: prefixPath + (renamed || path),
162
source: join(source, path),
164
- rename: renameUnderPath(rename, path),
163
+ rename: renameUnderPath(parent.rename, path),
164
})
165
}
166
}
@@ -169,46 +168,45 @@ export async function* walkNode(parent:VfsNode, ctx: Koa.Context, depth:number=0
168
console.debug('glob', source, e) // ENOTDIR, or lacking permissions
169
}
170
172
- async function* workItem(item: VfsNode) {
171
+ // item will be changed, so be sure to pass a temp node
172
+ async function* workItem(item: VfsNode, recur=false) {
173
+ const name = getNodeName(item)
174
// we basename for depth>0 where we already have the rest of the path in the parent's url, and would be duplicated
174
- const name = basename(getNodeName(item))
175
- const url = enforceFinal('/', parent.url || '') + name
176
- const temp = inheritFromParent(parent, {
177
- ...item,
175
+ const virtualBasename = basename(name)
176
+ const url = enforceFinal('/', parent.url || '') + virtualBasename
177
+ Object.assign(item, {
178
isTemp: true,
179
url,
180
parents: [ ...parent.parents||[], parent],
181
})
182
- applyMasks(temp, parent, name)
183
- if (!hasPermission(temp, 'can_see', ctx))
182
+ inheritFromParent(parent, item)
183
+ applyMasks(item, parent, virtualBasename)
184
+ if (ctx && !hasPermission(item, 'can_see', ctx))
185
return
185
- yield temp
186
- try {
187
- if (!depth || !await nodeIsDirectory(temp)) return
188
- inheritMasks(temp, parent, name)
189
- yield* walkNode(temp, ctx, depth - 1, getNodeName(temp) + '/')
190
- }
191
- catch{} // stat failed in nodeIsDirectory, ignore
186
+ yield item
187
+ if (!recur) return
188
+ inheritMasks(item, parent, virtualBasename)
189
+ yield* walkNode(item, ctx, depth - 1, name + '/')
190
}
191
}
194
-function applyMasks(item: VfsNode, parent: VfsNode, name: string) {
192
+function applyMasks(item: VfsNode, parent: VfsNode, virtualBasename: string) {
193
const { masks } = parent
194
if (!masks) return
195
for (const k in masks)
198
- if (k.startsWith('**/') && isMatch(name, k.slice(3))
199
- || !k.includes('/') && isMatch(name, k))
196
+ if (k.startsWith('**/') && isMatch(virtualBasename, k.slice(3))
197
+ || !k.includes('/') && isMatch(virtualBasename, k))
198
Object.assign(item, masks[k])
199
}
200
203
-function inheritMasks(item: VfsNode, parent: VfsNode, name:string) {
201
+function inheritMasks(item: VfsNode, parent: VfsNode, virtualBasename:string) {
202
const { masks } = parent
203
if (!masks) return
204
const o: Masks = {}
205
for (const k in masks)
206
if (k.startsWith('**/'))
207
o[k.slice(3)] = masks[k]
210
- else if (k.startsWith(name+'/'))
211
- o[k.slice(name.length+1)] = masks[k]
208
+ else if (k.startsWith(virtualBasename+'/'))
209
+ o[k.slice(virtualBasename.length+1)] = masks[k]
210
if (Object.keys(o).length)
211
item.masks = o
212
}