avoid indefinitely long delay when searching on Windows
Massimo Melina committed
Sep 25, 2024 at 22:09 UTC
9292a450a42cae750e988c510b92f986bbbb7fcf
5 files changed
+45
-39
admin/src/OptionsPage.ts
+1
-1
@@ -210,7 +210,7 @@ export default function OptionsPage() {
210
label: "Calculate ZIP size for", helperText: "If time is not enough, the browser will not show download percentage" },
211
212
{ k: 'descript_ion', comp: BoolField, ...isWindows && { sm: 4, md: 3 }, label: "Enable comments", helperText: "In file DESCRIPT.ION" },
213
- { k: 'show_hidden_files', comp: BoolField, sm: 4, md: 3, helperText: isWindows && "Showing makes search faster" },
213
+ { k: 'show_hidden_files', comp: BoolField, sm: 4, md: 3 },
214
{ k: 'descript_ion_encoding', sm: 4, md: 6, label: "Encoding of file DESCRIPT.ION", comp: SelectField, disabled: !values.descript_ion,
215
options: ['utf8',720,775,819,850,852,862,869,874,808, ..._.range(1250,1257),10029,20866,21866] },
216
package.json
+2
-1
@@ -59,6 +59,7 @@
59
"central.json",
60
"admin/**/*",
61
"frontend/**/*",
62
+ "**/node_modules/fswin/x64/*",
63
"**/node_modules/axios/dist/node/*"
64
],
65
"targets": [
@@ -78,9 +79,9 @@
79
"find-process": "^1.4.7",
80
"formidable": "^3.5.1",
81
"fs-x-attributes": "^1.0.2",
82
+ "fswin": "^3.24.829",
83
"iconv-lite": "^0.6.3",
84
"ip2location-nodejs": "^9.6.0",
83
- "is-hidden-file": "^1.1.2",
85
"koa": "^2.13.4",
86
"koa-compress": "^5.1.0",
87
"koa-mount": "^4.0.0",
prune_modules.js
+3
-7
@@ -12,13 +12,9 @@ fs.renameSync(ya+'do_c', ya+'doc')
12
13
process.chdir(dist)
14
console.log('more pruning')
15
-fs.rmSync(nm+'node-forge/dist', {recursive:true})
16
-fs.rmSync(nm+'node-forge/flash', {recursive:true})
17
-fs.rmSync(nm+'axios/lib', {recursive:true})
18
-fs.rmSync(nm+'react', {recursive:true})
19
-fs.rmSync(nm+'yaml/browser', {recursive:true})
20
-fs.rmSync(nm+'limiter/dist/esm', {recursive:true})
21
-for (const fn of glob.sync(['**/*.map', '**/*.tsbuildinfo', '*.bak']))
15
+for (const f of ['fswin/ia32', 'fswin/arm64', 'node-forge/dist', 'node-forge/flash', 'axios/lib', 'react', 'yaml/browser', 'limiter/dist/esm'])
16
+ fs.rmSync(nm+f, {recursive:true})
17
+for (const fn of glob.sync(['**/*.map', '**/*.tsbuildinfo', '**/*.bak', '**/*.ts', '**/license', '**/*.md']))
18
fs.unlinkSync(fn)
19
20
console.log('pruning lodash')
src/dirStream.ts
+31
-27
@@ -1,11 +1,11 @@
1
import { makeQ } from './makeQ'
2
import { stat, readdir } from 'fs/promises'
3
-import { runCmd } from './util-os'
3
import { IS_WINDOWS } from './const'
4
import { join } from 'path'
5
import { Readable } from 'stream'
7
-import { DAY, pendingPromise } from './cross'
6
+import { pendingPromise } from './cross'
7
import { Stats, Dirent } from 'node:fs'
8
+import fswin from 'fswin'
9
10
export interface DirStreamEntry extends Dirent {
11
closingBranch?: Promise<string>
@@ -18,8 +18,6 @@ export function createDirStream(startPath: string, { depth=0, hidden=true }) {
18
let stopped = false
19
let started = false
20
const closingQ: string[] = []
21
- const hiddenRoot = !hidden && IS_WINDOWS && getWindowsHiddenFiles(startPath) // produce first level faster
22
- const hiddenDeep = hiddenRoot && depth && getWindowsHiddenFiles(startPath, true)
21
const stream = new Readable({
22
objectMode: true,
23
read() {
@@ -50,26 +48,46 @@ export function createDirStream(startPath: string, { depth=0, hidden=true }) {
48
if (stopped) return
49
const base = join(startPath, path)
50
const subDirsDone: Promise<any>[] = []
53
- let last: DirStreamEntry | undefined = undefined
51
let n = 0
55
- for await (let entry of await readdir(base, { withFileTypes: true })) {
52
+ let last: DirStreamEntry | undefined
53
+ if (IS_WINDOWS) { // use native apis to read 'hidden' attribute
54
+ const entries = await new Promise<fswin.Find.File[]>(res => fswin.find(base + '\\*', res))
55
+ const methods = {
56
+ isDir: false,
57
+ isFile(){ return !this.isDir },
58
+ isDirectory(){ return this.isDir },
59
+ isBlockDevice(){ return false },
60
+ isCharacterDevice() { return false },
61
+ }
62
+ for (const f of entries) {
63
+ if (stopped) break
64
+ if (!hidden && f.IS_HIDDEN) continue
65
+ work(Object.assign(Object.create(methods), {
66
+ isDir: f.IS_DIRECTORY,
67
+ name: f.LONG_NAME,
68
+ stats: { size: f.SIZE, ctime: f.CREATION_TIME, mtime: f.LAST_WRITE_TIME } as Stats
69
+ }))
70
+ }
71
+ }
72
+ else for await (let entry of await readdir(base, { withFileTypes: true })) {
73
if (stopped) break
57
- if (!IS_WINDOWS && !hidden && entry.name[0] === '.')
74
+ if (!hidden && entry.name[0] === '.')
75
continue
76
const stats = entry.isSymbolicLink() && await stat(join(base, entry.name)).catch(() => null)
77
if (stats === null) continue
78
if (stats)
79
entry = new DirentFromStats(entry.name, stats)
80
+ const expanded: DirStreamEntry = entry
81
+ if (stats)
82
+ expanded.stats = stats
83
+ work(expanded)
84
+ }
85
+
86
+ function work(entry: DirStreamEntry) {
87
entry.path = (path && path + '/') + entry.name
64
- const hiddenFiles = await (path && hiddenDeep || hiddenRoot)
65
- if (hiddenFiles && hiddenFiles.includes(entry.path))
66
- continue
88
if (last && closingQ.length) // pending entries
89
last.closingBranch = Promise.resolve(closingQ.shift()!)
90
last = entry
70
- const expanded: DirStreamEntry = entry
71
- if (stats)
72
- expanded.stats = stats
91
if (depth > 0 && entry.isDirectory()) {
92
const branchDone = pendingPromise() // per-job
93
const job = () =>
@@ -97,20 +115,6 @@ export function createDirStream(startPath: string, { depth=0, hidden=true }) {
115
}
116
}
117
100
-let lastNotice = 0
101
-async function getWindowsHiddenFiles(path: string, depth=false) {
102
- const t = Date.now()
103
- const out = await runCmd('dir', ['/ah', '/b', depth ? '/s' : '/c', path.replaceAll('/', '\\')]) // cannot pass '', so we pass /c as a noop parameter
104
- .catch(()=>'') // error in case of no matching file
105
- const now = Date.now()
106
- if ((now - t) > 10_000 && (now - lastNotice) > DAY) {
107
- lastNotice = now
108
- console.log("A file list was heavily delayed. You can avoid this by enabling the option to show hidden files.")
109
- }
110
- const slice = !depth ? 0 : path.length + (path.at(-1) === '\\' ? 0 : 1)
111
- return out.trimEnd().split('\n').map(x => x.slice(slice).replaceAll('\\', '/'))
112
-}
113
-
118
type DirentStatsKeysIntersection = keyof Dirent & keyof Stats;
119
const kStats = Symbol('stats')
120
// Adapting an internal class in Node.js to mimic the behavior of `Dirent` when creating it manually from `Stats`.
src/vfs.ts
+8
-3
@@ -4,7 +4,7 @@ import fs from 'fs/promises'
4
import { basename, dirname, join, resolve } from 'path'
5
import {
6
dirStream, getOrSet, isDirectory, makeMatcher, setHidden, onlyTruthy, isValidFileName, throw_, VfsPerms, Who,
7
- isWhoObject, WHO_ANY_ACCOUNT, defaultPerms, PERM_KEYS, removeStarting, HTTP_SERVER_ERROR, try_
7
+ isWhoObject, WHO_ANY_ACCOUNT, defaultPerms, PERM_KEYS, removeStarting, HTTP_SERVER_ERROR, try_, _log
8
} from './misc'
9
import Koa from 'koa'
10
import _ from 'lodash'
@@ -14,7 +14,7 @@ import events from './events'
14
import { expandUsername } from './perm'
15
import { getCurrentUsername } from './auth'
16
import { Stats } from 'node:fs'
17
-import { isHiddenFile } from 'is-hidden-file'
17
+import fswin from 'fswin'
18
19
const showHiddenFiles = defineConfig('show_hidden_files', false)
20
@@ -115,7 +115,7 @@ export async function urlToNode(url: string, ctx?: Koa.Context, parent: VfsNode=
115
return urlToNode(rest, ctx, ret, getRest)
116
if (ret.source)
117
try {
118
- if (!showHiddenFiles.get() && isHiddenFile(ret.source))
118
+ if (!showHiddenFiles.get() && await isHiddenFile(ret.source))
119
throw 'hiddenFile'
120
const st = ret.stats || await fs.stat(ret.source) // check existence
121
ret.isFolder = st.isDirectory()
@@ -130,6 +130,11 @@ export async function urlToNode(url: string, ctx?: Koa.Context, parent: VfsNode=
130
return ret
131
}
132
133
+async function isHiddenFile(path: string) {
134
+ return IS_WINDOWS ? new Promise(res => fswin.getAttributes(path, x => res(x?.IS_HIDDEN)))
135
+ : path[0] === '.'
136
+}
137
+
138
export async function getNodeByName(name: string, parent: VfsNode) {
139
// does the tree node have a child that goes by this name, otherwise attempt disk
140
const child = parent.children?.find(isSameFilenameAs(name)) || childFromDisk()