deep dir search will have a better order now (still mixed because of parallelization)
Massimo Melina committed
Mar 21, 2026 at 18:33 UTC
56f9270bc5f788b8f5aff8ebf4e9435e66378d4f
4 files changed
+48
-9
src/makeQ.ts
+10
-4
@@ -1,22 +1,28 @@
1
-export function makeQ(parallelization=1) {
1
+const asap = globalThis.setImmediate || setTimeout
2
+export function makeQ(parallelization=1, max=Infinity) {
3
const running = new Set<Promise<unknown>>()
4
const queued: Array<() => Promise<unknown>> = []
5
return {
6
add(toAdd: typeof queued[0]) {
7
+ if (queued.length >= max + parallelization - running.size) // we may have some free slots that will be used at the next tick
8
+ return false
9
queued.push(toAdd)
7
- setTimeout(startNextIfPossible) // avoid nesting/stacking of jobs
10
+ asap(startNextIfPossible) // avoid calling now, as it would cause nesting/stacking of jobs
11
+ return true
12
},
13
isWorking() { return running.size > 0 },
14
isFree() { return running.size < parallelization },
15
+ setMax(newMax: number) { max = newMax },
16
+ queueSize() { return queued.length },
17
}
18
function startNextIfPossible() {
19
while (running.size < parallelization) {
14
- const job = queued.pop()
20
+ const job = queued.shift()
21
if (!job) break // finished
22
const working = job() // start the job
23
if (!working) continue // it was canceled
24
running.add(working)
19
- working.then(() => {
25
+ working.finally(() => {
26
running.delete(working)
27
startNextIfPossible()
28
})
src/serveGuiAndSharedFiles.ts
+2
-2
@@ -70,7 +70,7 @@ export const serveSharedFiles: Koa.Middleware = async (ctx, next) => {
70
const decPath = safeDecodeURIComponent(path, '')
71
const fn = basename(decPath)
72
const folderUri = pathEncode(dirname(decPath)) // re-encode to get readable urls
73
- const folder = await urlToNode(folderUri, ctx, vfs, true)
73
+ const folder = await urlToNode(folderUri, ctx, vfs, true) // we don't require the folder to already exist, but to be mapped on disk AND to have proper permissions
74
if (!folder)
75
return sendErrorPage(ctx, HTTP_NOT_FOUND)
76
ctx.state.uploadPath = decPath
@@ -163,7 +163,7 @@ async function sendFolderList(node: VfsNode, ctx: Koa.Context) {
163
|| URL.protocol + '//' + URL.host + ctx.state.revProxyPath
164
prepend = base + pathEncode(decodeURI(ctx.path)) // redo the encoding our way, keeping unicode chars unchanged
165
}
166
- const walker = walkNode(node, { ctx, depth: depth === '*' ? Infinity : Number(depth), parallelizeRecursion: false })
166
+ const walker = walkNode(node, { ctx, depth: depth === '*' ? Infinity : Number(depth), parallelizeRecursion: false }) // parallelization produces out-of-order results, and we don't want it like that here
167
ctx.body = asyncGeneratorToReadable(filterMapGenerator(walker, async el => {
168
const isFolder = nodeIsFolder(el)
169
return !folders && isFolder ? undefined
src/vfs.ts
+1
-1
@@ -108,7 +108,7 @@ export async function urlToNode(
108
url: string,
109
ctx?: Koa.Context,
110
parent: VfsNode=vfs,
111
- resolveMissing?: true | ((rest: string) => any)
111
+ resolveMissing?: true | ((rest: string) => any) // true means missing path segments still resolve to temporary nodes with a computed source path (used by upload flows that create folders on write)
112
) : Promise<VfsNode | undefined> {
113
let initialSlashes = 0
114
while (url[initialSlashes] === '/')
tests/test.ts
+35
-2
@@ -2,8 +2,8 @@ import test, { describe, before, after } from 'node:test';
2
import { promisify } from 'util'
3
import { srpClientSequence } from '../src/srp'
4
import * as srp from 'tssrp6a'
5
-import { createReadStream, existsSync, readFileSync, statfsSync, statSync } from 'fs'
6
-import { basename, dirname, resolve } from 'path'
5
+import { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statfsSync, statSync } from 'fs'
6
+import { basename, dirname, join, resolve } from 'path'
7
import { exec } from 'child_process'
8
import _ from 'lodash'
9
import yaml from 'yaml'
@@ -70,6 +70,39 @@ describe('basics', () => {
70
test('list', reqList('/f1/', { inList:['f2/', 'page/'] }))
71
test('search', reqList('f1', { inList:['f2/'], outList:['page'] }, { search:'2' }))
72
test('search root', reqList('/', { inList:['cantListPage/'], outList:['cantListPage/page/'] }, { search:'page' }))
73
+ test('search.fifo order', async () => {
74
+ // deep search queues subdirectory jobs via makeQ; verify results come in FIFO order.
75
+ // tree: root/{d01..d10}/sub/ — with LIFO the sub/ entries reverse relative to parent order (tau≈-1).
76
+ const dir = resolve(__dirname, '_fifo_test')
77
+ const dirCount = 10 // well above dirQ parallelization (3)
78
+ for (let i = 1; i <= dirCount; i++) {
79
+ const name = `d${String(i).padStart(2, '0')}`
80
+ mkdirSync(join(dir, name, 'sub'), { recursive: true })
81
+ }
82
+ try {
83
+ await reqList('/tests/_fifo_test', {
84
+ cb(data: any) {
85
+ const names: string[] = data.list.map((x: any) => x.n)
86
+ const parents = names.filter((n: string) => !n.includes('/'))
87
+ const subs = names.filter((n: string) => n.endsWith('sub/')).map((n: string) => n.split('/')[0])
88
+ // Kendall's tau: +1 = same order (FIFO), -1 = reversed (LIFO)
89
+ let concordant = 0, discordant = 0
90
+ for (let i = 0; i < parents.length; i++)
91
+ for (let j = i + 1; j < parents.length; j++) {
92
+ const d = subs.indexOf(parents[i]) - subs.indexOf(parents[j])
93
+ if (d > 0) discordant++
94
+ else if (d < 0) concordant++
95
+ }
96
+ const tau = (concordant - discordant) / (concordant + discordant)
97
+ if (tau <= 0)
98
+ throw `search results not FIFO; tau=${tau.toFixed(2)}, parents: ${parents}, subs: ${subs}`
99
+ }
100
+ }, { search: '*' })()
101
+ }
102
+ finally {
103
+ rmSync(dir, { recursive: true, force: true })
104
+ }
105
+ })
106
test('download.mime', req('/f1/f2/alfa.txt', { re:/abcd/, mime:'text/plain' }))
107
test('download.not modified', async () => {
108
let lm = ''