better code: split files
Massimo Melina committed
Jun 29, 2022 at 22:14 UTC
64a99854c0049c230a19bdbc7a0b302a77546a78
5 files changed
+210
-200
server/src/debounceAsync.ts
new
+51
@@ -0,0 +1,51 @@
1
+// like lodash.debounce, but also avoids async invocations to overlap
2
+export default function debounceAsync<CB extends (...args: any[]) => Promise<R>, R>(
3
+ callback: CB,
4
+ wait: number=100,
5
+ { leading=false, maxWait=Infinity }={}
6
+) {
7
+ let started = 0 // latest callback invocation
8
+ let runningCallback: Promise<R> | undefined // latest callback invocation result
9
+ let runningDebouncer: Promise<R | undefined> // latest wrapper invocation
10
+ let waitingSince = 0 // we are delaying invocation since
11
+ let whoIsWaiting: undefined | any[] // args' array object identifies the pending instance, and incidentally stores args
12
+ const interceptingWrapper = (...args:any[]) => runningDebouncer = debouncer.apply(null, args)
13
+ return Object.assign(interceptingWrapper, {
14
+ cancel: () => {
15
+ waitingSince = 0
16
+ whoIsWaiting = undefined
17
+ },
18
+ flush: () => runningCallback ?? exec(),
19
+ })
20
+
21
+ async function debouncer(...args:any[]) {
22
+ if (runningCallback)
23
+ return await runningCallback
24
+ whoIsWaiting = args
25
+ waitingSince ||= Date.now()
26
+ const waitingCap = maxWait - (Date.now() - (waitingSince || started))
27
+ const waitFor = Math.min(waitingCap, leading ? wait - (Date.now() - started) : wait)
28
+ if (waitFor > 0)
29
+ await new Promise(resolve => setTimeout(resolve, waitFor))
30
+ if (!whoIsWaiting) // canceled
31
+ return void(waitingSince = 0)
32
+ if (whoIsWaiting !== args) // another fresher call is waiting
33
+ return runningDebouncer
34
+ return await exec()
35
+ }
36
+
37
+ async function exec() {
38
+ if (!whoIsWaiting) return
39
+ waitingSince = 0
40
+ started = Date.now()
41
+ try {
42
+ runningCallback = callback.apply(null, whoIsWaiting)
43
+ return await runningCallback
44
+ }
45
+ finally {
46
+ whoIsWaiting = undefined
47
+ runningCallback = undefined
48
+ }
49
+ }
50
+}
51
+
server/src/misc.ts
+6
-200
@@ -1,20 +1,16 @@
1
// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
import { EventEmitter } from 'events'
4
-import fs from 'fs/promises'
5
-import { basename, dirname } from 'path'
6
-import { watch } from 'fs'
4
+import { basename } from 'path'
5
import _ from 'lodash'
8
-import { Readable } from 'stream'
6
import Koa from 'koa'
10
-import glob from 'fast-glob'
11
-import { IS_WINDOWS } from './const'
12
-import { execFile } from 'child_process'
7
import { Connection } from './connections'
8
import assert from 'assert'
15
-import https from 'node:https'
16
-import { RequestOptions } from 'https'
17
-import { IncomingMessage } from 'node:http'
9
+export * from './util-http'
10
+export * from './util-generators'
11
+export * from './util-files'
12
+import debounceAsync from './debounceAsync'
13
+export { debounceAsync }
14
15
export type Callback<IN=void, OUT=void> = (x:IN) => OUT
16
export type Dict<T = any> = Record<string, T>
@@ -23,16 +19,6 @@ export function enforceFinal(sub:string, s:string) {
19
return s.endsWith(sub) ? s : s+sub
20
}
21
26
-export async function isDirectory(path: string) {
27
- try { return (await fs.stat(path)).isDirectory() }
28
- catch { return false }
29
-}
30
-
31
-export async function isFile(path: string) {
32
- try { return (await fs.stat(path)).isFile() }
33
- catch { return false }
34
-}
35
-
22
export function prefix(pre:string, v:string|number, post:string='') {
23
return v ? pre+v+post : ''
24
}
@@ -53,46 +39,10 @@ export function wait(ms: number) {
39
return new Promise(res=> setTimeout(res,ms))
40
}
41
56
-export async function readFileBusy(path: string): Promise<string> {
57
- return fs.readFile(path, 'utf8').catch(e => {
58
- if ((e as any)?.code !== 'EBUSY')
59
- throw e
60
- console.debug('busy')
61
- return wait(100).then(()=> readFileBusy(path))
62
- })
63
-}
64
-
42
export function wantArray<T>(x?: void | T | T[]) {
43
return x == null ? [] : Array.isArray(x) ? x : [x]
44
}
45
69
-// callback can return undefined to skip element
70
-export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterator<IN>, filterMap: (el: IN) => Promise<OUT>) {
71
- for await (const x of generator) {
72
- const res:OUT = await filterMap(x)
73
- if (res !== undefined)
74
- yield res as Exclude<OUT,undefined>
75
- }
76
-}
77
-
78
-export async function asyncGeneratorToArray<T>(generator: AsyncIterable<T>): Promise<T[]> {
79
- const ret: T[] = []
80
- for await(const x of generator)
81
- ret.push(x)
82
- return ret
83
-}
84
-
85
-export function asyncGeneratorToReadable<T>(generator: AsyncIterable<T>) {
86
- const iterator = generator[Symbol.asyncIterator]()
87
- return new Readable({
88
- objectMode: true,
89
- read() {
90
- iterator.next().then(it =>
91
- this.push(it.done ? null : it.value))
92
- }
93
- })
94
-}
95
-
46
export function getOrSet<T>(o: Record<string,T>, k:string, creator:()=>T): T {
47
return k in o ? o[k]
48
: (o[k] = creator())
@@ -127,29 +77,6 @@ export function onFirstEvent(emitter:EventEmitter, events: string[], cb: (...arg
77
})
78
}
79
130
-export function watchDir(dir: string, cb: ()=>void) {
131
- try { watch(dir, cb) }
132
- catch {
133
- // failing watching the content of the dir, we try to monitor its parent, but filtering events only for our target dir
134
- const base = basename(dir)
135
- try {
136
- const watcher = watch(dirname(dir), (event,name) => {
137
- if (name !== base) return
138
- try {
139
- watch(dir, cb) // attempt at passing to a more specific watching
140
- watcher.close() // if we succeed, we give up the parent watching
141
- }
142
- catch {}
143
- cb()
144
- })
145
- }
146
- catch (e) {
147
- console.debug(String(e))
148
- return false
149
- }
150
- }
151
-}
152
-
80
export function pattern2filter(pattern: string){
81
const re = new RegExp(_.escapeRegExp(pattern), 'i')
82
return (s?:string) =>
@@ -187,65 +114,6 @@ export function onOff(em: EventEmitter, events: { [eventName:string]: (...args:
114
}
115
}
116
190
-// like lodash.debounce, but also avoids async invocations to overlap
191
-export function debounceAsync<CB extends (...args: any[]) => Promise<R>, R>(
192
- callback: CB,
193
- wait: number=100,
194
- { leading=false, maxWait=Infinity }={}
195
-) {
196
- let started = 0 // latest callback invocation
197
- let runningCallback: Promise<R> | undefined // latest callback invocation result
198
- let runningDebouncer: Promise<R | undefined> // latest wrapper invocation
199
- let waitingSince = 0 // we are delaying invocation since
200
- let whoIsWaiting: undefined | any[] // args' array object identifies the pending instance, and incidentally stores args
201
- const interceptingWrapper = (...args:any[]) => runningDebouncer = debouncer.apply(null, args)
202
- return Object.assign(interceptingWrapper, {
203
- cancel: () => {
204
- waitingSince = 0
205
- whoIsWaiting = undefined
206
- },
207
- flush: () => runningCallback ?? exec(),
208
- })
209
-
210
- async function debouncer(...args:any[]) {
211
- if (runningCallback)
212
- return await runningCallback
213
- whoIsWaiting = args
214
- waitingSince ||= Date.now()
215
- const waitingCap = maxWait - (Date.now() - (waitingSince || started))
216
- const waitFor = Math.min(waitingCap, leading ? wait - (Date.now() - started) : wait)
217
- if (waitFor > 0)
218
- await new Promise(resolve => setTimeout(resolve, waitFor))
219
- if (!whoIsWaiting) // canceled
220
- return void(waitingSince = 0)
221
- if (whoIsWaiting !== args) // another fresher call is waiting
222
- return runningDebouncer
223
- return await exec()
224
- }
225
-
226
- async function exec() {
227
- if (!whoIsWaiting) return
228
- waitingSince = 0
229
- started = Date.now()
230
- try {
231
- runningCallback = callback.apply(null, whoIsWaiting)
232
- return await runningCallback
233
- }
234
- finally {
235
- whoIsWaiting = undefined
236
- runningCallback = undefined
237
- }
238
- }
239
-}
240
-
241
-export function dirTraversal(s?: string) {
242
- return s && /(^|[/\\])\.\.($|[/\\])/.test(s)
243
-}
244
-
245
-export function isWindowsDrive(s?: string) {
246
- return s && /^[a-zA-Z]:$/.test(s)
247
-}
248
-
117
export function objRenameKey(o: Dict | undefined, from: string, to: string) {
118
if (!o || !o.hasOwnProperty(from) || from === to) return
119
o[to] = o[from]
@@ -266,43 +134,6 @@ export function isLocalHost(c: Connection | Koa.Context) {
134
return ip && (ip === '::1' || ip.endsWith('127.0.0.1'))
135
}
136
269
-export async function* dirStream(path: string) {
270
- const stats = await fs.stat(path)
271
- if (!stats.isDirectory())
272
- throw Error('ENOTDIR')
273
- const dirStream = glob.stream('*', {
274
- cwd: path,
275
- dot: true,
276
- onlyFiles: false,
277
- suppressErrors: true,
278
- })
279
- const skip = await getItemsToSkip(path)
280
- for await (let path of dirStream) {
281
- if (path instanceof Buffer)
282
- path = path.toString('utf8')
283
- if (skip?.includes(path))
284
- continue
285
- yield path
286
- }
287
-
288
- async function getItemsToSkip(path: string) {
289
- if (!IS_WINDOWS) return
290
- const out = await run('dir', ['/ah', '/b', path.replace(/\//g, '\\')])
291
- .catch(()=>'') // error in case of no matching file
292
- return out.split('\r\n').slice(0,-1)
293
- }
294
-}
295
-
296
-export function run(cmd: string, args: string[] = []): Promise<string> {
297
- return new Promise((resolve, reject) =>
298
- execFile('cmd', ['/c', cmd, ...args], (err, stdout) => {
299
- if (err)
300
- reject(err)
301
- else
302
- resolve(stdout)
303
- }))
304
-}
305
-
137
export function same(a: any, b: any) {
138
try {
139
assert.deepStrictEqual(a, b)
@@ -311,31 +142,6 @@ export function same(a: any, b: any) {
142
catch { return false }
143
}
144
314
-export function httpsString(url: string, options:RequestOptions={}): Promise<IncomingMessage & { ok: boolean, body: string }> {
315
- return httpsStream(url, options).then(res =>
316
- new Promise(resolve => {
317
- let buf = ''
318
- res.on('data', chunk => buf += chunk.toString())
319
- res.on('end', () => resolve(Object.assign(res, {
320
- ok: (res.statusCode || 400) < 400,
321
- body: buf
322
- })))
323
- })
324
- )
325
-}
326
-
327
-export function httpsStream(url: string, options:RequestOptions={}): Promise<IncomingMessage> {
328
- return new Promise((resolve, reject) => {
329
- https.request(url, options, res => {
330
- if (!res.statusCode || res.statusCode >= 400)
331
- throw res
332
- if (res.statusCode === 302 && res.headers.location)
333
- return resolve(httpsStream(res.headers.location, options))
334
- resolve(res)
335
- }).on('error', reject).end()
336
- })
337
-}
338
-
145
export function tryJson(s?: string) {
146
try { return s && JSON.parse(s) }
147
catch {}
server/src/util-files.ts
new
+95
@@ -0,0 +1,95 @@
1
+import fs from 'fs/promises'
2
+import { wait } from './misc'
3
+import { watch } from 'fs'
4
+import { basename, dirname } from 'path'
5
+import glob from 'fast-glob'
6
+import { IS_WINDOWS } from './const'
7
+import { execFile } from 'child_process'
8
+
9
+export async function isDirectory(path: string) {
10
+ try { return (await fs.stat(path)).isDirectory() }
11
+ catch { return false }
12
+}
13
+
14
+export async function isFile(path: string) {
15
+ try { return (await fs.stat(path)).isFile() }
16
+ catch { return false }
17
+}
18
+
19
+export async function readFileBusy(path: string): Promise<string> {
20
+ return fs.readFile(path, 'utf8').catch(e => {
21
+ if ((e as any)?.code !== 'EBUSY')
22
+ throw e
23
+ console.debug('busy')
24
+ return wait(100).then(()=> readFileBusy(path))
25
+ })
26
+}
27
+
28
+export function watchDir(dir: string, cb: ()=>void) {
29
+ try { watch(dir, cb) }
30
+ catch {
31
+ // failing watching the content of the dir, we try to monitor its parent, but filtering events only for our target dir
32
+ const base = basename(dir)
33
+ try {
34
+ const watcher = watch(dirname(dir), (event,name) => {
35
+ if (name !== base) return
36
+ try {
37
+ watch(dir, cb) // attempt at passing to a more specific watching
38
+ watcher.close() // if we succeed, we give up the parent watching
39
+ }
40
+ catch {}
41
+ cb()
42
+ })
43
+ }
44
+ catch (e) {
45
+ console.debug(String(e))
46
+ return false
47
+ }
48
+ }
49
+}
50
+
51
+export function dirTraversal(s?: string) {
52
+ return s && /(^|[/\\])\.\.($|[/\\])/.test(s)
53
+}
54
+
55
+export function isWindowsDrive(s?: string) {
56
+ return s && /^[a-zA-Z]:$/.test(s)
57
+}
58
+
59
+export async function* dirStream(path: string) {
60
+ const stats = await fs.stat(path)
61
+ if (!stats.isDirectory())
62
+ throw Error('ENOTDIR')
63
+ const dirStream = glob.stream('*', {
64
+ cwd: path,
65
+ dot: true,
66
+ onlyFiles: false,
67
+ suppressErrors: true,
68
+ })
69
+ const skip = await getItemsToSkip(path)
70
+ for await (let path of dirStream) {
71
+ if (path instanceof Buffer)
72
+ path = path.toString('utf8')
73
+ if (skip?.includes(path))
74
+ continue
75
+ yield path
76
+ }
77
+
78
+ async function getItemsToSkip(path: string) {
79
+ if (!IS_WINDOWS) return
80
+ const out = await run('dir', ['/ah', '/b', path.replace(/\//g, '\\')])
81
+ .catch(()=>'') // error in case of no matching file
82
+ return out.split('\r\n').slice(0,-1)
83
+ }
84
+}
85
+
86
+export function run(cmd: string, args: string[] = []): Promise<string> {
87
+ return new Promise((resolve, reject) =>
88
+ execFile('cmd', ['/c', cmd, ...args], (err, stdout) => {
89
+ if (err)
90
+ reject(err)
91
+ else
92
+ resolve(stdout)
93
+ }))
94
+}
95
+
server/src/util-generators.ts
new
+29
@@ -0,0 +1,29 @@
1
+// callback can return undefined to skip element
2
+import { Readable } from 'stream'
3
+
4
+export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterator<IN>, filterMap: (el: IN) => Promise<OUT>) {
5
+ for await (const x of generator) {
6
+ const res:OUT = await filterMap(x)
7
+ if (res !== undefined)
8
+ yield res as Exclude<OUT,undefined>
9
+ }
10
+}
11
+
12
+export async function asyncGeneratorToArray<T>(generator: AsyncIterable<T>): Promise<T[]> {
13
+ const ret: T[] = []
14
+ for await(const x of generator)
15
+ ret.push(x)
16
+ return ret
17
+}
18
+
19
+export function asyncGeneratorToReadable<T>(generator: AsyncIterable<T>) {
20
+ const iterator = generator[Symbol.asyncIterator]()
21
+ return new Readable({
22
+ objectMode: true,
23
+ read() {
24
+ iterator.next().then(it =>
25
+ this.push(it.done ? null : it.value))
26
+ }
27
+ })
28
+}
29
+
server/src/util-http.ts
new
+29
@@ -0,0 +1,29 @@
1
+import { RequestOptions } from 'https'
2
+import { IncomingMessage } from 'node:http'
3
+import https from 'node:https'
4
+
5
+export function httpsString(url: string, options:RequestOptions={}): Promise<IncomingMessage & { ok: boolean, body: string }> {
6
+ return httpsStream(url, options).then(res =>
7
+ new Promise(resolve => {
8
+ let buf = ''
9
+ res.on('data', chunk => buf += chunk.toString())
10
+ res.on('end', () => resolve(Object.assign(res, {
11
+ ok: (res.statusCode || 400) < 400,
12
+ body: buf
13
+ })))
14
+ })
15
+ )
16
+}
17
+
18
+export function httpsStream(url: string, options:RequestOptions={}): Promise<IncomingMessage> {
19
+ return new Promise((resolve, reject) => {
20
+ https.request(url, options, res => {
21
+ if (!res.statusCode || res.statusCode >= 400)
22
+ throw res
23
+ if (res.statusCode === 302 && res.headers.location)
24
+ return resolve(httpsStream(res.headers.location, options))
25
+ resolve(res)
26
+ }).on('error', reject).end()
27
+ })
28
+}
29
+