better code: split files
Massimo Melina committed
Jan 8, 2024 at 09:43 UTC
76f354f718a540e2b47562dbdb516e06dc870f3d
9 files changed
+135
-124
src/SendList.ts
new
+120
@@ -0,0 +1,120 @@
1
+import { Readable } from 'stream'
2
+import _ from 'lodash'
3
+import { LIST, wantArray } from './cross'
4
+import { Context } from 'koa'
5
+import { onOff } from './misc'
6
+import events from './events'
7
+
8
+type SendListFunc<T> = (list:SendListReadable<T>) => void
9
+// offer an api for a generic dynamic list. Suitable to be the result of an api.
10
+export class SendListReadable<T> extends Readable {
11
+ protected lastError: string | number | undefined
12
+ protected buffer: any[] = []
13
+ protected processBuffer: _.DebouncedFunc<any>
14
+ protected sent: undefined | T[]
15
+ constructor({ addAtStart, doAtStart, bufferTime, onEnd, diff }:
16
+ { bufferTime?: number, addAtStart?: T[], doAtStart?: SendListFunc<T>, onEnd?: SendListFunc<T>, diff?: boolean }={}) {
17
+ super({ objectMode: true, read(){} })
18
+ if (!bufferTime)
19
+ bufferTime = 200
20
+ if (diff)
21
+ this.sent = []
22
+ this.processBuffer = _.debounce(() => {
23
+ const {sent} = this
24
+ if (sent)
25
+ this.buffer = this.buffer.filter(([cmd, a, b]) => {
26
+ if (cmd === LIST.add)
27
+ return sent.push(...wantArray(a))
28
+ if (cmd === LIST.remove)
29
+ return _.remove(sent, a)
30
+ if (cmd !== LIST.update)
31
+ return true
32
+ const found = _.find(sent, a) as any
33
+ if (!found) return
34
+ for (const k in b)
35
+ if (b[k] === found[k])
36
+ delete b[k]
37
+ else {
38
+ found[k] = b[k]
39
+ b[k] ??= null // go and delete it, remotely
40
+ }
41
+ return !_.isEmpty(b)
42
+ })
43
+ if (!this.buffer.length) return
44
+ this.push(this.buffer)
45
+ this.buffer = []
46
+ }, bufferTime, { maxWait: bufferTime })
47
+ this.on('end', () => {
48
+ onEnd?.(this)
49
+ this.destroy()
50
+ })
51
+ setTimeout(() => doAtStart?.(this)) // work later, when list object has been received by Koa
52
+ if (addAtStart) {
53
+ for (const x of addAtStart)
54
+ this.add(x)
55
+ this.ready()
56
+ }
57
+ }
58
+ protected _push(rec: any) {
59
+ this.buffer.push(rec)
60
+ if (this.buffer.length > 10_000) // hard limit
61
+ this.processBuffer.flush()
62
+ else
63
+ this.processBuffer()
64
+ }
65
+ add(rec: T | T[]) {
66
+ this._push([LIST.add, rec])
67
+ }
68
+ remove(search: Partial<T>) {
69
+ const match = _.matches(search)
70
+ const idx = _.findIndex(this.buffer, x => match(x[1]))
71
+ const found = this.buffer[idx]
72
+ const op = found?.[0]
73
+ if (op === LIST.remove) return
74
+ if (found) {
75
+ this.buffer.splice(idx, 1)
76
+ if (op === LIST.add) return
77
+ }
78
+ this._push([LIST.remove, search])
79
+ }
80
+ update(search: Partial<T>, change: Partial<T>) {
81
+ if (_.isEmpty(change)) return
82
+ const match = _.matches(search)
83
+ const found = _.find(this.buffer, x => match(x[1]))
84
+ const op = found?.[0]
85
+ if (op === LIST.remove) return
86
+ if (op === LIST.add || op === LIST.update)
87
+ return Object.assign(found[op === LIST.add ? 1 : 2], change)
88
+ return this._push([LIST.update, search, change])
89
+ }
90
+ ready() { // useful to indicate the end of an initial phase, but we leave open for updates
91
+ this._push([LIST.ready])
92
+ }
93
+ custom(name: string, data: any) {
94
+ this._push(data === undefined ? [name] : [name, data])
95
+ }
96
+ props(props: object) {
97
+ this._push([LIST.props, props])
98
+ }
99
+ error(msg: NonNullable<typeof this.lastError>, close=false, props?: object) {
100
+ this._push([LIST.error, msg, props])
101
+ this.lastError = msg
102
+ if (close)
103
+ this.close()
104
+ }
105
+ getLastError() {
106
+ return this.lastError
107
+ }
108
+ close() {
109
+ this.processBuffer.flush()
110
+ this.push(null)
111
+ }
112
+ events(ctx: Context, eventMap: Parameters<typeof onOff>[1]) {
113
+ const off = onOff(events, eventMap)
114
+ ctx.res.once('close', off)
115
+ return this
116
+ }
117
+ isClosed() {
118
+ return this.destroyed
119
+ }
120
+}
src/adminApis.ts
+2
-1
@@ -1,6 +1,6 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
-import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'
3
+import { ApiError, ApiHandlers } from './apiMiddleware'
4
import { configFile, defineConfig, getWholeConfig, setConfig } from './config'
5
import { getBaseUrlOrDefault, getIps, getServerStatus, getUrls } from './listen'
6
import {
@@ -38,6 +38,7 @@ import { resolve } from 'path'
38
import { getErrorSections } from './errorPages'
39
import { ip2country } from './geo'
40
import { roots } from './roots'
41
+import { SendListReadable } from './SendList'
42
43
export const adminApis: ApiHandlers = {
44
src/api.get_file_list.ts
+2
-1
@@ -2,7 +2,7 @@
2
3
import { applyParentToChild, getNodeName, hasPermission, masksCouldGivePermission, nodeIsDirectory,
4
statusCodeForMissingPerm, urlToNode, VfsNode, walkNode } from './vfs'
5
-import { ApiError, ApiHandler, SendListReadable } from './apiMiddleware'
5
+import { ApiError, ApiHandler } from './apiMiddleware'
6
import { stat } from 'fs/promises'
7
import { mapPlugins } from './plugins'
8
import { asyncGeneratorToArray, dirTraversal, pattern2filter, WHO_NO_ONE } from './misc'
@@ -13,6 +13,7 @@ import { basename } from 'path'
13
import { updateConnectionForCtx } from './connections'
14
import { ctxAdminAccess } from './adminApis'
15
import { dontOverwriteUploading } from './upload'
16
+import { SendListReadable } from './SendList'
17
18
export interface DirEntry { n:string, s?:number, m?:Date, c?:Date, p?: string, comment?: string, web?: boolean, url?: string }
19
src/api.lang.ts
+2
-1
@@ -1,6 +1,6 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
-import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'
3
+import { ApiError, ApiHandlers } from './apiMiddleware'
4
import _ from 'lodash'
5
import glob from 'fast-glob'
6
import { readFile, rm, writeFile } from 'fs/promises'
@@ -8,6 +8,7 @@ import { HTTP_BAD_REQUEST, HTTP_NOT_ACCEPTABLE, HTTP_SERVER_ERROR } from './cons
8
import { tryJson } from './misc'
9
import { code2file, file2code } from './lang'
10
import EMBEDDED_TRANSLATIONS from './langs/embedded'
11
+import { SendListReadable } from './SendList'
12
13
const apis: ApiHandlers = {
14
src/api.monitor.ts
+2
-1
@@ -3,10 +3,11 @@
3
import _ from 'lodash'
4
import { Connection, getConnections } from './connections'
5
import { pendingPromise, shortenAgent, wait } from './misc'
6
-import { ApiHandlers, SendListReadable } from './apiMiddleware'
6
+import { ApiHandlers } from './apiMiddleware'
7
import Koa from 'koa'
8
import { totalGot, totalInSpeed, totalOutSpeed, totalSent } from './throttler'
9
import { getCurrentUsername } from './auth'
10
+import { SendListReadable } from './SendList'
11
12
export default {
13
src/api.plugins.ts
+2
-1
@@ -8,11 +8,12 @@ import {
8
import _ from 'lodash'
9
import assert from 'assert'
10
import { Callback, newObj, onOff, waitFor } from './misc'
11
-import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'
11
+import { ApiError, ApiHandlers } from './apiMiddleware'
12
import events from './events'
13
import { rm } from 'fs/promises'
14
import { downloadPlugin, getFolder2repo, readOnlineCompatiblePlugin, readOnlinePlugin, searchPlugins } from './github'
15
import { HTTP_FAILED_DEPENDENCY, HTTP_NOT_FOUND, HTTP_SERVER_ERROR } from './const'
16
+import { SendListReadable } from './SendList'
17
18
const apis: ApiHandlers = {
19
src/api.vfs.ts
+2
-1
@@ -4,7 +4,7 @@ import { getNodeName, isSameFilenameAs, nodeIsDirectory, saveVfs, urlToNode, vfs
4
permsFromParent, nodeIsLink } from './vfs'
5
import _ from 'lodash'
6
import { mkdir, stat } from 'fs/promises'
7
-import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'
7
+import { ApiError, ApiHandlers } from './apiMiddleware'
8
import { dirname, extname, join, resolve } from 'path'
9
import { dirStream, enforceFinal, isDirectory, isWindowsDrive, makeMatcher, PERM_KEYS, VfsNodeAdminSend } from './misc'
10
import { IS_WINDOWS, HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVER_ERROR, HTTP_CONFLICT, HTTP_NOT_ACCEPTABLE } from './const'
@@ -13,6 +13,7 @@ import { Stats } from 'fs'
13
import { getBaseUrlOrDefault, getServerStatus } from './listen'
14
import { promisify } from 'util'
15
import { execFile } from 'child_process'
16
+import { SendListReadable } from './SendList'
17
18
// to manipulate the tree we need the original node
19
async function urlToNodeOriginal(uri: string) {
src/apiMiddleware.ts
+1
-117
@@ -3,10 +3,8 @@
3
import Koa from 'koa'
4
import createSSE from './sse'
5
import { Readable } from 'stream'
6
-import { asyncGeneratorToReadable, LIST, onOff, removeStarting, wantArray } from './misc'
7
-import events from './events'
6
+import { asyncGeneratorToReadable, removeStarting } from './misc'
7
import { HTTP_BAD_REQUEST, HTTP_FOOL, HTTP_NOT_FOUND } from './const'
9
-import _ from 'lodash'
8
import { defineConfig } from './config'
9
10
export class ApiError extends Error {
@@ -79,117 +77,3 @@ export function apiMiddleware(apis: ApiHandlers) : Koa.Middleware {
77
function isAsyncGenerator(x: any): x is AsyncGenerator {
78
return typeof (x as AsyncGenerator)?.next === 'function'
79
}
82
-
83
-// offer an api for a generic dynamic list. Suitable to be the result of an api.
84
-type SendListFunc<T> = (list:SendListReadable<T>) => void
85
-export class SendListReadable<T> extends Readable {
86
- protected lastError: string | number | undefined
87
- protected buffer: any[] = []
88
- protected processBuffer: _.DebouncedFunc<any>
89
- protected sent: undefined | T[]
90
- constructor({ addAtStart, doAtStart, bufferTime, onEnd, diff }:
91
- { bufferTime?: number, addAtStart?: T[], doAtStart?: SendListFunc<T>, onEnd?: SendListFunc<T>, diff?: boolean }={}) {
92
- super({ objectMode: true, read(){} })
93
- if (!bufferTime)
94
- bufferTime = 200
95
- if (diff)
96
- this.sent = []
97
- this.processBuffer = _.debounce(() => {
98
- const {sent} = this
99
- if (sent)
100
- this.buffer = this.buffer.filter(([cmd, a, b]) => {
101
- if (cmd === LIST.add)
102
- return sent.push(...wantArray(a))
103
- if (cmd === LIST.remove)
104
- return _.remove(sent, a)
105
- if (cmd !== LIST.update)
106
- return true
107
- const found = _.find(sent, a) as any
108
- if (!found) return
109
- for (const k in b)
110
- if (b[k] === found[k])
111
- delete b[k]
112
- else {
113
- found[k] = b[k]
114
- b[k] ??= null // go and delete it, remotely
115
- }
116
- return !_.isEmpty(b)
117
- })
118
- if (!this.buffer.length) return
119
- this.push(this.buffer)
120
- this.buffer = []
121
- }, bufferTime, { maxWait: bufferTime })
122
- this.on('end', () => {
123
- onEnd?.(this)
124
- this.destroy()
125
- })
126
- setTimeout(() => doAtStart?.(this)) // work later, when list object has been received by Koa
127
- if (addAtStart) {
128
- for (const x of addAtStart)
129
- this.add(x)
130
- this.ready()
131
- }
132
- }
133
- protected _push(rec: any) {
134
- this.buffer.push(rec)
135
- if (this.buffer.length > 10_000) // hard limit
136
- this.processBuffer.flush()
137
- else
138
- this.processBuffer()
139
- }
140
- add(rec: T | T[]) {
141
- this._push([LIST.add, rec])
142
- }
143
- remove(search: Partial<T>) {
144
- const match = _.matches(search)
145
- const idx = _.findIndex(this.buffer, x => match(x[1]))
146
- const found = this.buffer[idx]
147
- const op = found?.[0]
148
- if (op === LIST.remove) return
149
- if (found) {
150
- this.buffer.splice(idx, 1)
151
- if (op === LIST.add) return
152
- }
153
- this._push([LIST.remove, search])
154
- }
155
- update(search: Partial<T>, change: Partial<T>) {
156
- if (_.isEmpty(change)) return
157
- const match = _.matches(search)
158
- const found = _.find(this.buffer, x => match(x[1]))
159
- const op = found?.[0]
160
- if (op === LIST.remove) return
161
- if (op === LIST.add || op === LIST.update)
162
- return Object.assign(found[op === LIST.add ? 1 : 2], change)
163
- return this._push([LIST.update, search, change])
164
- }
165
- ready() { // useful to indicate the end of an initial phase, but we leave open for updates
166
- this._push([LIST.ready])
167
- }
168
- custom(name: string, data: any) {
169
- this._push(data === undefined ? [name] : [name, data])
170
- }
171
- props(props: object) {
172
- this._push([LIST.props, props])
173
- }
174
- error(msg: NonNullable<typeof this.lastError>, close=false, props?: object) {
175
- this._push([LIST.error, msg, props])
176
- this.lastError = msg
177
- if (close)
178
- this.close()
179
- }
180
- getLastError() {
181
- return this.lastError
182
- }
183
- close() {
184
- this.processBuffer.flush()
185
- this.push(null)
186
- }
187
- events(ctx: Koa.Context, eventMap: Parameters<typeof onOff>[1]) {
188
- const off = onOff(events, eventMap)
189
- ctx.res.once('close', off)
190
- return this
191
- }
192
- isClosed() {
193
- return this.destroyed
194
- }
195
-}
src/frontEndApis.ts
+2
-1
@@ -1,6 +1,6 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3
-import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'
3
+import { ApiError, ApiHandlers } from './apiMiddleware'
4
import { get_file_list } from './api.get_file_list'
5
import * as api_auth from './api.auth'
6
import events from './events'
@@ -16,6 +16,7 @@ import { basename, dirname, join } from 'path'
16
import { getUploadMeta } from './upload'
17
import { apiAssertTypes } from './misc'
18
import { getCommentFor, setCommentFor } from './comments'
19
+import { SendListReadable } from './SendList'
20
21
export const frontEndApis: ApiHandlers = {
22
get_file_list,