admin/plugins: faster list on get-more
Massimo Melina committed
Dec 23, 2025 at 19:55 UTC
30b547d565a0f55a171bc5b06df8bc4812790ab1
4 files changed
+83
-37
src/AsapStream.ts
new
+44
@@ -0,0 +1,44 @@
1
+import { Readable } from 'stream'
2
+import { isAsyncIterable } from './cross'
3
+
4
+// produces as promises resolve, not sequentially
5
+export class AsapStream<T> extends Readable {
6
+ finished = false
7
+ constructor(private promises: Iterable<Promise<T> | T> | AsyncIterable<Promise<T> | T>) {
8
+ super({ objectMode: true })
9
+ }
10
+ _read() {
11
+ if (this.finished) return
12
+ this.finished = true
13
+ void (async () => {
14
+ const pending: Promise<T>[] = []
15
+ try {
16
+ if (isAsyncIterable(this.promises)) {
17
+ const iterator = this.promises[Symbol.asyncIterator]()
18
+ while (true) {
19
+ const { value, done } = await iterator.next()
20
+ if (done) break
21
+ const promise = Promise.resolve(value)
22
+ pending.push(promise)
23
+ promise.then(x => x !== undefined && this.push(x),
24
+ e => this.emit('error', e) )
25
+ }
26
+ }
27
+ else {
28
+ for (const p of this.promises) {
29
+ const promise = Promise.resolve(p)
30
+ pending.push(promise)
31
+ promise.then(x => x !== undefined && this.push(x),
32
+ e => this.emit('error', e) )
33
+ }
34
+ }
35
+ await Promise.allSettled(pending)
36
+ this.push(null)
37
+ }
38
+ catch (e) {
39
+ this.emit('error', e)
40
+ this.push(null)
41
+ }
42
+ })()
43
+ }
44
+}
src/cross.ts
+4
@@ -343,6 +343,10 @@ export function throw_(err: any): never {
343
throw err
344
}
345
346
+export function isAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): iterable is AsyncIterable<T> {
347
+ return Symbol.asyncIterator in iterable
348
+}
349
+
350
export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterator<IN>, filterMap: (el: IN) => Promise<OUT>) {
351
for await (const x of generator) {
352
const res:OUT = await filterMap(x)
src/github.ts
+34
-21
@@ -2,8 +2,7 @@
2
3
import events from './events'
4
import {
5
- httpString, httpStream, unzip, AsapStream, debounceAsync, asyncGeneratorToArray, retry, popKey, onlyTruthy, waitFor,
6
- HOUR, DAY
5
+ httpString, httpStream, unzip, AsapStream, debounceAsync, retry, popKey, onlyTruthy, waitFor, HOUR, DAY
6
} from './misc'
7
import {
8
DISABLING_SUFFIX, enablePlugin, findPluginByRepo, getInactivePlugins, getPluginInfo, isPluginRunning, mapPlugins,
@@ -240,25 +239,39 @@ async function isPluginBlacklisted(repo: string) {
239
}
240
241
export async function searchPlugins(text='', { skipRepos=[''] }={}) {
243
- // github doesn't allow complex search, so we have to do it multiple times and merge the results
244
- const searches = [
245
- ...text.split(' ').filter(Boolean).slice(0, 2).map(x => 'user:' + encodeURI(x)), // first 2 words can be the author of the plugin
246
- encodeURI(text), // search elsewhere, and results after the author search
247
- ]
248
- const list = await Promise.all(searches.map(x => asyncGeneratorToArray(apiGithubPaginated(`search/repositories?q=topic:hfs-plugin+${x}`))))
249
- const deduped = _.uniqBy(list.flat(), x => x.full_name)
250
- return new AsapStream(deduped.map(async it => { // using AsapStream we parallelize these promises and produce each result as it's ready
251
- const repo = it.full_name as string
252
- if (skipRepos.includes(repo) || await isPluginBlacklisted(repo)) return
253
- const pl = await readOnlineCompatiblePlugin(repo, it.default_branch).catch(() => undefined)
254
- if (!pl) return
255
- Object.assign(pl, { // inject some extra useful fields
256
- repo, // overwrite parsed value, that may be wrong
257
- downloading: downloading[repo],
258
- license: it.license?.spdx_id,
259
- }, _.pick(it, ['pushed_at', 'stargazers_count', 'default_branch']))
260
- return pl
261
- }))
242
+ const seen = new Set<string>()
243
+ return new AsapStream(pluginPromises())
244
+
245
+ async function *pluginPromises() {
246
+ // github doesn't allow complex search, so we have to do it multiple times and merge the results
247
+ const searches = [
248
+ ...text.split(' ').filter(Boolean).slice(0, 2).map(x => 'user:' + encodeURI(x)), // first 2 words can be the author of the plugin
249
+ encodeURI(text), // search elsewhere, and results after the author search
250
+ ]
251
+ for (const term of searches) {
252
+ for await (const it of apiGithubPaginated(`search/repositories?q=topic:hfs-plugin+${term}`)) {
253
+ const repo = it.full_name as string
254
+ if (!repo || seen.has(repo)) // avoid duplicates, as we search multiple times
255
+ continue
256
+ seen.add(repo)
257
+ if (skipRepos.includes(repo))
258
+ continue
259
+ yield (async () => {
260
+ if (await isPluginBlacklisted(repo))
261
+ return
262
+ const pl = await readOnlineCompatiblePlugin(repo, it.default_branch).catch(() => undefined)
263
+ if (!pl)
264
+ return
265
+ Object.assign(pl, {
266
+ repo,
267
+ downloading: downloading[repo],
268
+ license: it.license?.spdx_id,
269
+ }, _.pick(it, ['pushed_at', 'stargazers_count', 'default_branch']))
270
+ return pl
271
+ })()
272
+ }
273
+ }
274
+ }
275
}
276
277
export const alerts = storedMap.singleSync<string[]>('alerts', [])
src/misc.ts
+1
-16
@@ -8,6 +8,7 @@ export * from './util-files'
8
export * from './fileAttr'
9
export * from './cross'
10
export * from './debounceAsync'
11
+export * from './AsapStream'
12
import { Readable, Transform } from 'stream'
13
import { SocketAddress, BlockList } from 'node:net'
14
import { ApiError } from './apiMiddleware'
@@ -103,22 +104,6 @@ export function asyncGeneratorToReadable<T>(generator: AsyncIterable<T>) {
104
})
105
}
106
106
-// produces as promises resolve, not sequentially
107
-export class AsapStream<T> extends Readable {
108
- finished = false
109
- constructor(private promises: Promise<T>[]) {
110
- super({ objectMode: true })
111
- }
112
- _read() {
113
- if (this.finished) return
114
- this.finished = true
115
- for (const p of this.promises)
116
- p.then(x => x !== undefined && this.push(x),
117
- e => this.emit('error', e) )
118
- Promise.allSettled(this.promises).then(() => this.push(null))
119
- }
120
-}
121
-
107
export function apiAssertTypes(paramsByType: { [type:string]: { [name:string]: any } }) {
108
for (const [types,params] of Object.entries(paramsByType)) {
109
if (!_.isPlainObject(params))