fix: plugin/download-counter: slow conversion of big legacy files
Massimo Melina committed
Feb 2, 2025 at 14:05 UTC
d2b8cee116d9548b4c7861f3b6be2e1dcc80a2dd
1 file changed
+10
-7
plugins/download-counter/plugin.js
+10
-7
@@ -1,7 +1,7 @@
1
// other plugins can use ctx.state.download_counter_ignore to mark downloads that shouldn't be counted
2
3
exports.description = "Counts downloads for each file, and displays the total in the list or file menu"
4
-exports.version = 6.2 // reintroduced conversion of legacy data
4
+exports.version = 6.3 // fixed slow conversion of big legacy files
5
exports.apiRequired = 8.89 // openDb
6
7
exports.config = {
@@ -17,14 +17,17 @@ exports.configDialog = {
17
exports.init = async api => {
18
const db = await api.openDb('counters.kv', { defaultPutDelay: 5_000, maxPutDelay: 30_000 })
19
20
- try { // load legacy file
20
+ if (!db.size()) try { // load legacy file
21
const countersFile = 'counters.yaml'
22
const yaml = api.require('yaml')
23
- const { readFile, rename } = api.require('fs/promises')
24
- const data = await readFile(countersFile, 'utf8')
25
- for (const [k,v] of Object.entries(yaml.parse(data)))
26
- db.put(uri2key(k), v)
27
- rename(countersFile, countersFile + ' - old format, now converted, you can delete')
23
+ const input = api.require('fs').createReadStream(countersFile)
24
+ const readline = api.require('readline').createInterface({ input })
25
+ for await (const line of readline) {
26
+ const parsed = yaml.parse(line) // parsing a 3.7MB file as a whole takes 25x in my tests
27
+ for (const [k,v] of Object.entries(parsed))
28
+ db.put(uri2key(k), v)
29
+ }
30
+ api.require('fs').promises.rename(countersFile, countersFile + ' - old format, now converted, you can delete')
31
api.log("data converted")
32
}
33
catch(err) {