descript_ion_encoding
Massimo Melina committed
Oct 21, 2023 at 02:41 UTC
12916ccffd2270e34a414f97e08ad374685f7e07
4 files changed
+33
-18
admin/src/OptionsPage.ts
+5
-2
@@ -4,7 +4,7 @@ import { Box, Button, FormHelperText } from '@mui/material';
4
import { createElement as h, Fragment, useEffect, useRef } from 'react';
5
import { apiCall, useApiEx } from './api'
6
import { state, useSnapState } from './state'
7
-import { Link } from 'react-router-dom'
7
+import { Link as RouterLink } from 'react-router-dom'
8
import { CardMembership, EditNote, Refresh, Warning } from '@mui/icons-material'
9
import { Dict, iconTooltip, InLink, LinkBtn, MAX_TILES_SIZE, modifiedSx, REPO_URL, ipLocalHost,
10
wait, wikiLink, with_, try_, ipForUrl } from './misc'
@@ -85,7 +85,7 @@ export default function OptionsPage() {
85
startIcon: h(Refresh),
86
}, "Reload"),
87
h(Button, { // @ts-ignore
88
- component: Link,
88
+ component: RouterLink,
89
to: "/edit",
90
startIcon: h(EditNote),
91
}, "Edit config file"),
@@ -162,6 +162,9 @@ export default function OptionsPage() {
162
{ k: 'admin_net', comp: NetmaskField, label: "Admin-panel accessible from", placeholder: "any address",
163
helperText: h(Fragment, {}, "IP address of browser machine. ", h(WildcardsSupported))
164
},
165
+ { k: 'descript_ion', comp: BoolField, label: "Support file DESCRIPT.ION", helperText: "Old file format, used for comments" },
166
+ { k: 'descript_ion_encoding', label: "Encoding of file DESCRIPT.ION", comp: SelectField, disabled: !values.descript_ion,
167
+ options: ['utf8',720,775,819,850,852,862,869,874,808, ..._.range(1250,1257),10029,20866,21866] },
168
{ k: 'mime', comp: ArrayField, label: false, reorder: true, prepend: true, sm: 12,
169
fields: [
170
{ k: 'k', label: "File mask", $width: 1, $column: {
package.json
+1
@@ -71,6 +71,7 @@
71
"find-process": "^1.4.7",
72
"formidable": "^3.5.1",
73
"fs-x-attributes": "^1.0.2",
74
+ "iconv-lite": "^0.6.3",
75
"koa": "^2.13.4",
76
"koa-compress": "^5.1.0",
77
"koa-mount": "^4.0.0",
src/comments.ts
+22
-11
@@ -1,13 +1,15 @@
1
import { defineConfig } from './config'
2
import { dirname, join } from 'path'
3
import { basename } from './cross'
4
-import { parseFile } from './util-files'
4
+import { parseFile, parseFileCache } from './util-files'
5
import { writeFile } from 'fs/promises'
6
import { singleFromBatch } from './misc'
7
import _ from 'lodash'
8
+import iconv from 'iconv-lite'
9
10
export const DESCRIPT_ION = 'descript.ion'
11
export const descriptIon = defineConfig('descript_ion', true)
12
+const descriptIonEncoding = defineConfig('descript_ion_encoding', 'utf8')
13
14
export async function getCommentFor(path?: string) {
15
return !path || !descriptIon.get() ? undefined
@@ -29,7 +31,8 @@ export const setCommentFor = singleFromBatch(async (jobs: [path: string, comment
31
let txt = ''
32
comments.forEach((c, f) =>
33
txt += (f.includes(' ') ? `"${f}"` : f) + ' ' + (c.includes('\n') ? c.replaceAll('\n', '\\n') + MULTILINE_SUFFIX : c) + '\n')
32
- await writeFile(join(folder, DESCRIPT_ION), txt)
34
+ const buffer = iconv.encode(txt, descriptIonEncoding.get())
35
+ await writeFile(join(folder, DESCRIPT_ION), buffer)
36
}))
37
})
38
@@ -39,13 +42,21 @@ export function areCommentsEnabled() {
42
43
const MULTILINE_SUFFIX = '\x04\xc2'
44
function readDescription(path: string) {
42
- return parseFile(join(path, DESCRIPT_ION), txt => new Map(txt.split('\n').map(line => {
43
- const quoted = line[0] === '"' ? 1 : 0
44
- const i = quoted ? line.indexOf('"', 2) + 1 : line.indexOf(' ')
45
- const fn = line.slice(quoted, i - quoted)
46
- let comment = line.slice(i + 1)
47
- if (comment.endsWith(MULTILINE_SUFFIX))
48
- comment = comment.slice(0, -2).replaceAll('\\n', '\n')
49
- return [fn, comment]
50
- })))
45
+ return parseFile(join(path, DESCRIPT_ION), raw =>
46
+ // decoding could also be done with native TextDecoder.decode, but we need iconv for the encoding anyway
47
+ new Map(iconv.decode(raw, descriptIonEncoding.get()).split('\n').map(line => {
48
+ const quoted = line[0] === '"' ? 1 : 0
49
+ const i = quoted ? line.indexOf('"', 2) + 1 : line.indexOf(' ')
50
+ const fn = line.slice(quoted, i - quoted)
51
+ let comment = line.slice(i + 1)
52
+ if (comment.endsWith(MULTILINE_SUFFIX))
53
+ comment = comment.slice(0, -2).replaceAll('\\n', '\n')
54
+ return [fn, comment]
55
+ })))
56
}
57
+
58
+descriptIonEncoding.sub(() => { // invalidate cache at encoding change
59
+ for (const k of parseFileCache.keys())
60
+ if (k.endsWith(DESCRIPT_ION))
61
+ parseFileCache.delete(k)
62
+})
\ No newline at end of file
src/util-files.ts
+5
-5
@@ -170,14 +170,14 @@ export async function loadFileAttr(path: string, k: string) {
170
}
171
172
// read and parse a file, caching unless timestamp has changed
173
-const cache = new Map<string, { ts: Date, parsed: unknown }>()
174
-export async function parseFile<T>(path: string, parse: (raw: string) => T) {
173
+export const parseFileCache = new Map<string, { ts: Date, parsed: unknown }>()
174
+export async function parseFile<T>(path: string, parse: (raw: Buffer) => T) {
175
const { mtime: ts } = await stat(path)
176
- const cached = cache.get(path)
176
+ const cached = parseFileCache.get(path)
177
if (cached && Number(ts) === Number(cached.ts))
178
return cached.parsed as T
179
- const raw = await readFile(path, 'utf8')
179
+ const raw = await readFile(path)
180
const parsed = parse(raw)
181
- cache.set(path, { ts, parsed })
181
+ parseFileCache.set(path, { ts, parsed })
182
return parsed
183
}