| 1 | import { defineConfig } from './config' |
| 2 | import { dirname, basename, join } from 'path' |
| 3 | import { CFG } from './cross' |
| 4 | import { parseFile, parseFileCache, createSafeWriteStream, exists } from './util-files' |
| 5 | import { loadFileAttr, singleWorkerFromBatchWorker, storeFileAttr } from './misc' |
| 6 | import _ from 'lodash' |
| 7 | import iconv from 'iconv-lite' |
| 8 | import { unlink } from 'node:fs/promises' |
| 9 | import { expiringCache } from './expiringCache' |
| 10 | |
| 11 | export const DESCRIPT_ION = 'descript.ion' |
| 12 | export const DESCRIPT_ION_ALT = 'DESCRIPT.ION' |
| 13 | const commentsStorage = defineConfig<'' | 'attr' | 'attr+ion'>(CFG.comments_storage, '', |
| 14 | v => ['', 'attr+ion'].includes(v)) // compiled tell us if we are using descript.ion |
| 15 | defineConfig('descript_ion', true, (v, more) => { // legacy: convert previous setting |
| 16 | if (!v && more.version?.olderThan('0.57.0-alpha1')) |
| 17 | commentsStorage.set('attr') |
| 18 | }) |
| 19 | const descriptIonEncoding = defineConfig('descript_ion_encoding', 'utf8') |
| 20 | |
| 21 | function readFromDescriptIon(path: string) { |
| 22 | return usingDescriptIon() && readDescriptIon(dirname(path)).then(x => x.get(basename(path)), () => undefined) |
| 23 | } |
| 24 | |
| 25 | export function usingDescriptIon() { |
| 26 | return commentsStorage.compiled() |
| 27 | } |
| 28 | |
| 29 | const COMMENT_ATTR = 'comment' |
| 30 | |
| 31 | export async function getCommentFor(path?: string) { |
| 32 | return !path ? undefined : Promise.all([ |
| 33 | commentsStorage.get() ? loadFileAttr(path, COMMENT_ATTR) : undefined, |
| 34 | readFromDescriptIon(path) |
| 35 | ]).then(([fromAttr, fromIon]) => fromAttr || fromIon || undefined) |
| 36 | } |
| 37 | |
| 38 | export async function setCommentFor(path: string, comment: string) { |
| 39 | if (commentsStorage.get()) { |
| 40 | await storeFileAttr(path, COMMENT_ATTR, comment || undefined) |
| 41 | return setCommentDescriptIon(path, '') |
| 42 | } |
| 43 | return setCommentDescriptIon(path, comment) // should we also remove from file-attr? not sure, but for the time we won't because #1 storeFileAttr is not really deleting, and we would store a lot of empty attributes, #2 more people will switch from descript.ion to attr (because introduced later) than the opposite |
| 44 | } |
| 45 | |
| 46 | const setCommentDescriptIon = singleWorkerFromBatchWorker(async (jobs: [path: string, comment: string][]) => { |
| 47 | const byFolder = _.groupBy(jobs, job => dirname(job[0])) |
| 48 | return Promise.allSettled(_.map(byFolder, async (jobs, folder) => { |
| 49 | const comments = await readDescriptIon(folder).catch(() => new Map()) |
| 50 | for (const [path, comment] of jobs) { |
| 51 | const file = path.slice(folder.length + 1) |
| 52 | if (!comment) |
| 53 | comments.delete(file) |
| 54 | else |
| 55 | comments.set(file, comment) |
| 56 | } |
| 57 | const path = await filePathHelper(folder) |
| 58 | if (!comments.size) |
| 59 | return unlink(path) |
| 60 | // encode comments in descript.ion format |
| 61 | const ws = await createSafeWriteStream(path) |
| 62 | comments.forEach((comment, filename) => { |
| 63 | const multiline = comment.includes('\n') |
| 64 | const line = (filename.includes(' ') ? `"${filename}"` : filename) |
| 65 | + ' ' + (multiline ? comment.replaceAll('\n', '\\n') : comment) |
| 66 | ws.write( iconv.encode(line, descriptIonEncoding.get()) ) |
| 67 | if (multiline) |
| 68 | ws.write(MULTILINE_SUFFIX, 'binary') |
| 69 | ws.write('\n') |
| 70 | }) |
| 71 | await new Promise(res => ws.end(res)) |
| 72 | })) |
| 73 | }) |
| 74 | |
| 75 | export function areCommentsEnabled() { |
| 76 | return true // true since we introduced comments in file-attr |
| 77 | } |
| 78 | |
| 79 | async function filePathHelper(folder: string) { |
| 80 | const main = join(folder, DESCRIPT_ION) |
| 81 | const alt = join(folder, DESCRIPT_ION_ALT) |
| 82 | return await exists(alt) && !await exists(main) ? alt : main |
| 83 | } |
| 84 | |
| 85 | const MULTILINE_SUFFIX = Buffer.from([4, 0xC2]) |
| 86 | const pathCache = expiringCache<Promise<string>>(2_000) |
| 87 | // this can be called many times when listing a folder, and we want to also not check too often as it can be expensive, especially on a networked drive |
| 88 | async function readDescriptIon(path: string) { |
| 89 | return parseFile(await pathCache.try(path, filePathHelper), raw => { |
| 90 | // for simplicity, we "remove" the sequence MULTILINE_SUFFIX before iconv.decode messes it up |
| 91 | for (let i=0; i<raw.length; i++) |
| 92 | if (raw[i] === MULTILINE_SUFFIX[0] && raw[i+1] === MULTILINE_SUFFIX[1] && [undefined,13,10].includes(raw[i+2])) |
| 93 | raw[i] = raw[i+1] = 10 |
| 94 | // decoding could also be done with native TextDecoder.decode, but we need iconv for the encoding anyway |
| 95 | const decoded = iconv.decode(raw, descriptIonEncoding.get()) |
| 96 | const ret = new Map(decoded.split('\n').map(line => { |
| 97 | const quoted = line[0] === '"' ? 1 : 0 |
| 98 | const i = quoted ? line.indexOf('"', 2) + 1 : line.indexOf(' ') |
| 99 | const fn = line.slice(quoted, i - quoted) |
| 100 | const comment = line.slice(i + 1).replaceAll('\\n', '\n') |
| 101 | return [fn, comment] |
| 102 | })) |
| 103 | ret.delete('') |
| 104 | return ret |
| 105 | }, 2000).then(x => x.content) |
| 106 | } |
| 107 | |
| 108 | descriptIonEncoding.sub(() => { // invalidate cache at encoding change |
| 109 | for (const k of parseFileCache.keys()) |
| 110 | if (k.endsWith(DESCRIPT_ION) || k.endsWith(DESCRIPT_ION_ALT)) |
| 111 | parseFileCache.delete(k) |
| 112 | }) |