rename/move/delete operations will update file-attr.kv
Massimo Melina committed
Apr 27, 2026 at 15:37 UTC
c519f0195c4e8a713ce9ae3e2d2da8d42f02d2ac
5 files changed
+85
-21
src/fileAttr.ts
+72
-15
@@ -1,25 +1,25 @@
1
import { KvStorage } from '@rejetto/kvstorage'
2
import { promisify } from 'util'
3
import { access } from 'fs/promises'
4
-import { try_, tryJson } from './cross'
4
+import { onlyTruthy, try_, tryJson } from './cross'
5
import { onProcessExit } from './first'
6
import { utimes } from 'node:fs/promises'
7
import { statWithTimeout } from './util-files'
8
import { IS_WINDOWS } from './const'
9
+import { isAbsolute, join, relative } from 'path'
10
11
const fsx = try_(() => {
12
const lib = require('fs-x-attributes')
13
return { set: promisify(lib.set), get: promisify(lib.get) }
14
}, () => console.warn('fs-x-attributes not available'))
15
15
-const fileAttrDb = new KvStorage({ defaultPutDelay: 1000, maxPutDelay: 5000 })
16
-onProcessExit(() => fileAttrDb.close())
16
const FN = 'file-attr.kv'
18
-access(FN).then(() =>
19
- fileAttrDb.open(FN).catch(e =>
20
- console.error(String(e))),
21
- () => {})
17
+export const fileAttrDb = new KvStorage({ defaultPutDelay: 1000, maxPutDelay: 5000 })
18
+onProcessExit(() => fileAttrDb.close())
19
+fileAttrDb.open(FN).catch(e =>
20
+ console.error(String(e)))
21
const FILE_ATTR_PREFIX = 'user.hfs.' // user. prefix to be linux compatible
22
+const FILE_ATTR_KEY_SEPARATOR = '|'
23
24
/* @param v must be JSON-able or undefined */
25
export async function storeFileAttr(path: string, k: string, v: any) {
@@ -30,11 +30,7 @@ export async function storeFileAttr(path: string, k: string, v: any) {
30
return true
31
}
32
// fallback to our kv-storage
33
- if (!fileAttrDb.isOpen())
34
- if (!s && !v) return // file was probably deleted, and we were asked to remove a possible attribute, but there's no fileAttrDb, so we are done, don't create the db file for nothing
35
- else await fileAttrDb.open(FN)
36
- // pipe should be a safe separator
37
- return await fileAttrDb.put(`${path}|${k}`, v)?.catch((e: any) => {
33
+ return await fileAttrDb.put(fileAttrKey(path, k), v)?.catch((e: any) => {
34
console.error("Couldn't store metadata on", path, String(e.message || e))
35
return false
36
}) ?? true // if put is undefined, the value was already there
@@ -43,18 +39,79 @@ export async function storeFileAttr(path: string, k: string, v: any) {
39
export async function loadFileAttr(path: string, k: string) {
40
return await fsx?.get(path, FILE_ATTR_PREFIX + k)
41
.then((x: any) => x === '' ? undefined : tryJson(String(x)),
46
- () => fileAttrDb.isOpen() ? fileAttrDb.get(`${path}|${k}`) : null)
42
+ () => fileAttrDb.isOpen() ? fileAttrDb.get(fileAttrKey(path, k)) : null)
43
?? undefined // normalize, as we get null instead of undefined on windows
44
}
45
46
export async function purgeFileAttr() {
47
let n = 0
48
await Promise.all(Array.from(fileAttrDb.keys()).map(k => {
53
- const [fn] = k.split('|')
49
+ const fn = splitFileAttrKey(k)?.filePath
50
return fn && access(fn).catch(() =>
51
n++ && void fileAttrDb.del(k))
52
}))
53
if (n)
54
await fileAttrDb.rewrite()
55
console.log(`Removed ${n} entrie(s)`)
60
-}
\ No newline at end of file
56
+}
57
+
58
+export async function moveStoredFileAttrs(fromPath: string, toPath: string) {
59
+ try {
60
+ if (fromPath === toPath || !fileAttrDb.isOpen())
61
+ return
62
+ const entries = storedFileAttrEntries()
63
+ const affectedEntries = entries.filter(x => isSameOrInside(fromPath, x.filePath))
64
+ if (!affectedEntries.length)
65
+ return
66
+ const affectedWithValues = await Promise.all(affectedEntries.map(async x => ({
67
+ ...x,
68
+ value: await fileAttrDb.get(x.key)
69
+ })))
70
+ const oldDestinationKeys = entries.filter(x => isSameOrInside(toPath, x.filePath)).map(x => x.key)
71
+ // destination attrs must be cleared first because a replaced file may not have all attrs owned by the source
72
+ await Promise.all(oldDestinationKeys.map(k => fileAttrDb.del(k)))
73
+ await Promise.all(affectedWithValues.map(async ({ key, filePath, attr, value }) => {
74
+ const rel = relative(fromPath, filePath)
75
+ // physical path keys the fallback DB, so filesystem moves must carry descendant entries explicitly
76
+ await fileAttrDb.put(fileAttrKey(join(toPath, rel), attr), value)
77
+ await fileAttrDb.del(key)
78
+ }))
79
+ }
80
+ // metadata sync runs after the filesystem mutation, so it must not report the completed file operation as failed
81
+ catch(e: any) { console.error("Couldn't move metadata in file-attr DB", fromPath, toPath, String(e.message || e)) }
82
+}
83
+
84
+export async function deleteStoredFileAttrs(path: string) {
85
+ try {
86
+ if (!fileAttrDb.isOpen())
87
+ return
88
+ const keys = storedFileAttrEntries().filter(x => isSameOrInside(path, x.filePath)).map(x => x.key)
89
+ await Promise.all(keys.map(k => fileAttrDb.del(k)))
90
+ }
91
+ // metadata cleanup runs after deletion, so surfacing this would leave clients seeing a false delete failure
92
+ catch(e: any) { console.error("Couldn't delete metadata from file-attr DB", path, String(e.message || e)) }
93
+}
94
+
95
+function storedFileAttrEntries() {
96
+ return onlyTruthy(Array.from(fileAttrDb.keys()).map(splitFileAttrKey))
97
+}
98
+
99
+function splitFileAttrKey(key: string) {
100
+ const i = key.lastIndexOf(FILE_ATTR_KEY_SEPARATOR)
101
+ if (i < 0)
102
+ return
103
+ return {
104
+ key,
105
+ filePath: key.slice(0, i),
106
+ attr: key.slice(i + 1)
107
+ }
108
+}
109
+
110
+function fileAttrKey(path: string, attr: string) {
111
+ return path + FILE_ATTR_KEY_SEPARATOR + attr
112
+}
113
+
114
+function isSameOrInside(parent: string, path: string) {
115
+ const rel = relative(parent, path)
116
+ return rel === '' || Boolean(rel) && !rel.startsWith('..') && !isAbsolute(rel)
117
+}
src/frontEndApis.ts
+4
-2
@@ -17,7 +17,7 @@ import fs from 'fs'
17
import { mkdir, rename, copyFile, unlink } from 'fs/promises'
18
import { basename, dirname, join } from 'path'
19
import { getUploadMeta } from './upload'
20
-import { apiAssertTypes, pathDecode, pathEncode, popKey } from './misc'
20
+import { apiAssertTypes, moveStoredFileAttrs, pathDecode, pathEncode, popKey } from './misc'
21
import { getCommentFor, setCommentFor } from './comments'
22
import { SendListReadable } from './SendList'
23
import { ctxAdminAccess } from './adminApis'
@@ -188,7 +188,8 @@ export async function moveFiles(uri_from: any, uri_to: any, ctx: Koa.Context, ov
188
if (e.code !== 'EXDEV') throw e // exdev = different drive
189
await copyFile(src, dest)
190
await unlink(src)
191
- }).catch(e => e.code || String(e))
191
+ }).then(() => moveStoredFileAttrs(src, dest))
192
+ .catch(e => e.code || String(e))
193
}))
194
}
195
}
@@ -210,6 +211,7 @@ export async function requestedRename(node: VfsNode | undefined, newName: string
211
try {
212
const destSource = join(dirname(node.source), newName)
213
await rename(node.source, destSource)
214
+ await moveStoredFileAttrs(node.source, destSource)
215
getCommentFor(node.source).then(c => {
216
if (!c) return
217
void setCommentFor(node.source!, '')
src/listen.ts
+2
-1
@@ -24,6 +24,7 @@ import { storedMap } from './persistence'
24
import { argv } from './argv'
25
import { consoleHint } from './consoleLog'
26
import { onProcessExit } from './first'
27
+import { fileAttrDb } from './fileAttr'
28
29
interface ServerExtra { name: string, error?: string, busy?: Promise<string> }
30
let httpSrv: undefined | http.Server & ServerExtra
@@ -52,7 +53,7 @@ const commonServerOptions: http.ServerOptions = {
53
// these are properties that can be assigned to the server object
54
const commonServerAssign = { headersTimeout: 30_000, timeout: MINUTE } // 'headersTimeout' is not recognized by type lib, and 'timeout' is not effective when passed in parameters
55
55
-const readyToListen = Promise.all([ storedMap.isOpening(), events.once('app') ])
56
+const readyToListen = Promise.all([ storedMap.isOpening(), fileAttrDb.isOpening(), events.once('app') ])
57
58
const considerHttp = debounceAsync(async () => {
59
await readyToListen
src/serveGuiAndSharedFiles.ts
+3
-2
@@ -20,8 +20,8 @@ import { serveGuiFiles } from './serveGuiFiles'
20
import mount from 'koa-mount'
21
import { baseUrl } from './listen'
22
import {
23
- asyncGeneratorToReadable, filterMapGenerator, isValidFileName, loadFileCached, pathEncode, safeDecodeURIComponent,
24
- try_,
23
+ asyncGeneratorToReadable, deleteStoredFileAttrs, filterMapGenerator, isValidFileName, loadFileCached, pathEncode,
24
+ safeDecodeURIComponent, try_,
25
} from './misc'
26
import XXH from 'xxhashjs'
27
import fs from 'fs'
@@ -112,6 +112,7 @@ export const serveSharedFiles: Koa.Middleware = async (ctx, next) => {
112
if ((await events.emitAsync('deleting', { node, ctx }))?.isDefaultPrevented())
113
return ctx.status = HTTP_FAILED_DEPENDENCY
114
await rm(source, { recursive: true })
115
+ await deleteStoredFileAttrs(source)
116
void setCommentFor(source, '') // necessary only to clean a possible descript.ion or kvstorage
117
return ctx.status = HTTP_OK
118
} catch (e: any) {
src/webdav.ts
+4
-1
@@ -24,6 +24,7 @@ import { defineConfig } from './config'
24
import { expiringCache } from './expiringCache'
25
import { XMLParser } from 'fast-xml-parser'
26
import _ from 'lodash'
27
+import { deleteStoredFileAttrs } from './fileAttr'
28
29
const forceWebdavLogin = defineConfig<boolean|string, null|RegExp>(CFG.force_webdav_login, true, compileWebdavAgentRegex)
30
const webdavInitialAuth = defineConfig<boolean|string, null|RegExp>(CFG.webdav_initial_auth, 'WebDAVFS', compileWebdavAgentRegex)
@@ -160,7 +161,9 @@ export const webdav: Koa.Middleware = async (ctx, next) => {
161
canOverwrite.delete(overwriteGraceKey)
162
const node = await urlToNode(path, ctx)
163
if (node?.source)
163
- await rm(node.source).catch(() => {})
164
+ await rm(node.source)
165
+ .then(() => deleteStoredFileAttrs(node.source!))
166
+ .catch(() => {})
167
}
168
if (x && ctx.length === undefined) // missing length can make PUT fail
169
ctx.req.headers['content-length'] = x