main
ts 541 lines 22.4 KB
Raw
1 // This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
3 import fs from 'fs/promises'
4 import { basename, dirname, join, resolve } from 'path'
5 import {
6 makeMatcher, setHidden, onlyTruthy, isValidFileName, throw_, VfsPerms, WhoVfs, debounceAsync,
7 isWhoObject, WHO_ANY_ACCOUNT, WHO_ADMIN, defaultPerms, PERM_KEYS, HTTP_SERVER_ERROR, try_, matches, Promisable,
8 statWithTimeout, safeDecodeURIComponent, getUncHost, Who,
9 } from './misc'
10 import Koa from 'koa'
11 import _ from 'lodash'
12 import { defineConfig, setConfig } from './config'
13 import { HTTP_FORBIDDEN, HTTP_UNAUTHORIZED, IS_MAC, IS_WINDOWS } from './const'
14 import events from './events'
15 import { ctxBelongsTo } from './perm'
16 import { getCurrentUsername } from './auth'
17 import { Stats } from 'node:fs'
18 import fswin from 'fswin'
19 import { DESCRIPT_ION, DESCRIPT_ION_ALT, usingDescriptIon } from './comments'
20 import { walkDir } from './walkDir'
21 import { Readable } from 'node:stream'
22 import { ctxAdminAccess } from './adminApis'
23
24 const showHiddenFiles = defineConfig('show_hidden_files', false)
25
26 type Masks = Record<string, VfsNode>
27
28 export interface VfsNodeStored extends VfsPerms {
29 name?: string
30 source?: string
31 url?: string
32 target?: string
33 children?: VfsNode[]
34 default?: string | false // we could have used empty string to override inherited default, but false is clearer, even reading the yaml, and works well with pickProps(), where empty strings are removed
35 mime?: string | Record<string, string>
36 rename?: Record<string, string>
37 masks?: Masks // express fields for descendants that are not in the tree
38 accept?: string
39 comment?: string
40 icon?: string
41 order?: number
42 }
43 export interface VfsNode extends VfsNodeStored { // include fields that are only filled at run-time
44 isTemp?: true // this node doesn't belong to the tree and was created by necessity
45 original?: VfsNode // if this is a temp node but reflecting an existing node
46 parent?: VfsNode // available when original is available (therefore, only for isTemp)
47 isFolder?: boolean // use nodeIsFolder() instead of relying on this field
48 stats?: Promisable<Stats>
49 }
50
51 export function permsFromParent(parent: VfsNode, child: VfsNode) {
52 const ret: VfsPerms = {}
53 for (const k of PERM_KEYS) {
54 let p: VfsNode | undefined = parent
55 let inheritedPerm: WhoVfs | undefined
56 while (p) {
57 inheritedPerm = p[k]
58 // in case of object without children, parent is skipped in favor of the parent's parent
59 if (!isWhoObject(inheritedPerm)) break
60 inheritedPerm = inheritedPerm.children
61 if (inheritedPerm !== undefined) break
62 p = p.parent
63 }
64 if (inheritedPerm !== undefined && child[k] === undefined) // small optimization: don't expand the object
65 ret[k] = inheritedPerm
66 }
67 return _.isEmpty(ret) ? undefined : ret
68 }
69
70 function inheritFromParent(child: VfsNode) {
71 const { parent } = child
72 if (!parent) return
73 Object.assign(child, permsFromParent(parent, child))
74 if (typeof parent.mime === 'object' && typeof child.mime === 'object')
75 _.defaults(child.mime, parent.mime)
76 else
77 if (parent.mime) child.mime ??= parent.mime
78 if (parent.accept) child.accept ??= parent.accept
79 if (parent.default) child.default ??= parent.default
80 return child
81 }
82
83 export function isSameFilenameAs(name: string) {
84 const normalized = normalizeFilename(name)
85 return (other: string | VfsNode) =>
86 normalized === normalizeFilename(typeof other === 'string' ? other : getNodeName(other))
87 }
88
89 export function normalizeFilename(x: string) {
90 return (IS_WINDOWS || IS_MAC ? x.toLocaleLowerCase() : x).normalize()
91 }
92
93 export async function applyParentToChild(child: VfsNode | undefined, parent: VfsNode, name?: string) {
94 const ret: VfsNode = {
95 original: child, // this can be overridden by passing an 'original' in `child`
96 ...child,
97 isFolder: child?.isFolder ?? (child?.children?.length! > 0 || undefined), // isFolder is hidden in original node, so we must copy it explicitly
98 isTemp: true,
99 parent,
100 }
101 name ||= child ? getNodeName(child) : ''
102 inheritMasks(ret, parent, name)
103 await parentMaskApplier(parent)(ret, name)
104 inheritFromParent(ret)
105 return ret
106 }
107
108 export async function urlToNode(
109 url: string,
110 ctx?: Koa.Context,
111 parent: VfsNode=vfs,
112 allowMissing?: boolean // true means missing path segments still resolve to temporary nodes with a computed source path
113 ) : Promise<VfsNode | undefined> {
114 let initialSlashes = 0
115 while (url[initialSlashes] === '/')
116 initialSlashes++
117 let nextSlash = url.indexOf('/', initialSlashes)
118 const slice = url.slice(initialSlashes, nextSlash < 0 ? undefined : nextSlash)
119 if (!slice)
120 return parent
121 const name = safeDecodeURIComponent(slice, '')
122 if (!name) // failed decoding
123 return
124 const hasTrailingSlash = url.endsWith('/')
125 const rest = nextSlash < 0 ? '' : url.slice(nextSlash+1, hasTrailingSlash ? -1 : undefined)
126 const assumeFolder = allowMissing && (rest > '' || hasTrailingSlash)
127 const ret = await getNodeByName(name, parent, assumeFolder)
128 if (!ret)
129 return
130 if (rest || ret?.original)
131 return urlToNode(rest, ctx, ret, allowMissing)
132 if (ret.source)
133 if (!showHiddenFiles.get() && await isHiddenFile(ret.source)
134 || !allowMissing && await setIsFolder(ret) === undefined) // undefined = not found on disk
135 return
136 return ret
137 }
138
139 export async function nodeStats(node: VfsNode) {
140 if (node.stats || !node.source)
141 return node.stats
142 const stats = statWithTimeout(node.source).catch(() => {
143 setHidden(node, { stats: null }) // don't cache rejected promises
144 })
145 setHidden(node, { stats })
146 return stats
147 }
148
149 async function isHiddenFile(path: string) {
150 return IS_WINDOWS ? new Promise(res => fswin.getAttributes(path, x => res(x?.IS_HIDDEN)))
151 : path[path.lastIndexOf('/') + 1] === '.'
152 }
153
154 export async function getNodeByName(name: string, parent: VfsNode, assumeMissingToBeFolder=false) {
155 // does the tree node have a child that goes by this name, otherwise attempt disk
156 let child = parent.children?.find(isSameFilenameAs(name))
157 if (child) // found as vfs node
158 await setIsFolder(child) // in case it's pointing to a folder that didn't exist at loading time
159 else
160 child = await childFromDisk()
161 return child && applyParentToChild(child, parent, name)
162
163 async function childFromDisk() {
164 if (!parent.source) return
165 const ret: VfsNode = {}
166 let onDisk = name
167 if (parent.rename) { // reverse the mapping
168 for (const [from, to] of Object.entries(parent.rename))
169 if (name === to) {
170 onDisk = from
171 break // found, search no more
172 }
173 ret.rename = renameUnderPath(parent.rename, name)
174 }
175 if (!isValidFileName(onDisk)) return
176 ret.source = join(parent.source, onDisk)
177 ret.original = undefined // this will overwrite the 'original' set in applyParentToChild, so we know this is not part of the vfs
178 await setIsFolder(ret)
179 if (assumeMissingToBeFolder)
180 ret.isFolder ??= true
181 return ret
182 }
183 }
184
185 const smartUncFolderDetection = defineConfig('smart_unc_folder_detection', false)
186
187 async function setIsFolder(node: VfsNode) {
188 if (!node.source) return
189 const isFolder = /[\\/]$/.test(node.source)
190 || smartUncFolderDetection.get() && getUncHost(node.source) && !basename(node.source).includes('.') // no dot = folder; not very reliable but fast for unreachable unc hosts, and it's an opt-in
191 || await nodeStats(node).then(x => x?.isDirectory(), () => undefined)
192 setHidden(node, { isFolder })
193 return isFolder
194 }
195
196 export let vfs: VfsNode = {}
197 defineConfig('vfs', vfs).sub(async x => {
198 await reviewVfs(x)
199 console.log('VFS ready')
200 })
201
202 async function reviewVfs(data=vfs) {
203 await (async function recur(node) {
204 if (node.source && !node.children?.length && node.isFolder === undefined)
205 await setIsFolder(node)
206 if (node.children)
207 await Promise.allSettled(node.children.map(recur))
208 })(data)
209 vfs = data
210 }
211
212 export const saveVfs = debounceAsync(async () => {
213 await reviewVfs()
214 await setConfig({ vfs }, true)
215 })
216
217 export function isRoot(node: VfsNode) {
218 return node === vfs
219 }
220
221 export function getNodeName(node: VfsNode) {
222 if (isRoot(node))
223 return ''
224 if (node.name)
225 return node.name
226 const { source } = node
227 if (!source)
228 return '' // shoulnd't happen
229 if (source === '/')
230 return 'root' // better name than
231 if (/^[a-zA-Z]:\\?$/.test(source))
232 return source.slice(0, 2) // exclude trailing slash
233 const base = basename(source)
234 if (/^[./\\]*$/.test(base)) // if empty or special-chars-only
235 return basename(resolve(source)) // resolve to try to get more
236 if (base.includes('\\') && !source.includes('/')) // source was Windows but now we are running posix. This probably happens only debugging, so it's DX
237 return source.slice(source.lastIndexOf('\\') + 1)
238 return base
239 }
240
241 // this is sync
242 export function nodeIsFolder(node: VfsNode) {
243 return node.isFolder ?? node.original?.isFolder
244 ?? (nodeIsLink(node) ? false : (node.children?.length! > 0 || !node.source || reconsider()))
245
246 function reconsider() {
247 // a networked source may be offline at startup, and become online later: recalculate in the background
248 nodeStats(node).then(s => {
249 if (s)
250 setHidden(node.original || node, { isFolder: s.isDirectory() })
251 }, () => {})
252 return undefined
253 }
254 }
255
256 export async function getDefaultFile(node: VfsNode, ctx: Koa.Context) {
257 return node.default && nodeIsFolder(node) && await urlToNode(node.default, ctx, node) || undefined
258 }
259
260 export function nodeIsLink(node: VfsNode) {
261 return node.url
262 }
263
264 export function hasPermission(node: VfsNode, perm: keyof VfsPerms, ctx: Koa.Context): boolean {
265 return !statusCodeForMissingPerm(node, perm, ctx, false)
266 }
267
268 export function statusCodeForMissingPerm(node: VfsNode, perm: keyof VfsPerms, ctx: Koa.Context, assign=true) {
269 const ret = getCode()
270 if (ret && assign) {
271 ctx.status = ret
272 ctx.body = ret === HTTP_UNAUTHORIZED ? "Unauthorized" : "Forbidden"
273 }
274 return ret
275
276 function getCode() {
277 if ((isRoot(node) || node.original) && perm === 'can_delete' // we currently don't allow deleting of vfs nodes from frontend
278 || !node.source && perm === 'can_upload') // Upload possible only if we know where to store. First check node.source because is supposedly faster.
279 return HTTP_FORBIDDEN
280 // calculate value of permission resolving references to other permissions, avoiding infinite loop
281 let who: WhoVfs | undefined
282 let max = PERM_KEYS.length
283 let cur = perm
284 do {
285 who = node[cur]
286 if (isWhoObject(who))
287 who = who.this
288 who ??= defaultPerms[cur]
289 if (typeof who !== 'string' || who === WHO_ANY_ACCOUNT || who === WHO_ADMIN)
290 break
291 if (!max--) {
292 console.error(`Endless loop in permission ${perm}=${node[perm] ?? defaultPerms[perm]} for ${node.url || getNodeName(node)}`)
293 return HTTP_SERVER_ERROR
294 }
295 cur = who
296 } while (1)
297 if (isWhoObject(who) || isWhoVfsPerms(who))
298 throw Error(`permission type-guard: ${JSON.stringify(who)}`)
299 const eventName = 'checkVfsPermission'
300 if (events.anyListener(eventName)) {
301 const first = _.max(events.emit(eventName, { who, node, perm, ctx }))
302 if (first !== undefined)
303 return first
304 }
305
306 return simpleWhoToError(who, ctx)
307 ?? throw_(Error(`invalid permission: ${perm}=${try_(() => JSON.stringify(who))}`))
308 }
309 }
310
311 export function simpleWhoToError(who: Who, ctx: Koa.Context) {
312 if (Array.isArray(who))
313 return ctxBelongsTo(ctx, who) ? 0 : HTTP_UNAUTHORIZED
314 return typeof who === 'boolean' ? (who ? 0 : HTTP_FORBIDDEN)
315 : who === WHO_ANY_ACCOUNT ? (getCurrentUsername(ctx) ? 0 : HTTP_UNAUTHORIZED)
316 : who === WHO_ADMIN ? (ctxAdminAccess(ctx) ? 0 : HTTP_UNAUTHORIZED)
317 : undefined
318 }
319
320 function isWhoVfsPerms(who: WhoVfs | undefined): who is keyof VfsPerms {
321 return typeof who === 'string' && (PERM_KEYS as readonly string[]).includes(who)
322 }
323
324 interface WalkNodeOptions {
325 ctx?: Koa.Context,
326 depth?: number,
327 prefixPath?: string,
328 requiredPerm?: undefined | keyof VfsPerms,
329 onlyFolders?: boolean,
330 onlyFiles?: boolean,
331 parallelizeRecursion?: boolean,
332 }
333 // it's the responsibility of the caller to verify you have list permission on parent, as callers have different needs.
334 export async function* walkNode(parent: VfsNode, {
335 ctx,
336 depth = Infinity,
337 prefixPath = '',
338 requiredPerm,
339 onlyFolders = false,
340 onlyFiles = false,
341 parallelizeRecursion = true,
342 }: WalkNodeOptions = {}) {
343 let started = false
344 const stream = new Readable({
345 objectMode: true,
346 async read() {
347 if (started) return // for simplicity, we care about starting, and never suspend
348 started = true
349 const { source } = parent
350 const taken = new Set()
351 const maskApplier = parentMaskApplier(parent)
352 const visitLater: [VfsNode, string][] = []
353 const childrenWorking = parent.children?.length && Promise.all(parent.children.map(async child => {
354 if (ctx?.isAborted()) return
355 const nodeName = getNodeName(child)
356 const name = prefixPath + nodeName
357 taken?.add(normalizeFilename(name))
358 const item = { ...child, original: child, name, parent }
359 if (await cantSee(item)) return
360 if (item.source && !item.children?.length) // real items must be accessible, unless there's more to it
361 try { await fs.access(item.source) }
362 catch { return }
363 const isFolder = nodeIsFolder(child)
364 if (onlyFiles ? !isFolder : (!onlyFolders || isFolder))
365 stream.push(item)
366 if (!depth || !isFolder || cantRecur(item)) return
367 inheritMasks(item, parent)
368 visitLater.push([item, name]) // prioritize siblings
369 }))
370
371 try {
372 if (!source)
373 return
374 if (requiredPerm && ctx // no permission, no reason to continue (at least for dynamic elements)
375 && !hasPermission(parent, requiredPerm, ctx)
376 && !masksCouldGivePermission(parent.masks, requiredPerm))
377 return
378
379 const pathMaskApplier = parentMaskApplier(parent, true)
380 try {
381 await walkDir(source, { depth, ctx, hidden: showHiddenFiles.get(), parallelizeRecursion }, async entry => {
382 if (ctx?.isAborted())
383 return null
384 if (usingDescriptIon() && (entry.name === DESCRIPT_ION || entry.name === DESCRIPT_ION_ALT))
385 return
386 const {path} = entry // this path is not the original deprecated property: we are overwriting/reusing it
387 const isFolder = entry.isDirectory()
388 let renamed = parent.rename?.[path]
389 if (renamed) {
390 const dir = dirname(path) // if `path` isn't just the name, copy its dir in renamed
391 if (dir !== '.')
392 renamed = dir + '/' + renamed
393 }
394 const name = prefixPath + (renamed || path)
395 if (taken?.has(normalizeFilename(name))) // taken by vfs node above
396 return false // false just in case it's a folder
397 const item: VfsNode = { name, isFolder, source: join(source, path), parent, stats: entry.stats }
398 // masks containing '/' must be matched against the relative path while keeping walkDir recursion enabled
399 await pathMaskApplier(item, renamed || path)
400 if (await cantSee(item)) // can't see: don't produce and don't recur
401 return false
402 if (onlyFiles ? !isFolder : (!onlyFolders || isFolder))
403 stream.push(item)
404 if (cantRecur(item))
405 return false
406 })
407 }
408 catch(e) {
409 console.debug('walkNode', source, String(e)) // ENOTDIR, or lacking permissions
410 }
411 }
412 finally {
413 await childrenWorking
414 for (const [item, name] of visitLater)
415 for await (const x of walkNode(item, { depth: depth - 1, prefixPath: name + '/', ctx, requiredPerm, onlyFolders, onlyFiles, parallelizeRecursion })) {
416 if (ctx?.isAborted())
417 return stream.push(null)
418 stream.push(x)
419 }
420 stream.push(null)
421 }
422
423 function cantRecur(item: VfsNode) {
424 return ctx && !hasPermission(item, 'can_list', ctx)
425 }
426
427 // item will be changed, so be sure to pass a temp node
428 async function cantSee(item: VfsNode) {
429 await maskApplier(item)
430 inheritFromParent(item)
431 if (ctx && !hasPermission(item, 'can_see', ctx)) return true
432 item.isTemp = true
433 }
434 }
435 })
436
437 // must use a stream to be able to work with the callback-based mechanism of walkDir, but Readable is not typed so we wrap it with a generator
438 for await (const item of stream) {
439 if (ctx?.isAborted()) return
440 yield item as VfsNode
441 }
442 }
443
444 export function masksCouldGivePermission(masks: Masks | undefined, perm: keyof VfsPerms): boolean {
445 return masks !== undefined && Object.values(masks).some(props =>
446 props[perm] || masksCouldGivePermission(props.masks, perm))
447 }
448
449 export function parentMaskApplier(parent: VfsNode, pathBased=false) {
450 // rules are met in the parent.masks object from nearest to farthest, but since we finally apply with _.defaults, the nearest has precedence in the final result
451 const matchers = onlyTruthy(_.map(parent.masks, (mods, mask) => {
452 if (!mods) return
453 const mustBeFolder = (() => { // undefined if no restriction is requested
454 if (mask.at(-1) !== '|') return // parse special flag syntax as suffix |FLAG| inside the key. This allows specifying different flags with the same mask using separate keys. To avoid syntax conflicts with the rest of the file-mask, we look for an ending pipe, as it has no practical use. Ending-pipe was preferred over starting-pipe to leave the rest of the logic (inheritMasks) untouched.
455 const i = mask.lastIndexOf('|', mask.length - 2)
456 if (i < 0) return
457 const type = mask.slice(i + 1, -1)
458 mask = mask.slice(0, i) // remove
459 return type === 'folders'
460 })()
461 if (pathBased) {
462 if (!mask.includes('/')) return
463 // avoid evaluating twice masks like **/*.png because parentMaskApplier already handles them by basename
464 const m = /^(!?)\*\*\//.exec(mask)
465 // this keeps the fast basename path as source-of-truth for patterns that collapse to a filename after **/
466 if (m && !mask.slice(m[0].length).includes('/')) return
467 }
468 else {
469 const m = /^(!?)\*\*\//.exec(mask) // ** globstar matches also zero subfolders, so this mask must be applied here too
470 mask = m ? m[1] + mask.slice(m[0].length) : !mask.includes('/') ? mask : ''
471 if (!mask) return
472 }
473 return mask && { matcher: makeMatcher(mask), mods, mustBeFolder }
474 }))
475 return async (item: VfsNode, virtualName=(pathBased ? _.identity : basename)(getNodeName(item))!) => {
476 // depth traversal passes full relative paths, while node traversal still matches only basenames
477 let isFolder: boolean | undefined = undefined
478 for (const { matcher, mods, mustBeFolder } of matchers) {
479 if (mustBeFolder !== undefined) {
480 isFolder ??= nodeIsFolder(item)
481 if (mustBeFolder !== isFolder) continue
482 }
483 if (!matcher(virtualName)) continue
484 item.masks &&= _.merge(_.cloneDeep(mods.masks), item.masks) // item.masks must take precedence
485 _.defaults(item, mods)
486 }
487 }
488 }
489
490 // propagates masks, don't apply
491 function inheritMasks(item: VfsNode, parent: VfsNode, virtualBasename=getNodeName(item)) {
492 const { masks } = parent
493 if (!masks) return
494 const o: Masks = {}
495 for (const [k,v] of Object.entries(masks)) {
496 if (k.startsWith('**')) {
497 o[k] = v
498 continue
499 }
500 const i = k.indexOf('/')
501 if (i < 0) continue
502 if (!matches(virtualBasename, k.slice(0, i))) continue
503 o[k.slice(i + 1)] = v
504 }
505 if (Object.keys(o).length)
506 item.masks = Object.assign(o, item.masks) // don't change item.masks object as it is the same object of item.original
507 }
508
509 function renameUnderPath(rename:undefined | Record<string,string>, path: string) {
510 if (!rename) return rename
511 const match = path+'/'
512 rename = Object.fromEntries(Object.entries(rename).map(([k, v]) =>
513 [k.startsWith(match) ? k.slice(match.length) : '', v]))
514 delete rename['']
515 return _.isEmpty(rename) ? undefined : rename
516 }
517
518 events.on('accountRenamed', ({ from, to }) => {
519 ;(function renameInNode(n: VfsNode) {
520 for (const k of PERM_KEYS)
521 renameInPerm(n[k])
522
523 if (n.masks)
524 Object.values(n.masks).forEach(renameInNode)
525 n.children?.forEach(renameInNode)
526 })(vfs)
527 saveVfs()
528
529 function renameInPerm(a?: WhoVfs) {
530 if (isWhoObject(a)) {
531 renameInPerm(a.this)
532 renameInPerm(a.children)
533 return
534 }
535 if (Array.isArray(a))
536 for (let i=0; i < a.length; i++)
537 if (a[i] === from)
538 a[i] = to
539 }
540
541 })