b603e024 · samitouri/QOSami-HFS-1 · siGit Code & Deploy

config.md

+2 -1

@@ -126,7 +126,8 @@ Configuration can be done in several ways

126 Setting one will trigger a test request to google.com. You can skip this with env HFS_SKIP_PROXY_TEST=1 .

127 - `auto_basic` automatically detect (based on user-agent) when the basic web interface should be served, to support legacy browsers. Default is true. No UI.

128 You can disable it setting it to `false`, or recognize additional user-agents by setting a regular expression.

129 -- `authorization_header` support Authentication HTTP header. Default is true. No UI.

129 +- `file_timeout` number of seconds to wait before giving up when accessing a file. Default is 3. No UI.

130 +- `authorization_header` enable support for the HTTP `Authorization` header. Default is true. No UI.

131 - `cache_control_disk_files` number of seconds after which the browser should bypass the cache and check the server for an updated version of the file. Default is 5. No UI.

132 - `disable_custom_html` disable the content of `custom_html`. Default is false.

133 - `split_uploads` The size in megabytes of the chunks the upload will be split into. Default is none.

shared/api.ts

+1

@@ -14,6 +14,7 @@ const timeoutByApi: Dict = {

14 login: 90,

15 get_status: 20, // can be lengthy on slow machines because of the find-process-on-busy-port feature

16 check_update: 20,

17 + get_vfs: 20, // multiple sources may be slow

18 }

19

20 interface ApiCallOptions {

src/api.get_file_list.ts

+4 -5

@@ -5,9 +5,8 @@ import {

5 statusCodeForMissingPerm, urlToNode, VfsNode, walkNode

6 } from './vfs'

7 import { ApiError, ApiHandler } from './apiMiddleware'

8 -import { stat } from 'fs/promises'

8 import { mapPlugins } from './plugins'

10 -import { asyncGeneratorToArray, pattern2filter, WHO_NO_ONE } from './misc'

9 +import { asyncGeneratorToArray, pattern2filter, statWithTimeout, WHO_NO_ONE } from './misc'

10 import { HTTP_METHOD_NOT_ALLOWED, HTTP_NOT_FOUND } from './const'

11 import Koa from 'koa'

12 import { getCommentFor, areCommentsEnabled } from './comments'

@@ -36,7 +35,7 @@ export const get_file_list: ApiHandler = async ({ uri='/', offset, limit, c, onl

35 if (!node)

36 return fail(HTTP_NOT_FOUND)

37 admin &&= ctxAdminAccess(ctx) // validate 'admin' flag

39 - if (await hasDefaultFile(node, ctx) || !nodeIsFolder(node)) // in case of files without permission, we are provided with the frontend, and the location is the file itself

38 + if (await hasDefaultFile(node, ctx) || !nodeIsFolder(node)) // for files without permission, the frontend is sent, and the location is the file itself

39 // so, we first check if you have a permission problem, to tell frontend to show login, otherwise we fall back to method_not_allowed, as it's proper for files.

40 return fail(!admin && statusCodeForMissingPerm(node, 'can_read', ctx) ? undefined : HTTP_METHOD_NOT_ALLOWED)

41 if (!admin && statusCodeForMissingPerm(node, 'can_list', ctx))

@@ -116,7 +115,7 @@ export const get_file_list: ApiHandler = async ({ uri='/', offset, limit, c, onl

115 return name ? { n: name, url, target: node.target } : null

116 const isFolder = nodeIsFolder(node)

117 try {

119 - const st = source ? node.stats || await stat(source).catch(e => {

118 + const st = source ? node.stats || await statWithTimeout(source).catch(e => {

119 if (!isFolder || !node.children?.length) // folders with virtual children, keep them

120 throw e

121 }) : undefined

@@ -128,7 +127,7 @@ export const get_file_list: ApiHandler = async ({ uri='/', offset, limit, c, onl

127 const pr = node.can_read === WHO_NO_ONE && !(isFolder && filesInsideCould()) ? 'r'

128 : !hasPermission(node, 'can_read', ctx) ? 'R'

129 : ''

131 - // for delete, the diff is based on can_delete_children instead of can_delete, because it will produce fewer data

130 + // for delete, the diff is based on can_delete_children instead of can_delete, because it will produce less data

131 const pd = Boolean(can_delete_children) === hasPermission(node, 'can_delete', ctx) ? '' : can_delete_children ? 'd' : 'D'

132 const pa = Boolean(can_archive) === hasPermission(node, 'can_archive', ctx) ? '' : can_archive ? 'a' : 'A'

133 const pu = !isFolder || Boolean(can_upload) === hasPermission(node, 'can_upload', ctx) ? '' : can_upload ? 'u' : 'U'

src/api.log.ts

+2 -3

@@ -2,10 +2,9 @@ import { ApiHandlers } from './apiMiddleware'

2 import _ from 'lodash'

3 import { consoleLog } from './consoleLog'

4 import { HTTP_BAD_REQUEST, HTTP_NOT_ACCEPTABLE, HTTP_NOT_FOUND, wait } from './cross'

5 -import { apiAssertTypes } from './misc'

5 +import { apiAssertTypes, statWithTimeout } from './misc'

6 import events from './events'

7 import { getRotatedFiles, loggers } from './log'

8 -import { stat } from 'fs/promises'

8 import { SendListReadable } from './SendList'

9 import { forceDownload, serveFile } from './serveFile'

10 import { ips } from './ips'

@@ -13,7 +12,7 @@ import { disconnectionsLog } from './connections'

12

13 export default {

14 async get_log_info() {

16 - const current = Object.fromEntries(await Promise.all(loggers.map(async x => [x.name, await stat(x.path).then(s => s.size, () => 0)])))

15 + const current = Object.fromEntries(await Promise.all(loggers.map(async x => [x.name, await statWithTimeout(x.path).then(s => s.size, () => 0)])))

16 return { current, rotated: await getRotatedFiles() }

17 },

18

src/api.vfs.ts

+5 -5

@@ -5,12 +5,12 @@ import {

5 permsFromParent, nodeIsLink, VfsNodeStored, isRoot

6 } from './vfs'

7 import _ from 'lodash'

8 -import { mkdir, stat } from 'fs/promises'

8 +import { mkdir } from 'fs/promises'

9 import { ApiError, ApiHandlers } from './apiMiddleware'

10 import { dirname, extname, join, resolve } from 'path'

11 import {

12 enforceFinal, enforceStarting, isDirectory, isValidFileName, isWindowsDrive, makeMatcher, PERM_KEYS,

13 - VfsNodeAdminSend, wait

13 + statWithTimeout, VfsNodeAdminSend

14 } from './misc'

15 import {

16 IS_WINDOWS, HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVER_ERROR, HTTP_CONFLICT, HTTP_NOT_ACCEPTABLE,

@@ -39,7 +39,7 @@ const apis: ApiHandlers = {

39

40 async function recur(node=vfs): Promise<VfsNodeAdminSend> {

41 const { source } = node

42 - const stats = !source ? undefined : (node.stats || await stat(source!).catch(() => undefined))

42 + const stats = !source ? undefined : (node.stats || await statWithTimeout(source!).catch(() => undefined))

43 const isDir = !nodeIsLink(node) && (!source || (stats?.isDirectory() ?? (source.endsWith('/') || node.children?.length! > 0)))

44 const copyStats: Pick<VfsNodeAdminSend, 'size' | 'birthtime' | 'mtime'> = stats ? _.pick(stats, ['size', 'birthtime', 'mtime'])

45 : { size: source ? -1 : undefined }

@@ -56,7 +56,7 @@ const apis: ApiHandlers = {

56 inherited,

57 byMasks: _.isEmpty(byMasks) ? undefined : byMasks,

58 website: Boolean(node.children?.find(isSameFilenameAs('index.html')))

59 - || isDir && source && await stat(join(source, 'index.html')).then(() => true, () => undefined)

59 + || isDir && source && await statWithTimeout(join(source, 'index.html')).then(() => true, () => undefined)

60 || undefined,

61 name: getNodeName(node),

62 type: isDir ? 'folder' : undefined,

@@ -214,7 +214,7 @@ const apis: ApiHandlers = {

214 if (!files || fileMask && !matching(name))

215 return

216 try {

217 - const stats = entry.stats || await stat(join(path, name))

217 + const stats = entry.stats || await statWithTimeout(join(path, name))

218 list.add({

219 n: name,

220 s: stats.size,

src/config.ts

+3 -2

@@ -9,9 +9,10 @@ import { debounceAsync } from './debounceAsync'

9 import { statSync } from 'fs'

10 import { join, resolve } from 'path'

11 import events from './events'

12 -import { copyFile, stat } from 'fs/promises'

12 +import { copyFile } from 'fs/promises'

13 import { produce, setAutoFreeze } from 'immer'

14 import { argv } from './argv'

15 +import { statWithTimeout } from './util-files'

16

17 setAutoFreeze(false) // we still want to mess with objects later (eg: account.belongs)

18

@@ -182,7 +183,7 @@ const saveDebounced = debounceAsync(async () => {

183 // keep backup

184 const bak = filePath + '.bak'

185 const aWeekAgo = Date.now() - DAY * 7

185 - if (await stat(bak).then(x => aWeekAgo > x.mtimeMs, () => true))

186 + if (await statWithTimeout(bak).then(x => aWeekAgo > x.mtimeMs, () => true))

187 await copyFile(filePath, bak).catch(() => {}) // ignore errors

188

189 await configFile.save(stringify({ ...state, version: VERSION }))

src/fileAttr.ts

+3 -2

@@ -4,7 +4,8 @@ import { promisify } from 'util'

4 import { access } from 'fs/promises'

5 import { try_, tryJson } from './cross'

6 import { onProcessExit } from './first'

7 -import { utimes, stat } from 'node:fs/promises'

7 +import { utimes } from 'node:fs/promises'

8 +import { statWithTimeout } from './util-files'

9 import { IS_WINDOWS } from './const'

10

11 const fsx = try_(() => {

@@ -21,7 +22,7 @@ const FILE_ATTR_PREFIX = 'user.hfs.' // user. prefix to be linux compatible

22

23 /* @param v must be JSON-able or undefined */

24 export async function storeFileAttr(path: string, k: string, v: any) {

24 - const s = await stat(path).catch(() => null)

25 + const s = await statWithTimeout(path).catch(() => null)

26 // since we don't have fsx.remove, we simulate it with an empty string

27 if (s && await fsx?.set(path, FILE_ATTR_PREFIX + k, v === undefined ? '' : JSON.stringify(v)).then(() => 1, () => 0)) {

28 if (IS_WINDOWS) utimes(path, s.atime, s.mtime) // restore timestamps, necessary only on Windows

src/geo.ts

+4 -4

@@ -1,6 +1,6 @@

1 import { defineConfig } from './config'

2 -import { CFG, DAY, httpStream, isIpLan, isLocalHost, unzip } from './misc'

3 -import { stat, rename, unlink } from 'node:fs/promises'

2 +import { CFG, DAY, httpStream, isIpLan, isLocalHost, statWithTimeout, unzip } from './misc'

3 +import { rename, unlink } from 'node:fs/promises'

4 import { IP2Location } from 'ip2location-nodejs'

5 import _ from 'lodash'

6 import { Middleware } from 'koa'

@@ -40,7 +40,7 @@ async function checkFiles() {

40 const URL = `https://download.ip2location.com/lite/${ZIP_FILE}.ZIP`

41 const LOCAL_FILE = 'geo_ip.bin'

42 const TEMP = LOCAL_FILE + '.downloading'

43 - const { mtime=0 } = await stat(LOCAL_FILE).catch(() => ({ mtime: 0 }))

43 + const { mtime=0 } = await statWithTimeout(LOCAL_FILE).catch(() => ({ mtime: 0 }))

44 const name = 'geo-ip db'

45 const now = Date.now()

46 if (+mtime < now - 31 * DAY) // month-old or non-existing

@@ -48,7 +48,7 @@ async function checkFiles() {

48 const req = await httpStream(URL)

49 console.log(`downloading ${name}`)

50 await unzip(req, path => path.toUpperCase().endsWith(ZIP_FILE) && TEMP)

51 - await stat(TEMP) // check existence

51 + await statWithTimeout(TEMP) // check existence

52 if (isOpen())

53 ip2location.close()

54 await unlink(LOCAL_FILE).catch(() => {})

src/log.ts

+3 -4

@@ -6,9 +6,8 @@ import { Writable } from 'stream'

6 import { defineConfig } from './config'

7 import { createWriteStream, renameSync, statSync } from 'fs'

8 import * as util from 'util'

9 -import { stat } from 'fs/promises'

9 import _ from 'lodash'

11 -import { createFileWithPath, prepareFolder } from './util-files'

10 +import { createFileWithPath, prepareFolder, statWithTimeout } from './util-files'

11 import { getCurrentUsername } from './auth'

12 import { DAY, makeNetMatcher, tryJson, Dict, Falsy, CFG, strinsert, repeat, formatTimestamp, HTTP_NOT_FOUND } from './misc'

13 import { extname } from 'path'

@@ -33,7 +32,7 @@ class Logger {

32 if (!path)

33 return this.stream = undefined

34 try {

36 - const stats = await stat(path)

35 + const stats = await statWithTimeout(path)

36 this.last = stats.mtime

37 }

38 catch {

@@ -139,7 +138,7 @@ export const logMw: Koa.Middleware = async (ctx, next) => {

138 if (events.anyListener(logger.name)) // small optimization: this event can happen often, while most times there's no listener, and the parameters object is constructed pointlessly. A benchmark measured it 20% faster (just the line), while maybe it was not necessary.

139 events.emit(logger.name, { ctx, length, user, ts: now, uri, extra })

140 debounce(() => // once in a while we check if the file is still good (not deleted, etc), or we'll reopen it

142 - stat(logger.path).catch(() => logger.reopen())) // async = smoother but we may lose some entries

141 + statWithTimeout(logger.path).catch(() => logger.reopen())) // async = smoother but we may lose some entries

142 stream!.write(util.format( format,

143 ctx.ip,

144 user || '-',

src/serveGuiFiles.ts

+2 -3

@@ -1,7 +1,6 @@

2

3 import Koa from 'koa'

4 -import fs from 'fs/promises'

4 import {

5 API_VERSION, MIME_AUTO, FRONTEND_URI, HTTP_METHOD_NOT_ALLOWED, HTTP_NO_CONTENT, HTTP_NOT_FOUND,

6 PLUGINS_PUB_URI, VERSION, SPECIAL_URI, ICONS_URI, DEV

@@ -13,7 +12,7 @@ import { ApiError } from './apiMiddleware'

12 import { join, extname } from 'path'

13 import {

14 CFG, debounceAsync, formatBytes, FRONTEND_OPTIONS, isPrimitive, newObj, objSameKeys, onlyTruthy, parseFileContent,

16 - enforceStarting

15 + enforceStarting, statWithTimeout

16 } from './misc'

17 import { favicon, title } from './adminApis'

18 import { customHtml, getAllSections, getSection } from './customHtml'

@@ -68,7 +67,7 @@ function adjustBundlerLinks(ctx: Koa.Context, uri: string, data: string | Buffer

67

68 const getFaviconTimestamp = debounceAsync(async () => {

69 const f = favicon.get()

71 - return !f ? 0 : fs.stat(f).then(x => x?.mtimeMs || 0, () => 0)

70 + return !f ? 0 : statWithTimeout(f).then(x => x?.mtimeMs || 0, () => 0)

71 }, { retain: 5_000 })

72

73 async function treatIndex(ctx: Koa.Context, filesUri: string, body: string) {

src/update.ts

+3 -3

@@ -4,10 +4,10 @@ import { apiGithubPaginated, getProjectInfo, getRepoInfo } from './github'

4 import { ARGS_FILE, HFS_REPO, IS_BINARY, IS_WINDOWS, PREVIOUS_TAG, RUNNING_BETA } from './const'

5 import { dirname, join } from 'path'

6 import { spawn, spawnSync } from 'child_process'

7 -import { DAY, exists, debounceAsync, unzip, prefix, xlate, HOUR, httpWithBody } from './misc'

7 +import { DAY, exists, debounceAsync, unzip, prefix, xlate, HOUR, httpWithBody, statWithTimeout } from './misc'

8 import { createReadStream, existsSync, renameSync, unlinkSync, writeFileSync } from 'fs'

9 import { pluginsWatcher } from './plugins'

10 -import { chmod, rename, stat, writeFile } from 'fs/promises'

10 +import { chmod, rename, writeFile } from 'fs/promises'

11 import open from 'open'

12 import { currentVersion, defineConfig, versionToScalar } from './config'

13 import { cmdEscape, RUNNING_AS_SERVICE } from './util-os'

@@ -151,7 +151,7 @@ export async function update(tagOrUrl: string='') {

151 join(binPath, path === binFile ? newBinFile : path))

152 const newBin = join(binPath, newBinFile)

153 if (!IS_WINDOWS) {

154 - const { mode } = await stat(bin)

154 + const { mode } = await statWithTimeout(bin)

155 await chmod(newBin, mode).catch(console.error)

156 }

157 await rename(INSTALLED_FN, PREVIOUS_FN).catch(e => e?.code !== 'ENOENT' && console.warn(String(e)))

src/util-files.ts

+10 -3

@@ -1,7 +1,8 @@

2

3 import { access, mkdir, readFile, stat } from 'fs/promises'

4 -import { Promisable, try_, wait, isWindowsDrive } from './cross'

4 +import { Promisable, try_, wait, isWindowsDrive, haveTimeout } from './cross'

5 +import { defineConfig } from './config'

6 import { createWriteStream, mkdirSync, watch, ftruncate } from 'fs'

7 import { basename, dirname } from 'path'

8 import glob from 'fast-glob'

@@ -11,8 +12,14 @@ import { Readable } from 'stream'

12 // @ts-ignore

13 import unzipper from 'unzip-stream'

14

15 +const fileTimeout = defineConfig('file_timeout', 3, x => x * 1000)

16 +

17 +export function statWithTimeout(path: string) {

18 + return haveTimeout(fileTimeout.compiled(), stat(path))

19 +}

20 +

21 export async function isDirectory(path: string) {

15 - try { return (await stat(path)).isDirectory() }

22 + try { return (await statWithTimeout(path)).isDirectory() }

23 catch {}

24 }

25

@@ -148,7 +155,7 @@ export function exists(path: string) {

155 // parse a file, caching unless timestamp has changed

156 export const parseFileCache = new Map<string, { ts: Date, parsed: unknown }>()

157 export async function parseFile<T>(path: string, parse: (path: string) => T) {

151 - const { mtime: ts } = await stat(path)

158 + const { mtime: ts } = await statWithTimeout(path)

159 const cached = parseFileCache.get(path)

160 if (cached && Number(ts) === Number(cached.ts))

161 return cached.parsed as T

src/vfs.ts

+2 -1

@@ -5,6 +5,7 @@ import { basename, dirname, join, resolve } from 'path'

5 import {

6 makeMatcher, setHidden, onlyTruthy, isValidFileName, throw_, VfsPerms, Who, debounceAsync,

7 isWhoObject, WHO_ANY_ACCOUNT, defaultPerms, PERM_KEYS, removeStarting, HTTP_SERVER_ERROR, try_, matches,

8 + statWithTimeout,

9 } from './misc'

10 import Koa from 'koa'

11 import _ from 'lodash'

@@ -136,7 +137,7 @@ export async function urlToNode(url: string, ctx?: Koa.Context, parent: VfsNode=

137 export async function nodeStats(ret: VfsNode) {

138 if (ret.stats)

139 return ret.stats

139 - const stats = ret.source ? await fs.stat(ret.source) : undefined

140 + const stats = ret.source ? await statWithTimeout(ret.source) : undefined

141 setHidden(ret, { stats })

142 return stats

143 }

src/walkDir.ts

+3 -3

@@ -1,5 +1,5 @@

1 import { makeQ } from './makeQ'

2 -import { stat, opendir } from 'fs/promises'

2 +import { opendir } from 'fs/promises'

3 import { IS_WINDOWS } from './const'

4 import { join } from 'path'

5 import { pendingPromise, Promisable } from './cross'

@@ -8,7 +8,7 @@ import events from './events'

8 import _ from 'lodash'

9 import { Context } from 'koa'

10 import fswin from 'fswin'

11 -import { isDirectory } from './util-files'

11 +import { isDirectory, statWithTimeout } from './util-files'

12

13 export interface DirStreamEntry extends Dirent {

14 closingBranch?: Promise<string>

@@ -70,7 +70,7 @@ export function walkDir(path: string, { depth = 0, hidden = true, parallelizeRec

70 if (stopped) break

71 if (!hidden && entry.name[0] === '.' && !IS_WINDOWS)

72 continue

73 - const stats = entry.isSymbolicLink?.() && await stat(join(base, entry.name)).catch(() => null)

73 + const stats = entry.isSymbolicLink?.() && await statWithTimeout(join(base, entry.name)).catch(() => null)

74 if (stats === null) continue

75 if (stats)

76 entry = new DirentFromStats(entry.name, stats)

src/zip.ts

+2 -3

@@ -2,10 +2,9 @@

2

3 import { getNodeName, hasPermission, nodeIsFolder, nodeIsLink, urlToNode, VfsNode, walkNode, statusCodeForMissingPerm } from './vfs'

4 import Koa from 'koa'

5 -import { filterMapGenerator, isWindowsDrive, safeDecodeURIComponent, wantArray } from './misc'

5 +import { filterMapGenerator, isWindowsDrive, safeDecodeURIComponent, statWithTimeout, wantArray } from './misc'

6 import { QuickZipStream } from './QuickZipStream'

7 import { createReadStream } from 'fs'

8 -import fs from 'fs/promises'

8 import { defineConfig } from './config'

9 import { basename, dirname } from 'path'

10 import { applyRange, forceDownload, monitorAsDownload } from './serveFile'

@@ -56,7 +55,7 @@ export async function zipStreamFromFolder(node: VfsNode, ctx: Koa.Context) {

55 if (nodeIsFolder(el))

56 return { path: name + '/' }

57 if (!source) return

59 - const st = el.stats || await fs.stat(source)

58 + const st = el.stats || await statWithTimeout(source)

59 if (!st || !st.isFile())

60 return

61 return {

fix: admin/shared: vfs not loading in case of slow (or offline) networked sources