| 1 | // This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt |
| 2 | |
| 3 | import Koa from 'koa' |
| 4 | import { createReadStream, stat, Stats } from 'fs' |
| 5 | import { HTTP_BAD_REQUEST, HTTP_FORBIDDEN, HTTP_METHOD_NOT_ALLOWED, HTTP_NO_CONTENT, HTTP_NOT_FOUND, HTTP_NOT_MODIFIED, |
| 6 | HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_RANGE_NOT_SATISFIABLE, HTTP_TOO_MANY_REQUESTS, MIME_AUTO } from './const' |
| 7 | import { getNodeName, VfsNode } from './vfs' |
| 8 | import mimetypes from 'mime-types' |
| 9 | import { defineConfig } from './config' |
| 10 | import { CFG, Dict, makeMatcher, matches, try_ } from './misc' |
| 11 | import _ from 'lodash' |
| 12 | import { basename } from 'path' |
| 13 | import { promisify } from 'util' |
| 14 | import { getConnection, updateConnection } from './connections' |
| 15 | import { getCurrentUsername } from './auth' |
| 16 | import { sendErrorPage } from './errorPages' |
| 17 | import { Readable } from 'stream' |
| 18 | import { createHash } from 'crypto' |
| 19 | import iconv from 'iconv-lite' |
| 20 | |
| 21 | const allowedReferer = defineConfig('allowed_referer', '') |
| 22 | const maxDownloads = downloadLimiter(defineConfig(CFG.max_downloads, 0), () => true) |
| 23 | const maxDownloadsPerIp = downloadLimiter(defineConfig(CFG.max_downloads_per_ip, 0), ctx => ctx.ip) |
| 24 | const maxDownloadsPerAccount = downloadLimiter(defineConfig(CFG.max_downloads_per_account, 0), ctx => getCurrentUsername(ctx) || undefined) |
| 25 | |
| 26 | const GUI_ASSET_MIME = /^(image\/|audio\/|video\/|font\/|text\/css$|(?:text|application)\/(?:javascript|ecmascript)$|application\/x-javascript$)/i |
| 27 | |
| 28 | function toAsciiEquivalent(s: string) { |
| 29 | return iconv.encode(iconv.decode(Buffer.from(s), 'utf-8'), 'ascii').toString().replaceAll('?', '') |
| 30 | } |
| 31 | |
| 32 | export function forceDownload(ctx: Koa.Context, name: string) { |
| 33 | disposition(ctx, name, true) |
| 34 | } |
| 35 | |
| 36 | export function disposition(ctx: Koa.Context, name: string, forceDownload=false) { |
| 37 | // override Koa's question-mark fallback for decomposed Unicode filenames on Windows |
| 38 | ctx.attachment(name, { |
| 39 | type: forceDownload ? 'attachment' : 'inline', |
| 40 | fallback: toAsciiEquivalent(name), |
| 41 | }) |
| 42 | } |
| 43 | |
| 44 | export async function serveFileNode(ctx: Koa.Context, node: VfsNode) { |
| 45 | const { source, mime } = node |
| 46 | const name = getNodeName(node) |
| 47 | const mimeString = typeof mime === 'string' ? mime |
| 48 | : _.find(mime, (_val,mask) => matches(name, mask)) |
| 49 | if (allowedReferer.get()) { |
| 50 | const ref = try_(() => new URL(ctx.get('referer')||'').host) |
| 51 | if (ref && ref !== ctx.host // automatically accept if the referer is basically the hosting domain |
| 52 | && !matches(ref, allowedReferer.get())) |
| 53 | return ctx.status = HTTP_FORBIDDEN |
| 54 | } |
| 55 | |
| 56 | ctx.state.vfsNode = node // useful to tell service files from files shared by the user |
| 57 | const download = 'dl' in ctx.query |
| 58 | disposition(ctx, name, download) |
| 59 | const fetchDest = ctx.get('sec-fetch-dest') |
| 60 | ctx.state.considerAsGui ??= !download && ctx.get('referer')?.endsWith('/') |
| 61 | && (fetchDest ? fetchDest !== 'document' && fetchDest !== 'empty' // modern clients |
| 62 | // legacy clients often send Accept: */* for archive downloads, so the served mime is a safer signal than request headers here |
| 63 | : GUI_ASSET_MIME.test(mimeString || mimetypes.lookup(source||'') || '')) |
| 64 | await serveFile(ctx, source||'', mimeString) |
| 65 | |
| 66 | if (await maxDownloadsPerAccount(ctx) === undefined) // returning false will not execute other limits |
| 67 | await maxDownloads(ctx) || await maxDownloadsPerIp(ctx) |
| 68 | } |
| 69 | |
| 70 | const mimeCfg = defineConfig<Dict<string>, (name: string) => string | undefined>('mime', {}, obj => { |
| 71 | const matchers = Object.keys(obj).map(k => makeMatcher(k)) |
| 72 | const values = Object.values(obj) |
| 73 | return (name: string) => values[matchers.findIndex(matcher => matcher(name))] |
| 74 | }) |
| 75 | |
| 76 | // after this number of seconds, the browser should check the server to see if there's a newer version of the file |
| 77 | const cacheControlDiskFiles = defineConfig('cache_control_disk_files', 5) |
| 78 | |
| 79 | export async function serveFile(ctx: Koa.Context, filePath:string, mime?:string, cached?: { stats: Stats, content: string | Buffer }) { |
| 80 | if (!filePath) |
| 81 | return |
| 82 | mime ??= mimeCfg.compiled()(basename(filePath)) |
| 83 | if (mime === undefined || mime === MIME_AUTO) |
| 84 | mime = mimetypes.lookup(filePath) || '' |
| 85 | if (mime) |
| 86 | ctx.type = mime |
| 87 | if (ctx.method === 'OPTIONS') { |
| 88 | ctx.status = HTTP_NO_CONTENT |
| 89 | ctx.set({ Allow: 'OPTIONS, GET, HEAD' }) |
| 90 | return |
| 91 | } |
| 92 | if (ctx.method !== 'GET') |
| 93 | return ctx.status = HTTP_METHOD_NOT_ALLOWED |
| 94 | try { |
| 95 | const stats = cached?.stats || await promisify(stat)(filePath) // using fs's function instead of fs/promises, because only the former is supported by pkg |
| 96 | if (!stats.isFile()) |
| 97 | return ctx.status = HTTP_METHOD_NOT_ALLOWED |
| 98 | const t = stats.mtime.toUTCString() |
| 99 | ctx.set('Last-Modified', t) |
| 100 | ctx.set('Etag', createHash('sha256').update(filePath).update(t).digest('hex')) |
| 101 | ctx.state.fileSource = filePath |
| 102 | ctx.state.fileStats = stats |
| 103 | ctx.status = HTTP_OK |
| 104 | if (ctx.fresh) |
| 105 | return ctx.status = HTTP_NOT_MODIFIED |
| 106 | if (cached) |
| 107 | return ctx.body = cached.content |
| 108 | const cc = cacheControlDiskFiles.get() |
| 109 | if (_.isNumber(cc)) |
| 110 | ctx.set('Cache-Control', `max-age=${cc}`) |
| 111 | const { size } = stats |
| 112 | const range = applyRange(ctx, size) |
| 113 | if (ctx.status >= 400) return // applyRange may have set an error |
| 114 | ctx.body = createReadStream(filePath, range || undefined) |
| 115 | if (ctx.state.vfsNode) |
| 116 | monitorAsDownload(ctx, size, range?.start) |
| 117 | } |
| 118 | catch (e: any) { |
| 119 | const status = { |
| 120 | ENOENT: HTTP_NOT_FOUND, |
| 121 | ENOTDIR: HTTP_NOT_FOUND, |
| 122 | EACCES: HTTP_FORBIDDEN, |
| 123 | EPERM: HTTP_FORBIDDEN, |
| 124 | }[String(e?.code)] |
| 125 | if (!status) |
| 126 | throw e |
| 127 | ctx.status = status |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | export function monitorAsDownload(ctx: Koa.Context, size?: number, offset?: number) { |
| 132 | if (!(ctx.body instanceof Readable)) |
| 133 | throw 'incompatible body' |
| 134 | const conn = getConnection(ctx) |
| 135 | ctx.body.on('end', () => |
| 136 | updateConnection(conn, {}, { opProgress: 1 }) ) |
| 137 | updateConnection(conn, {}, { |
| 138 | opProgress: 0, |
| 139 | opTotal: size, |
| 140 | opOffset: size && offset && (offset / size), |
| 141 | }) |
| 142 | } |
| 143 | |
| 144 | declare module "koa" { |
| 145 | interface DefaultState { |
| 146 | opProgress?: number |
| 147 | opTotal?: number |
| 148 | opOffset?: number |
| 149 | vfsNode?: VfsNode |
| 150 | includesLastByte?: boolean |
| 151 | fileSource?: string |
| 152 | fileStats?: Stats |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | export function applyRange(ctx: Koa.Context, totalSize=ctx.response.length): { start: number, end: number } | void { |
| 157 | ctx.set('Accept-Ranges', 'bytes') |
| 158 | const { range } = ctx.request.header |
| 159 | if (!range || isNaN(totalSize)) { |
| 160 | ctx.state.includesLastByte = true |
| 161 | if (!isNaN(totalSize)) |
| 162 | ctx.response.length = totalSize |
| 163 | return |
| 164 | } |
| 165 | const [unit, ranges] = range.split('=') |
| 166 | if (unit !== 'bytes') |
| 167 | return badRequest('bad range unit') |
| 168 | if (ranges?.includes(',')) |
| 169 | return badRequest('multi-range not supported') |
| 170 | const bytes = ranges?.split('-') |
| 171 | if (bytes?.length !== 2) |
| 172 | return badRequest('bad range') |
| 173 | const max = totalSize - 1 |
| 174 | const [startTxt, endTxt] = bytes |
| 175 | const start = startTxt ? Number(startTxt) : Math.max(0, totalSize-Number(endTxt)) // a negative start is relative to the end |
| 176 | const end = (startTxt && endTxt) ? Math.min(max, Number(endTxt)) : max |
| 177 | if (isNaN(start) || startTxt && endTxt && isNaN(end)) |
| 178 | return badRequest('bad range') |
| 179 | // we don't support last-bytes without knowing max |
| 180 | if (isNaN(end) && isNaN(max) || end > max || start > max || start > end) { |
| 181 | ctx.set('Content-Range', `bytes */${totalSize}`) |
| 182 | return badRequest('Requested Range Not Satisfiable', HTTP_RANGE_NOT_SATISFIABLE) |
| 183 | } |
| 184 | ctx.state.includesLastByte = end === max |
| 185 | ctx.status = HTTP_PARTIAL_CONTENT |
| 186 | ctx.set('Content-Range', `bytes ${start}-${isNaN(end) ? '' : end}/${isNaN(totalSize) ? '*' : totalSize}`) |
| 187 | ctx.response.length = end - start + 1 |
| 188 | return { start, end } |
| 189 | |
| 190 | function badRequest(message: string, status=HTTP_BAD_REQUEST) { |
| 191 | ctx.status = status |
| 192 | ctx.body = message |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | function downloadLimiter<T>(configMax: { get: () => number | undefined }, cbKey: (ctx: Koa.Context) => T | undefined) { |
| 197 | const map = new Map<T, number>() |
| 198 | return (ctx: Koa.Context) => { |
| 199 | if (!ctx.body || ctx.state.considerAsGui) return // !body = no file sent, cache hit |
| 200 | const k = cbKey(ctx) |
| 201 | if (k === undefined) return // undefined = skip limit |
| 202 | const max = configMax.get() |
| 203 | const now = map.get(k) || 0 |
| 204 | if (max && now >= max) { |
| 205 | ctx.set('retry-after', '60') |
| 206 | return sendErrorPage(ctx, HTTP_TOO_MANY_REQUESTS) |
| 207 | .then(() => true) // true = limit exceeded |
| 208 | } |
| 209 | map.set(k, now + 1) |
| 210 | ctx.req.on('close', () => { |
| 211 | const n = map.get(k)! |
| 212 | if (n > 1) |
| 213 | map.set(k, n - 1) |
| 214 | else |
| 215 | map.delete(k) |
| 216 | }) |
| 217 | return false // limit is enforced but passed |
| 218 | } |
| 219 | } |