@samitouri / QOSami-HFS / commits / 49841ada

dev: add_vfs.link

Massimo Melina committed Oct 12, 2023 at 00:02 UTC 49841adaf52f58f7b99868501705758b3afa2f62

13 files changed +353 -296

admin/src/FileForm.ts

+4 -3

@@ -227,7 +227,7 @@ interface LinkFieldProps extends FieldProps<string> {

227 function LinkField({ value, statusApi }: LinkFieldProps) {

228 const { data, reload, error } = statusApi

229 const urls: string[] = data?.urls.https || data?.urls.http

230 - const link = (data?.baseUrl || urls?.[0] || '') + value

230 + const link = (data?.baseUrl || '') + value

231 return h(Box, { display: 'flex' },

232 !urls ? 'error' : // check data is ok

233 h(DisplayField, {

@@ -249,11 +249,12 @@ function LinkField({ value, statusApi }: LinkFieldProps) {

249 export async function changeBaseUrl() {

250 return new Promise(async resolve => {

251 const res = await apiCall('get_status')

252 + const { base_url } = await apiCall('get_config', { only: ['base_url'] })

253 const urls: string[] = res.urls.https || res.urls.http

254 const { close } = newDialog({

255 title: "Base address",

256 Content() {

256 - const [v, setV] = useState(res.baseUrl || '')

257 + const [v, setV] = useState(base_url || '')

258 const proto = new URL(v || urls[0]).protocol + '//'

259 const host = urls.includes(v) ? '' : v.slice(proto.length)

260 const check = h(Check, { sx: { ml: 2 } })

@@ -290,7 +291,7 @@ export async function changeBaseUrl() {

291 icon: Save,

292 children: "Save",

293 async onClick() {

293 - if (v !== res.baseUrl)

294 + if (v !== base_url)

295 await apiCall('set_config', { values: { base_url: v.replace(/\/$/, '') } })

296 resolve(v)

297 close()

admin/src/InternetPage.ts

+4 -3

@@ -20,6 +20,7 @@ export default function InternetPage() {

20 const [mapping, setMapping] = useState(false)

21 const [verifyAgain, setVerifyAgain] = useState(false)

22 const status = useApiEx('get_status')

23 + const { data: config } = useApiEx('get_config', { only: ['base_url'] })

24 const localColor = with_([status.data?.http?.error, status.data?.https?.error], ([h, s]) =>

25 h && s ? 'error' : h || s ? 'warning' : 'success')

26 const { data: nat, reload: reloadNat, error, loading, element } = useApiEx<GetNat>('get_nat')

@@ -102,13 +103,13 @@ export default function InternetPage() {

103 }

104

105 function baseUrlBox() {

105 - const url = status.data?.baseUrl

106 + const url = config?.base_url

107 const hostname = url && new URL(url).hostname

108 const domain = !isIP(hostname) && hostname

109 return status.element || h(Card, {}, h(CardContent, {},

110 h(Box, { fontSize: 'x-large', mb: 2 }, "Address / Domain"),

111 h(Flex, { flexWrap: 'wrap', alignItems: 'center' },

111 - status.data?.baseUrl || "Automatic, not configured",

112 + url || "Automatic, not configured",

113 h(Button, {

114 size: 'small',

115 onClick() { changeBaseUrl().then(status.reload) }

@@ -157,7 +158,7 @@ export default function InternetPage() {

158 if (!verifyAgain && !await confirmDialog("This test will check if your server is working properly on the Internet")) return

159 setChecking(true)

160 try {

160 - const url = status.data?.baseUrl

161 + const url = config.base_url

162 const urlResult = url && await apiCall('self_check', { url }).catch(() =>

163 alertDialog(md(`Sorry, we couldn't verify your configured address ${url} 😰\nstill, we are going to test your IP address 🤞`), 'warning'))

164 if (urlResult?.success) {

src/acme.ts new

+112

@@ -0,0 +1,112 @@

1 +import { DAY, Dict, HOUR, HTTP_BAD_REQUEST, HTTP_FAILED_DEPENDENCY, HTTP_OK, repeat } from './cross'

2 +import { createServer, RequestListener } from 'http'

3 +import { Middleware } from 'koa'

4 +import { getNatInfo, upnpClient } from './nat'

5 +import { cert, getCertObject, getServerStatus, privateKey } from './listen'

6 +import { ApiError } from './apiMiddleware'

7 +import acme from 'acme-client'

8 +import { debounceAsync } from './debounceAsync'

9 +import fs from 'fs/promises'

10 +import { defineConfig } from './config'

11 +import events from './events'

12 +import { selfCheck } from './selfCheck'

13 +

14 +let acmeMiddlewareEnabled = false

15 +const acmeTokens: Dict<string> = {}

16 +const acmeListener: RequestListener = (req, res) => { // node format

17 + const BASE = '/.well-known/acme-challenge/'

18 + if (!req.url?.startsWith(BASE)) return

19 + const token = req.url.slice(BASE.length)

20 + console.debug("got http challenge", token)

21 + res.statusCode = HTTP_OK

22 + res.end(acmeTokens[token])

23 + return true

24 +}

25 +export const acmeMiddleware: Middleware = (ctx, next) => { // koa format

26 + if (!acmeMiddlewareEnabled || !Boolean(acmeListener(ctx.req, ctx.res)))

27 + return next()

28 +}

29 +

30 +async function generateSSLCert(domain: string, email?: string) {

31 + // will answer challenge through our koa app (if on port 80) or must we spawn a dedicated server?

32 + const { upnp, externalPort } = await getNatInfo()

33 + const { http } = await getServerStatus()

34 + const tempSrv = externalPort === 80 || http.listening && http.port === 80 ? undefined : createServer(acmeListener)

35 + if (tempSrv)

36 + await new Promise<void>((resolve) =>

37 + tempSrv.listen(80, resolve).on('error', (e: any) => {

38 + console.debug("cannot listen on 80", e.code || e)

39 + resolve() // go on anyway

40 + }) )

41 + acmeMiddlewareEnabled = true

42 + console.debug('acme challenge server ready')

43 + try {

44 + const checkUrl = `http://${domain}`

45 + let check = await selfCheck(checkUrl) // some check services may not consider the domain, but we already verified that

46 + if (check && !check.success && upnp && externalPort !== 80) { // consider a short-lived mapping

47 + console.debug("setting temporary port forward")

48 + // @ts-ignore

49 + await upnpClient.createMapping({ private: 80, public: { host: '', port: 80 }, description: 'hfs temporary', ttl: 30 }).catch(() => {})

50 + check = await selfCheck(checkUrl) // repeat test

51 + }

52 + //if (!check) throw new ApiError(HTTP_FAILED_DEPENDENCY, "couldn't test port 80")

53 + if (!check?.success)

54 + throw new ApiError(HTTP_FAILED_DEPENDENCY, "port 80 is not working on the specified domain")

55 + const acmeClient = new acme.Client({

56 + accountKey: await acme.crypto.createPrivateKey(),

57 + directoryUrl: acme.directory.letsencrypt.production

58 + })

59 + const [key, csr] = await acme.crypto.createCsr({ commonName: domain })

60 + const cert = await acmeClient.auto({

61 + csr,

62 + email,

63 + challengePriority: ['http-01'],

64 + skipChallengeVerification: true, // on NAT, trying to connect to your external ip will likely get your modem instead of the challenge server

65 + termsOfServiceAgreed: true,

66 + async challengeCreateFn(_, c, ka) {

67 + console.debug("producing challenge")

68 + acmeTokens[c.token] = ka

69 + },

70 + async challengeRemoveFn(_, c) {

71 + delete acmeTokens[c.token]

72 + },

73 + })

74 + return { key, cert }

75 + }

76 + finally {

77 + acmeMiddlewareEnabled = false

78 + if (tempSrv) await new Promise(res => tempSrv.close(res))

79 + console.debug('acme terminated')

80 + }

81 +}

82 +

83 +export const makeCert = debounceAsync(async (domain: string, email?: string) => {

84 + if (!domain) return new ApiError(HTTP_BAD_REQUEST, 'bad params')

85 + const res = await generateSSLCert(domain, email)

86 + const CERT_FILE = 'acme.cert'

87 + const KEY_FILE = 'acme.key'

88 + await fs.writeFile(CERT_FILE, res.cert)

89 + await fs.writeFile(KEY_FILE, res.key)

90 + cert.set(CERT_FILE) // update config

91 + privateKey.set(KEY_FILE)

92 +}, 0)

93 +

94 +defineConfig('acme_renew', false) // handle config changes

95 +events.once('https ready', () => repeat(HOUR, renewCert))

96 +

97 +export const acme_domain = defineConfig<string>('acme_domain', '')

98 +export const acme_email = defineConfig<string>('acme_email', '')

99 +

100 +// checks if the cert is near expiration date, and if so renews it

101 +const renewCert = debounceAsync(async () => {

102 + const cert = getCertObject()

103 + if (!cert) return

104 + const now = new Date()

105 + const validTo = new Date(cert.validTo)

106 + // not expiring in a month

107 + if (now > new Date(cert.validFrom) && now < validTo && validTo.getTime() - now.getTime() >= 30 * DAY)

108 + return console.log("certificate still good")

109 + await makeCert(acme_domain.get(), acme_email.get())

110 + .catch(e => console.log("error renewing certificate: ", String(e)))

111 +}, 0, { retain: DAY, retainFailure: HOUR })

112 +

src/adminApis.ts

+3 -3

@@ -2,7 +2,7 @@

3 import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'

4 import { configFile, defineConfig, getWholeConfig, setConfig } from './config'

5 -import { getIps, getServerStatus, getUrls } from './listen'

5 +import { getBaseUrlOrDefault, getIps, getServerStatus, getUrls } from './listen'

6 import {

7 API_VERSION,

8 BUILD_TIMESTAMP,

@@ -23,7 +23,7 @@ import { debounceAsync, isLocalHost, makeNetMatcher, onOff, tryJson, wait, waitF

23 import events from './events'

24 import { accountCanLoginAdmin, accountsConfig, getFromAccount } from './perm'

25 import Koa from 'koa'

26 -import { baseUrl, getProxyDetected } from './middlewares'

26 +import { getProxyDetected } from './middlewares'

27 import { writeFile } from 'fs/promises'

28 import { createReadStream } from 'fs'

29 import * as readline from 'readline'

@@ -104,7 +104,7 @@ export const adminApis: ApiHandlers = {

104 ...await getServerStatus(),

105 urls: await getUrls(),

106 ips: await getIps(false),

107 - baseUrl: baseUrl.get(), // can be retrieved with get_config, but it's very handy with urls and low overhead. Case is different because the context is

107 + baseUrl: getBaseUrlOrDefault(),

108 updatePossible: !updateSupported() ? false : await localUpdateAvailable() ? 'local' : true,

109 proxyDetected: getProxyDetected(),

110 frpDetected: localhostAdmin.get() && !getProxyDetected()

src/api.net.ts

+28 -269

@@ -1,253 +1,42 @@

3 import { ApiError, ApiHandlers } from './apiMiddleware'

4 -import { Client } from 'nat-upnp-ts'

5 -import { HTTP_BAD_REQUEST, HTTP_FAILED_DEPENDENCY, HTTP_OK, HTTP_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE,

6 - HTTP_PRECONDITION_FAILED, IS_MAC, IS_WINDOWS, SPECIAL_URI

7 -} from './const'

4 +import { HTTP_FAILED_DEPENDENCY, HTTP_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, HTTP_PRECONDITION_FAILED } from './const'

5 import _ from 'lodash'

9 -import { cert, getCertObject, getIps, getServerStatus, privateKey } from './listen'

6 +import { getCertObject } from './listen'

7 import { getProjectInfo } from './github'

11 -import { httpString } from './util-http'

12 -import { exec } from 'child_process'

13 -import {

14 - apiAssertTypes, DAY, debounceAsync, haveTimeout, HOUR, MINUTE, objSameKeys, onlyTruthy, repeat, Dict,

15 - GetNat, promiseBestEffort, wantArray

16 -} from './misc'

17 -import acme from 'acme-client'

18 -import fs from 'fs/promises'

19 -import { createServer, RequestListener } from 'http'

20 -import { Middleware } from 'koa'

8 +import { apiAssertTypes, objSameKeys, onlyTruthy, promiseBestEffort } from './misc'

9 import { lookup, Resolver } from 'dns/promises'

22 -import { defineConfig } from './config'

23 -import events from './events'

24 -import { isIP, isIPv6 } from 'net'

25 -

26 -const upnpClient = new Client({ timeout: 4_000 })

27 -const originalMethod = upnpClient.getGateway

28 -// other client methods call getGateway too, so this will ensure they reuse this same result

29 -upnpClient.getGateway = debounceAsync(() => originalMethod.apply(upnpClient), 0, { retain: HOUR, retainFailure: 30_000 })

30 -upnpClient.getGateway().catch(() => {})

31 -

32 -export let externalIp = '' // poll external ip

33 -repeat(10 * MINUTE, () => upnpClient.getPublicIp().then(v => {

34 - if (v !== externalIp)

35 - getPublicIps.clearRetain()

36 - return externalIp = v

37 -}))

38 -

39 -const getNatInfo = debounceAsync(async () => {

40 - const gettingIps = getPublicIps() // don't wait, do it in parallel

41 - const res = await upnpClient.getGateway().catch(() => null)

42 - const status = await getServerStatus()

43 - const mappings = res && await haveTimeout(5_000, upnpClient.getMappings()).catch(() => null)

44 - console.debug('mappings found', mappings?.map(x => x.description))

45 - const gatewayIp = res ? new URL(res.gateway.description).hostname : await findGateway().catch(() => undefined)

46 - const localIp = res?.address || (await getIps())[0]

47 - const internalPort = status?.https?.listening && status.https.port || status?.http?.listening && status.http.port || undefined

48 - const mapped = _.find(mappings, x => x.private.host === localIp && x.private.port === internalPort)

49 - return {

50 - upnp: Boolean(res),

51 - localIp,

52 - gatewayIp,

53 - publicIps: await gettingIps,

54 - externalIp,

55 - mapped,

56 - internalPort,

57 - externalPort: mapped?.public.port,

58 - proto: status?.https?.listening ? 'https' : status?.http?.listening ? 'http' : undefined,

59 - } satisfies GetNat

60 -})

61 -

62 -const getPublicIps = debounceAsync(async () => {

63 - const res = await getProjectInfo()

64 - const groupedByVersion = Object.values(_.groupBy(res.publicIpServices, x => x.v ?? 4))

65 - const ips = await promiseBestEffort(groupedByVersion.map(singleVersion =>

66 - Promise.any(singleVersion.map(async (svc: any) => {

67 - if (typeof svc === 'string')

68 - svc = { type: 'http', url: svc }

69 - console.debug("trying ip service", svc.url || svc.name)

70 - if (svc.type === 'http')

71 - return httpString(svc.url)

72 - if (svc.type !== 'dns') throw "unsupported"

73 - const resolver = new Resolver({ timeout: 2_000 })

74 - resolver.setServers(svc.ips)

75 - return resolver.resolve(svc.name, svc.dnsRecord)

76 - }).map(async ret => {

77 - const validIps = wantArray(await ret).map(x => x.trim()).filter(isIP)

78 - if (!validIps.length) throw "no good"

79 - return validIps

80 - }) )))

81 - return _.uniq(ips.flat())

82 -}, 0, { retain: 10 * MINUTE })

83 -

84 -function findGateway(): Promise<string | undefined> {

85 - return new Promise((resolve, reject) =>

86 - exec(IS_WINDOWS || IS_MAC ? 'netstat -rn' : 'route -n', (err, out) => {

87 - if (err) return reject(err)

88 - const re = IS_WINDOWS ? /(?:0\.0\.0\.0 +){2}([\d.]+)/ : IS_MAC ? /default +([\d.]+)/ : /^0\.0\.0\.0 +([\d.]+)/

89 - resolve(re.exec(out)?.[1])

90 - }) )

91 -}

92 -

93 -let acmeMiddlewareEnabled = false

94 -const acmeTokens: Dict<string> = {}

95 -const acmeListener: RequestListener = (req, res) => { // node format

96 - const BASE = '/.well-known/acme-challenge/'

97 - if (!req.url?.startsWith(BASE)) return

98 - const token = req.url.slice(BASE.length)

99 - console.debug("got http challenge", token)

100 - res.statusCode = HTTP_OK

101 - res.end(acmeTokens[token])

102 - return true

103 -}

104 -export const acmeMiddleware: Middleware = (ctx, next) => { // koa format

105 - if (!acmeMiddlewareEnabled || !Boolean(acmeListener(ctx.req, ctx.res)))

106 - return next()

107 -}

108 -

109 -let selfCheckMiddlewareEnabled = false

110 -const CHECK_URL = SPECIAL_URI + 'self-check'

111 -export const selfCheckMiddleware: Middleware = (ctx, next) => { // koa format

112 - if (!selfCheckMiddlewareEnabled || !ctx.url.startsWith(CHECK_URL))

113 - return next()

114 - ctx.body = 'HFS'

115 -}

116 -

117 -async function checkDomain(domain: string) {

118 - const resolver = new Resolver()

119 - const prjInfo = await getProjectInfo()

120 - resolver.setServers(prjInfo.dnsServers)

121 - const settled = await Promise.allSettled([

122 - resolver.resolve(domain, 'A'),

123 - resolver.resolve(domain, 'AAAA'),

124 - lookup(domain).then(x => [x.address]),

125 - ])

126 - // merge all results

127 - const domainIps = _.uniq(onlyTruthy(settled.map(x => x.status === 'fulfilled' && x.value)).flat())

128 - if (!domainIps.length)

129 - throw new ApiError(HTTP_FAILED_DEPENDENCY, "domain not working")

130 - const { publicIps } = await getNatInfo() // do this before stopping the server

131 - for (const v6 of [false, true]) {

132 - const domainIpsThisVersion = domainIps.filter(x => isIPv6(x) === v6)

133 - const ipsThisVersion = publicIps.filter(x => isIPv6(x) === v6)

134 - if (domainIpsThisVersion.length && ipsThisVersion.length && !_.intersection(domainIpsThisVersion, ipsThisVersion).length)

135 - throw new ApiError(HTTP_PRECONDITION_FAILED, `configure your domain to point to ${ipsThisVersion} (currently on ${domainIpsThisVersion[0]}) – a change can take hours to be effective`)

136 - }

137 -}

138 -

139 -async function generateSSLCert(domain: string, email?: string) {

140 - // will answer challenge through our koa app (if on port 80) or must we spawn a dedicated server?

141 - const { upnp, externalPort } = await getNatInfo()

142 - const { http } = await getServerStatus()

143 - const tempSrv = externalPort === 80 || http.listening && http.port === 80 ? undefined : createServer(acmeListener)

144 - if (tempSrv)

145 - await new Promise<void>((resolve) =>

146 - tempSrv.listen(80, resolve).on('error', (e: any) => {

147 - console.debug("cannot listen on 80", e.code || e)

148 - resolve() // go on anyway

149 - }) )

150 - acmeMiddlewareEnabled = true

151 - console.debug('acme challenge server ready')

152 - try {

153 - const checkUrl = `http://${domain}`

154 - let check = await selfCheck(checkUrl) // some check services may not consider the domain, but we already verified that

155 - if (check && !check.success && upnp && externalPort !== 80) { // consider a short-lived mapping

156 - console.debug("setting temporary port forward")

157 - // @ts-ignore

158 - await upnpClient.createMapping({ private: 80, public: { host: '', port: 80 }, description: 'hfs temporary', ttl: 30 }).catch(() => {})

159 - check = await selfCheck(checkUrl) // repeat test

160 - }

161 - //if (!check) throw new ApiError(HTTP_FAILED_DEPENDENCY, "couldn't test port 80")

162 - if (!check?.success)

163 - throw new ApiError(HTTP_FAILED_DEPENDENCY, "port 80 is not working on the specified domain")

164 - const acmeClient = new acme.Client({

165 - accountKey: await acme.crypto.createPrivateKey(),

166 - directoryUrl: acme.directory.letsencrypt.production

167 - })

168 - const [key, csr] = await acme.crypto.createCsr({ commonName: domain })

169 - const cert = await acmeClient.auto({

170 - csr,

171 - email,

172 - challengePriority: ['http-01'],

173 - skipChallengeVerification: true, // on NAT, trying to connect to your external ip will likely get your modem instead of the challenge server

174 - termsOfServiceAgreed: true,

175 - async challengeCreateFn(_, c, ka) {

176 - console.debug("producing challenge")

177 - acmeTokens[c.token] = ka

178 - },

179 - async challengeRemoveFn(_, c) {

180 - delete acmeTokens[c.token]

181 - },

182 - })

183 - return { key, cert }

184 - }

185 - finally {

186 - acmeMiddlewareEnabled = false

187 - if (tempSrv) await new Promise(res => tempSrv.close(res))

188 - console.debug('acme terminated')

189 - }

190 -}

191 -

192 -async function checkService(url: string, serviceKey: string) {

193 - interface PortScannerService {

194 - type?: string

195 - url: string

196 - headers: {[k: string]: string}

197 - method: string

198 - body?: string

199 - regexpFailure: string

200 - regexpSuccess: string

201 - }

202 - const prjInfo = await getProjectInfo()

203 - console.log(`checking server ${url}`)

204 - selfCheckMiddlewareEnabled = true

205 - try {

206 - const parsed = new URL(url)

207 - const family = !isIP(parsed.hostname) ? undefined : isIPv6(parsed.hostname) ? 6 : 4

208 - for (const services of _.chunk(_.shuffle<PortScannerService>(prjInfo[serviceKey]), 2)) {

209 - try {

210 - return await Promise.any(services.map(async (svc) => {

211 - if (!svc.url || svc.type) throw 'unsupported ' + svc.type // only default type supported for now

212 - let { url: serviceUrl, body, regexpSuccess, regexpFailure, ...rest } = svc

213 - const service = new URL(serviceUrl).hostname

214 - console.log('trying external service', service)

215 - body = applySymbols(body)

216 - serviceUrl = applySymbols(serviceUrl)!

217 - const res = await haveTimeout(10_000, httpString(serviceUrl, { family, ...rest, body }))

218 - const success = new RegExp(regexpSuccess).test(res)

219 - const failure = new RegExp(regexpFailure).test(res)

220 - if (success === failure) throw 'inconsistent: ' + service + ': ' + res // this result cannot be trusted

221 - console.debug(service, 'responded', success)

222 - return { success, service, url }

223 - }))

224 - }

225 - catch (e: any) {

226 - console.debug(e?.errors?.map(String) || e?.cause || String(e))

227 - }

228 - }

229 -

230 - function applySymbols(s?: string) {

231 - return s?.replace('$IP', parsed.hostname)

232 - .replace('$PORT', parsed.port || (parsed.protocol === 'https:' ? '443' : '80'))

233 - .replace('$URL', url.replace(/\/$/, '') + CHECK_URL)

234 - }

235 - }

236 - finally {

237 - selfCheckMiddlewareEnabled = false

238 - }

239 -}

240 -

241 -function selfCheck(url: string) {

242 - return checkService(url, 'selfCheckServices')

243 -}

10 +import { isIPv6 } from 'net'

11 +import { getNatInfo, upnpClient } from './nat'

12 +import { makeCert } from './acme'

13 +import { selfCheck } from './selfCheck'

15 const apis: ApiHandlers = {

16 get_nat: getNatInfo,

248 - check_domain({ domain }) {

18 + async check_domain({ domain }) {

19 apiAssertTypes({ string: domain })

250 - return checkDomain(domain)

20 + const resolver = new Resolver()

21 + const prjInfo = await getProjectInfo()

22 + resolver.setServers(prjInfo.dnsServers)

23 + const settled = await Promise.allSettled([

24 + resolver.resolve(domain, 'A'),

25 + resolver.resolve(domain, 'AAAA'),

26 + lookup(domain).then(x => [x.address]),

27 + ])

28 + // merge all results

29 + const domainIps = _.uniq(onlyTruthy(settled.map(x => x.status === 'fulfilled' && x.value)).flat())

30 + if (!domainIps.length)

31 + throw new ApiError(HTTP_FAILED_DEPENDENCY, "domain not working")

32 + const { publicIps } = await getNatInfo() // do this before stopping the server

33 + for (const v6 of [false, true]) {

34 + const domainIpsThisVersion = domainIps.filter(x => isIPv6(x) === v6)

35 + const ipsThisVersion = publicIps.filter(x => isIPv6(x) === v6)

36 + if (domainIpsThisVersion.length && ipsThisVersion.length && !_.intersection(domainIpsThisVersion, ipsThisVersion).length)

37 + throw new ApiError(HTTP_PRECONDITION_FAILED, `configure your domain to point to ${ipsThisVersion} (currently on ${domainIpsThisVersion[0]}) – a change can take hours to be effective`)

38 + }

39 + return {}

40 },

42 async map_port({ external, internal }) {

@@ -291,34 +80,4 @@ const apis: ApiHandlers = {

80 }

81 }

294 -export const acme_domain = defineConfig<string>('acme_domain', '')

295 -export const acme_email = defineConfig<string>('acme_email', '')

296 -

297 -export const makeCert = debounceAsync(async (domain: string, email?: string) => {

298 - if (!domain) return new ApiError(HTTP_BAD_REQUEST, 'bad params')

299 - const res = await generateSSLCert(domain, email)

300 - const CERT_FILE = 'acme.cert'

301 - const KEY_FILE = 'acme.key'

302 - await fs.writeFile(CERT_FILE, res.cert)

303 - await fs.writeFile(KEY_FILE, res.key)

304 - cert.set(CERT_FILE) // update config

305 - privateKey.set(KEY_FILE)

306 -}, 0)

307 -

308 -defineConfig('acme_renew', false) // handle config changes

309 -events.once('https ready', () => repeat(HOUR, renewCert))

310 -

311 -// checks if the cert is near expiration date, and if so renews it

312 -const renewCert = debounceAsync(async () => {

313 - const cert = getCertObject()

314 - if (!cert) return

315 - const now = new Date()

316 - const validTo = new Date(cert.validTo)

317 - // not expiring in a month

318 - if (now > new Date(cert.validFrom) && now < validTo && validTo.getTime() - now.getTime() >= 30 * DAY)

319 - return console.log("certificate still good")

320 - await makeCert(acme_domain.get(), acme_email.get())

321 - .catch(e => console.log("error renewing certificate: ", String(e)))

322 -}, 0, { retain: DAY, retainFailure: HOUR })

323 -

83 export default apis

\ No newline at end of file

src/api.vfs.ts

+15 -7

@@ -6,13 +6,14 @@ import _ from 'lodash'

6 import { stat } from 'fs/promises'

7 import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'

8 import { dirname, extname, join, resolve } from 'path'

9 -import { dirStream, isDirectory, isWindowsDrive, makeMatcher, PERM_KEYS, VfsNodeAdminSend } from './misc'

9 +import { dirStream, enforceFinal, isDirectory, isWindowsDrive, makeMatcher, PERM_KEYS, VfsNodeAdminSend } from './misc'

10 import {

11 IS_WINDOWS,

12 HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVER_ERROR, HTTP_CONFLICT, HTTP_NOT_ACCEPTABLE,

13 } from './const'

14 import { getDrives } from './util-os'

15 import { Stats } from 'fs'

16 +import { getBaseUrlOrDefault } from './listen'

18 // to manipulate the tree we need the original node

19 async function urlToNodeOriginal(uri: string) {

@@ -98,25 +99,32 @@ const apis: ApiHandlers = {

99 async add_vfs({ parent, source, name }) {

100 if (!source && !name)

101 return new ApiError(HTTP_BAD_REQUEST, 'name or source required')

101 - parent = parent ? await urlToNodeOriginal(parent) : vfs

102 - if (!parent)

102 + const parentNode = parent ? await urlToNodeOriginal(parent) : vfs

103 + if (!parentNode)

104 return new ApiError(HTTP_NOT_FOUND, 'parent not found')

104 - if (!await nodeIsDirectory(parent))

105 + if (!await nodeIsDirectory(parentNode))

106 return new ApiError(HTTP_NOT_ACCEPTABLE, 'parent not a folder')

107 if (isWindowsDrive(source))

108 source += '\\' // slash must be included, otherwise it will refer to the cwd of that drive

109 + const isDir = source && await isDirectory(source)

110 + if (source && isDir === undefined)

111 + return new ApiError(HTTP_NOT_FOUND, 'source not found')

112 const child = { source, name }

113 name = getNodeName(child) // could be not given as input

114 const ext = extname(name)

115 const noExt = ext ? name.slice(0, -ext.length) : name

116 let idx = 2

113 - while (parent.children?.find(isSameFilenameAs(name)))

117 + while (parentNode.children?.find(isSameFilenameAs(name)))

118 name = `${noExt} ${idx++}${ext}`

119 child.name = name

120 simplifyName(child)

117 - ;(parent.children ||= []).unshift(child)

121 + ;(parentNode.children ||= []).unshift(child)

122 await saveVfs()

119 - return { name }

123 + const link = getBaseUrlOrDefault()

124 + + (parent ? enforceFinal('/', parent) : '/')

125 + + encodeURIComponent(getNodeName(child))

126 + + (isDir ? '/' : '')

127 + return { name, link }

128 },

129

130 async del_vfs({ uris }) {

src/cross.ts

+2 -2

@@ -112,8 +112,8 @@ export function objSameKeys<S extends object,VR=any>(src: S, newValue:(value:Tru

112 return Object.fromEntries(Object.entries(src).map(([k,v]) => [k, newValue(v,k as keyof S)])) as { [K in keyof S]:VR }

113 }

114

115 -export function enforceFinal(sub:string, s:string) {

116 - return !s || s.endsWith(sub) ? s : s+sub

115 +export function enforceFinal(sub:string, s:string, evenEmpty=false) {

116 + return !evenEmpty && !s || s.endsWith(sub) ? s : s+sub

117 }

118

119 export function truthy<T>(value: T): value is Truthy<T> {

src/index.ts

+2 -1

@@ -19,7 +19,8 @@ import { ok } from 'assert'

19 import _ from 'lodash'

20 import { randomId } from './misc'

21 import session from 'koa-session'

22 -import { acmeMiddleware, selfCheckMiddleware } from './api.net'

22 +import { selfCheckMiddleware } from './selfCheck'

23 +import { acmeMiddleware } from './acme'

25 ok(_.intersection(Object.keys(frontEndApis), Object.keys(adminApis)).length === 0) // they share same endpoints, don't clash

src/listen.ts

+20 -4

@@ -14,9 +14,9 @@ import findProcess from 'find-process'

14 import { anyAccountCanLoginAdmin } from './adminApis'

15 import _ from 'lodash'

16 import { X509Certificate } from 'crypto'

17 -import { externalIp } from './api.net'

17 import events from './events'

18 import { isIPv6 } from 'net'

19 +import { defaultBaseUrl } from './nat'

21 interface ServerExtra { name: string, error?: string, busy?: Promise<string> }

22 let httpSrv: undefined | http.Server & ServerExtra

@@ -24,6 +24,12 @@ let httpsSrv: undefined | http.Server & ServerExtra

25 const openBrowserAtStart = defineConfig('open_browser_at_start', !DEV)

27 +export const baseUrl = defineConfig('base_url', '')

28 +

29 +export function getBaseUrlOrDefault() {

30 + return baseUrl.get() || defaultBaseUrl.get()

31 +}

32 +

33 export function getHttpsWorkingPort() {

34 return httpsSrv?.listening && (httpsSrv.address() as any)?.port

35 }

@@ -73,6 +79,8 @@ export function getCertObject() {

80 const considerHttps = debounceAsync(async () => {

81 stopServer(httpsSrv).then()

82 + defaultBaseUrl.proto = 'http'

83 + defaultBaseUrl.port = getCurrentPort(httpSrv) ?? 0

84 let port = httpsPortCfg.get()

85 try {

86 while (!app)

@@ -111,6 +119,8 @@ const considerHttps = debounceAsync(async () => {

119 httpsSrv.on('connection', newConnection)

120 printUrls(httpsSrv.name)

121 events.emit('https ready')

122 + defaultBaseUrl.proto = 'https'

123 + defaultBaseUrl.port = getCurrentPort(httpsSrv) ?? 0

124 })

125

126

@@ -218,6 +228,10 @@ export function stopServer(srv?: http.Server) {

228 })

229 }

230

231 +function getCurrentPort(srv: typeof httpSrv) {

232 + return (srv?.address() as any)?.port as number | undefined

233 +}

234 +

235 export async function getServerStatus() {

236 return {

237 http: await serverStatus(httpSrv, portCfg.get()),

@@ -230,7 +244,7 @@ export async function getServerStatus() {

244 return {

245 ..._.pick(srv, ['listening', 'error']),

246 busy,

233 - port: (srv?.address() as any)?.port as number || configuredPort,

247 + port: getCurrentPort(srv) || configuredPort,

248 configuredPort,

249 srv,

250 }

@@ -243,11 +257,13 @@ export async function getIps(external=true) {

257 nets && !ignore.test(name)

258 && v4first(onlyTruthy(nets.map(net => !net.internal && net.address)))[0] // for each interface we consider only 1 address

259 )).flat()

246 - const e = external && externalIp

260 + const e = external && defaultBaseUrl.externalIp

261 if (e && !ips.includes(e))

262 ips.unshift(e)

249 - return v4first(ips)

263 + const ret = v4first(ips)

264 .filter((x,i,a) => a.length > 1 || !x.startsWith('169.254')) // 169.254 = dhcp failure on the interface, but keep it if it's our only one

265 + defaultBaseUrl.localIp = ret[0] || ''

266 + return ret

267

268 function v4first(a: string[]) {

269 return _.sortBy(a, isIPv6) // works because `false` comes first

src/middlewares.ts

+1 -3

@@ -34,7 +34,7 @@ import formidable from 'formidable'

34 import { uploadWriter } from './upload'

35 import { allowAdmin, favicon } from './adminApis'

36 import { constants } from 'zlib'

37 -import { getHttpsWorkingPort } from './listen'

37 +import { baseUrl, getHttpsWorkingPort } from './listen'

38 import { defineConfig } from './config'

39 import { getLangData } from './lang'

@@ -172,8 +172,6 @@ const errorMessages = {

172 [HTTP_FORBIDDEN]: "Forbidden",

173 }

174

175 -export const baseUrl = defineConfig('base_url', '')

176 -

175 async function sendFolderList(node: VfsNode, ctx: Koa.Context) {

176 let { depth=0, folders, prepend } = ctx.query

177 ctx.type = 'text'

src/nat.ts new

+97

@@ -0,0 +1,97 @@

1 +import { proxy } from 'valtio'

2 +import { Client } from 'nat-upnp-ts'

3 +import { debounceAsync } from './debounceAsync'

4 +import { GetNat, haveTimeout, HOUR, MINUTE, promiseBestEffort, repeat, wantArray } from './cross'

5 +import { getProjectInfo } from './github'

6 +import _ from 'lodash'

7 +import { httpString } from './util-http'

8 +import { Resolver } from 'dns/promises'

9 +import { isIP } from 'net'

10 +import { baseUrl, getIps, getServerStatus } from './listen'

11 +import { exec } from 'child_process'

12 +import { IS_MAC, IS_WINDOWS } from './const'

13 +

14 +export const defaultBaseUrl = proxy({

15 + proto: 'http',

16 + publicIps: [] as string[],

17 + externalIp: '',

18 + localIp: '',

19 + port: 0,

20 + get() {

21 + const defPort = this.proto === 'https' ? 443 : 80

22 + return `${this.proto}://${ this.publicIps[0] || this.externalIp || this.localIp}${!this.port || this.port === defPort ? '' : ':' + this.port}`

23 + }

24 +})

25 +

26 +export function getBaseUrlOrDefault() {

27 + return baseUrl.get() || defaultBaseUrl.get()

28 +}

29 +

30 +export const upnpClient = new Client({ timeout: 4_000 })

31 +const originalMethod = upnpClient.getGateway

32 +// other client methods call getGateway too, so this will ensure they reuse this same result

33 +upnpClient.getGateway = debounceAsync(() => originalMethod.apply(upnpClient), 0, { retain: HOUR, retainFailure: 30_000 })

34 +upnpClient.getGateway().catch(() => {})

35 +

36 +// poll external ip

37 +repeat(10 * MINUTE, () => upnpClient.getPublicIp().then(v => {

38 + if (v === defaultBaseUrl.externalIp) return

39 + getPublicIps.clearRetain()

40 + return defaultBaseUrl.externalIp = v

41 +}))

42 +

43 +const getPublicIps = debounceAsync(async () => {

44 + const res = await getProjectInfo()

45 + const groupedByVersion = Object.values(_.groupBy(res.publicIpServices, x => x.v ?? 4))

46 + const ips = await promiseBestEffort(groupedByVersion.map(singleVersion =>

47 + Promise.any(singleVersion.map(async (svc: any) => {

48 + if (typeof svc === 'string')

49 + svc = { type: 'http', url: svc }

50 + console.debug("trying ip service", svc.url || svc.name)

51 + if (svc.type === 'http')

52 + return httpString(svc.url)

53 + if (svc.type !== 'dns') throw "unsupported"

54 + const resolver = new Resolver({ timeout: 2_000 })

55 + resolver.setServers(svc.ips)

56 + return resolver.resolve(svc.name, svc.dnsRecord)

57 + }).map(async ret => {

58 + const validIps = wantArray(await ret).map(x => x.trim()).filter(isIP)

59 + if (!validIps.length) throw "no good"

60 + return validIps

61 + }) )))

62 + return _.uniq(ips.flat())

63 +}, 0, { retain: 10 * MINUTE })

64 +

65 +export const getNatInfo = debounceAsync(async () => {

66 + const gettingIps = getPublicIps() // don't wait, do it in parallel

67 + const res = await upnpClient.getGateway().catch(() => null)

68 + const status = await getServerStatus()

69 + const mappings = res && await haveTimeout(5_000, upnpClient.getMappings()).catch(() => null)

70 + console.debug('mappings found', mappings?.map(x => x.description))

71 + const gatewayIp = res ? new URL(res.gateway.description).hostname : await findGateway().catch(() => undefined)

72 + const localIp = res?.address || (await getIps())[0]

73 + const internalPort = status?.https?.listening && status.https.port || status?.http?.listening && status.http.port || undefined

74 + const mapped = _.find(mappings, x => x.private.host === localIp && x.private.port === internalPort)

75 + const externalPort = mapped?.public.port

76 + defaultBaseUrl.port = externalPort || internalPort || 0

77 + return {

78 + upnp: Boolean(res),

79 + localIp,

80 + gatewayIp,

81 + publicIps: defaultBaseUrl.publicIps = await gettingIps,

82 + externalIp: defaultBaseUrl.externalIp,

83 + mapped,

84 + internalPort,

85 + externalPort,

86 + proto: status?.https?.listening ? 'https' : status?.http?.listening ? 'http' : '',

87 + } satisfies GetNat

88 +})

89 +

90 +function findGateway(): Promise<string | undefined> {

91 + return new Promise((resolve, reject) =>

92 + exec(IS_WINDOWS || IS_MAC ? 'netstat -rn' : 'route -n', (err, out) => {

93 + if (err) return reject(err)

94 + const re = IS_WINDOWS ? /(?:0\.0\.0\.0 +){2}([\d.]+)/ : IS_MAC ? /default +([\d.]+)/ : /^0\.0\.0\.0 +([\d.]+)/

95 + resolve(re.exec(out)?.[1])

96 + }) )

97 +}

src/selfCheck.ts new

+64

@@ -0,0 +1,64 @@

1 +import { SPECIAL_URI } from './cross-const'

2 +import { Middleware } from 'koa'

3 +import { getProjectInfo } from './github'

4 +import { isIP, isIPv6 } from 'net'

5 +import _ from 'lodash'

6 +import { haveTimeout } from './cross'

7 +import { httpString } from './util-http'

8 +

9 +let selfCheckMiddlewareEnabled = false

10 +const CHECK_URL = SPECIAL_URI + 'self-check'

11 +export const selfCheckMiddleware: Middleware = (ctx, next) => { // koa format

12 + if (!selfCheckMiddlewareEnabled || !ctx.url.startsWith(CHECK_URL))

13 + return next()

14 + ctx.body = 'HFS'

15 +}

16 +

17 +export async function selfCheck(url: string) {

18 + interface PortScannerService {

19 + type?: string

20 + url: string

21 + headers: {[k: string]: string}

22 + method: string

23 + body?: string

24 + regexpFailure: string

25 + regexpSuccess: string

26 + }

27 + const prjInfo = await getProjectInfo()

28 + console.log(`checking server ${url}`)

29 + selfCheckMiddlewareEnabled = true

30 + try {

31 + const parsed = new URL(url)

32 + const family = !isIP(parsed.hostname) ? undefined : isIPv6(parsed.hostname) ? 6 : 4

33 + for (const services of _.chunk(_.shuffle<PortScannerService>(prjInfo.selfCheckServices), 2)) {

34 + try {

35 + return await Promise.any(services.map(async (svc) => {

36 + if (!svc.url || svc.type) throw 'unsupported ' + svc.type // only default type supported for now

37 + let { url: serviceUrl, body, regexpSuccess, regexpFailure, ...rest } = svc

38 + const service = new URL(serviceUrl).hostname

39 + console.log('trying external service', service)

40 + body = applySymbols(body)

41 + serviceUrl = applySymbols(serviceUrl)!

42 + const res = await haveTimeout(10_000, httpString(serviceUrl, { family, ...rest, body }))

43 + const success = new RegExp(regexpSuccess).test(res)

44 + const failure = new RegExp(regexpFailure).test(res)

45 + if (success === failure) throw 'inconsistent: ' + service + ': ' + res // this result cannot be trusted

46 + console.debug(service, 'responded', success)

47 + return { success, service, url }

48 + }))

49 + }

50 + catch (e: any) {

51 + console.debug(e?.errors?.map(String) || e?.cause || String(e))

52 + }

53 + }

54 +

55 + function applySymbols(s?: string) {

56 + return s?.replace('$IP', parsed.hostname)

57 + .replace('$PORT', parsed.port || (parsed.protocol === 'https:' ? '443' : '80'))

58 + .replace('$URL', url.replace(/\/$/, '') + CHECK_URL)

59 + }

60 + }

61 + finally {

62 + selfCheckMiddlewareEnabled = false

63 + }

64 +}

src/util-files.ts

+1 -1

@@ -16,7 +16,7 @@ import fsx from 'fs-x-attributes'

17 export async function isDirectory(path: string) {

18 try { return (await fs.stat(path)).isDirectory() }

19 - catch { return false }

19 + catch {}

20 }

22 export async function readFileBusy(path: string): Promise<string> {