admin/internet: better verification, using domain and checking http response, not just open port
Massimo Melina committed
Oct 8, 2023 at 00:46 UTC
5edf2c0999be06a4dc7e81e3921d7f1d524e7eb8
5 files changed
+105
-39
admin/src/InternetPage.ts
+15
-3
@@ -157,11 +157,22 @@ export default function InternetPage() {
157
if (!verifyAgain && !await confirmDialog("This test will check if your server is working properly on the Internet")) return
158
setChecking(true)
159
try {
160
- const res = await apiCall('check_server', {})
160
+ const url = status.data?.baseUrl
161
+ const urlResult = url && await apiCall('self_check', { url }).catch(() =>
162
+ alertDialog(md(`Sorry, we couldn't verify your configured address ${url} 😰\nstill, we are going to test your IP address 🤞`), 'warning'))
163
+ if (urlResult?.success) {
164
+ setCheckResult(true)
165
+ return alertDialog(h(Box, {}, "Your server is responding correctly over the Internet:",
166
+ h('ul', {}, h('li', {}, urlResult.url))), 'success')
167
+ }
168
+ if (urlResult?.success === false)
169
+ await alertDialog(md(`Your configured address ${url} doesn't seem to work 😰\nstill, we are going to test your IP address 🤞`), 'warning')
170
+ const res = await apiCall('self_check', {})
171
if (res.some((x: any) => x.success)) {
172
setCheckResult(true)
163
- const specify = res.every((x: any) => x.success) ? '' : ` with address ${res.map((x: any) => x.ip).join(' + ')}`
164
- return toast("Your server is responding correctly over the Internet" + specify, 'success')
173
+ const mild = urlResult.success === false && md(`Your server is responding over the Internet 👍\nbut not with configured address ${url} 👎\njust on your IP:`)
174
+ return alertDialog(h(Box, {}, mild || "Your server is responding correctly over the Internet:",
175
+ h('ul', {}, ...res.map((x: any) => h('li', {}, x.url)))), mild ? 'warning' : 'success')
176
}
177
setCheckResult(false)
178
if (wrongMap)
@@ -232,6 +243,7 @@ export default function InternetPage() {
243
await apiCall('map_port', { external })
244
reloadNat()
245
if (msg) toast(msg, 'success')
246
+ setCheckResult(undefined) // things have changed, invalidate check result
247
}
248
catch(e) {
249
if (errMsg) {
central.json
+12
@@ -23,6 +23,18 @@
23
"regexpSuccess": "reachable\": true"
24
}
25
],
26
+ "selfCheckServices": [
27
+ {
28
+ "url": "https://api.val.town/v6/run/rejetto.checkHfs?args=[%22$URL%22]",
29
+ "regexpFailure": "false",
30
+ "regexpSuccess": "true"
31
+ },
32
+ {
33
+ "url": "http://hfstest.rejetto.com/v3?url=$URL",
34
+ "regexpFailure": "\"error\"",
35
+ "regexpSuccess": "\"good\""
36
+ }
37
+ ],
38
"publicIpServices": [
39
"http://ipv4.icanhazip.com",
40
{ "v": 6, "type": "http", "url": "http://ipv6.icanhazip.com" },
src/api.net.ts
+74
-34
@@ -2,9 +2,8 @@
2
3
import { ApiError, ApiHandlers } from './apiMiddleware'
4
import { Client } from 'nat-upnp-ts'
5
-import {
6
- HTTP_BAD_REQUEST, HTTP_FAILED_DEPENDENCY, HTTP_OK, HTTP_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, HTTP_PRECONDITION_FAILED,
7
- IS_MAC, IS_WINDOWS
5
+import { HTTP_BAD_REQUEST, HTTP_FAILED_DEPENDENCY, HTTP_OK, HTTP_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE,
6
+ HTTP_PRECONDITION_FAILED, IS_MAC, IS_WINDOWS, SPECIAL_URI
7
} from './const'
8
import _ from 'lodash'
9
import { cert, getCertObject, getIps, getServerStatus, privateKey } from './listen'
@@ -31,7 +30,11 @@ upnpClient.getGateway = debounceAsync(() => originalMethod.apply(upnpClient), 0,
30
upnpClient.getGateway().catch(() => {})
31
32
export let externalIp = '' // poll external ip
34
-repeat(10 * MINUTE, () => upnpClient.getPublicIp().then(v => externalIp = v))
33
+repeat(10 * MINUTE, () => upnpClient.getPublicIp().then(v => {
34
+ if (v !== externalIp)
35
+ getPublicIps.clearRetain()
36
+ return externalIp = v
37
+}))
38
39
const getNatInfo = debounceAsync(async () => {
40
const gettingIps = getPublicIps() // don't wait, do it in parallel
@@ -52,10 +55,11 @@ const getNatInfo = debounceAsync(async () => {
55
mapped,
56
internalPort,
57
externalPort: mapped?.public.port,
58
+ proto: status?.https?.listening ? 'https' : status?.http?.listening ? 'http' : undefined,
59
} satisfies GetNat
60
})
61
58
-async function getPublicIps() {
62
+const getPublicIps = debounceAsync(async () => {
63
const res = await getProjectInfo()
64
const groupedByVersion = Object.values(_.groupBy(res.publicIpServices, x => x.v ?? 4))
65
const ips = await promiseBestEffort(groupedByVersion.map(singleVersion =>
@@ -75,7 +79,7 @@ async function getPublicIps() {
79
return validIps
80
}) )))
81
return _.uniq(ips.flat())
78
-}
82
+}, 0, { retain: 10 * MINUTE })
83
84
function findGateway(): Promise<string | undefined> {
85
return new Promise((resolve, reject) =>
@@ -102,6 +106,14 @@ export const acmeMiddleware: Middleware = (ctx, next) => { // koa format
106
return next()
107
}
108
109
+let selfCheckMiddlewareEnabled = false
110
+const CHECK_URL = SPECIAL_URI + 'self-check'
111
+export const selfCheckMiddleware: Middleware = (ctx, next) => { // koa format
112
+ if (!selfCheckMiddlewareEnabled || !ctx.url.startsWith(CHECK_URL))
113
+ return next()
114
+ ctx.body = 'HFS'
115
+}
116
+
117
async function checkDomain(domain: string) {
118
const resolver = new Resolver()
119
const prjInfo = await getProjectInfo()
@@ -138,11 +150,13 @@ async function generateSSLCert(domain: string, email?: string) {
150
acmeMiddlewareEnabled = true
151
console.debug('acme challenge server ready')
152
try {
141
- let check = await checkPort(domain, 80) // some check services may not consider the domain, but we already verified that
153
+ const checkUrl = `http://${domain}`
154
+ let check = await selfCheck(checkUrl) // some check services may not consider the domain, but we already verified that
155
if (check && !check.success && upnp && externalPort !== 80) { // consider a short-lived mapping
156
+ console.debug("setting temporary port forward")
157
// @ts-ignore
158
await upnpClient.createMapping({ private: 80, public: { host: '', port: 80 }, description: 'hfs temporary', ttl: 30 }).catch(() => {})
145
- check = await checkPort(domain, 80) // repeat test
159
+ check = await selfCheck(checkUrl) // repeat test
160
}
161
//if (!check) throw new ApiError(HTTP_FAILED_DEPENDENCY, "couldn't test port 80")
162
if (!check?.success)
@@ -175,39 +189,59 @@ async function generateSSLCert(domain: string, email?: string) {
189
}
190
}
191
178
-async function checkPort(ip: string, port: number) {
192
+async function checkService(url: string, serviceKey: string) {
193
interface PortScannerService {
194
+ type?: string
195
url: string
196
headers: {[k: string]: string}
197
method: string
183
- selector: string
198
body?: string
199
regexpFailure: string
200
regexpSuccess: string
201
}
202
const prjInfo = await getProjectInfo()
189
- console.log(`checking server ${ip}:${port}`)
190
- for (const services of _.chunk(_.shuffle<PortScannerService>(prjInfo.checkServerServices), 2)) {
191
- try {
192
- return await Promise.any(services.map(async ({ url, body, selector, regexpSuccess, regexpFailure, ...rest }) => {
193
- const service = new URL(url).hostname
194
- console.log('trying service', service)
195
- const res = await httpString(applySymbols(url)!, { family: isIPv6(ip) ? 6 : 4, body: applySymbols(body), ...rest })
196
- const success = new RegExp(regexpSuccess).test(res)
197
- const failure = new RegExp(regexpFailure).test(res)
198
- if (success === failure) throw console.debug('inconsistent:' + service) // this result cannot be trusted
199
- console.debug(service, 'responded', success)
200
- return { success, service, ip, port }
201
- }))
203
+ console.log(`checking server ${url}`)
204
+ selfCheckMiddlewareEnabled = true
205
+ try {
206
+ const parsed = new URL(url)
207
+ const family = !isIP(parsed.hostname) ? undefined : isIPv6(parsed.hostname) ? 6 : 4
208
+ for (const services of _.chunk(_.shuffle<PortScannerService>(prjInfo[serviceKey]), 2)) {
209
+ try {
210
+ return await Promise.any(services.map(async (svc) => {
211
+ if (!svc.url || svc.type) throw 'unsupported ' + svc.type // only default type supported for now
212
+ let { url: serviceUrl, body, regexpSuccess, regexpFailure, ...rest } = svc
213
+ const service = new URL(serviceUrl).hostname
214
+ console.log('trying external service', service)
215
+ body = applySymbols(body)
216
+ serviceUrl = applySymbols(serviceUrl)!
217
+ const res = await haveTimeout(10_000, httpString(serviceUrl, { family, ...rest, body }))
218
+ const success = new RegExp(regexpSuccess).test(res)
219
+ const failure = new RegExp(regexpFailure).test(res)
220
+ if (success === failure) throw 'inconsistent: ' + service + ': ' + res // this result cannot be trusted
221
+ console.debug(service, 'responded', success)
222
+ return { success, service, url }
223
+ }))
224
+ }
225
+ catch (e: any) {
226
+ console.debug(e?.errors?.map(String) || e?.cause || String(e))
227
+ }
228
}
203
- catch {}
204
- }
229
206
- function applySymbols(s?: string) {
207
- return s?.replace('$IP', ip).replace('$PORT', String(port))
230
+ function applySymbols(s?: string) {
231
+ return s?.replace('$IP', parsed.hostname)
232
+ .replace('$PORT', parsed.port || (parsed.protocol === 'https:' ? '443' : '80'))
233
+ .replace('$URL', url.replace(/\/$/, '') + CHECK_URL)
234
+ }
235
+ }
236
+ finally {
237
+ selfCheckMiddlewareEnabled = false
238
}
239
}
240
241
+function selfCheck(url: string) {
242
+ return checkService(url, 'selfCheckServices')
243
+}
244
+
245
const apis: ApiHandlers = {
246
get_nat: getNatInfo,
247
@@ -230,15 +264,21 @@ const apis: ApiHandlers = {
264
return {}
265
},
266
233
- async check_server({ port }) {
234
- const { publicIps, internalPort, externalPort } = await getNatInfo()
235
- if (!publicIps.length)
267
+ async self_check({ url }) {
268
+ if (url)
269
+ return await selfCheck(url)
270
+ || new ApiError(HTTP_SERVICE_UNAVAILABLE)
271
+ const nat = await getNatInfo()
272
+ if (!nat.publicIps.length)
273
return new ApiError(HTTP_FAILED_DEPENDENCY, 'cannot detect public ip')
237
- if (!internalPort)
274
+ if (!nat.internalPort)
275
return new ApiError(HTTP_FAILED_DEPENDENCY, 'no internal port')
239
- port ||= externalPort || internalPort
240
- const res = await promiseBestEffort(publicIps.map(ip => checkPort(ip, port)))
241
- return res.length ? res : new ApiError(HTTP_SERVICE_UNAVAILABLE)
276
+ const finalPort = nat.externalPort || nat.internalPort
277
+ const proto = nat.proto || (getCertObject() ? 'https' : 'http')
278
+ const defPort = proto === 'https' ? 443 : 80
279
+ const results = onlyTruthy(await promiseBestEffort(nat.publicIps.map(ip =>
280
+ selfCheck(`${proto}://${ip}${finalPort === defPort ? '' : ':' + finalPort}`) )))
281
+ return results.length ? results : new ApiError(HTTP_SERVICE_UNAVAILABLE)
282
},
283
284
async make_cert({domain, email}) {
src/cross.ts
+2
-1
@@ -35,6 +35,7 @@ export interface GetNat {
35
mapped?: Mapping
36
internalPort?: number
37
externalPort?: number
38
+ proto?: string
39
}
40
41
export interface VfsPerms {
@@ -104,7 +105,7 @@ export function wait<T=undefined>(ms: number, val?: T): Promise<T | undefined> {
105
}
106
107
export function haveTimeout<T>(ms: number, job: Promise<T>, error?: any) {
107
- return Promise.race([job, wait(ms).then(() => { throw error })])
108
+ return Promise.race([job, wait(ms).then(() => { throw error || Error('timeout') })])
109
}
110
111
export function objSameKeys<S extends object,VR=any>(src: S, newValue:(value:Truthy<S[keyof S]>, key:keyof S)=>VR) {
src/index.ts
+2
-1
@@ -19,7 +19,7 @@ import { ok } from 'assert'
19
import _ from 'lodash'
20
import { randomId } from './misc'
21
import session from 'koa-session'
22
-import { acmeMiddleware } from './api.net'
22
+import { acmeMiddleware, selfCheckMiddleware } from './api.net'
23
24
ok(_.intersection(Object.keys(frontEndApis), Object.keys(adminApis)).length === 0) // they share same endpoints, don't clash
25
@@ -27,6 +27,7 @@ process.title = 'HFS ' + VERSION
27
const keys = process.env.COOKIE_SIGN_KEYS?.split(',') || [randomId(30)]
28
export const app = new Koa({ keys })
29
app.use(someSecurity)
30
+ .use(selfCheckMiddleware)
31
.use(acmeMiddleware)
32
.use(session({ key: 'hfs_$id', signed: true, rolling: true, sameSite: 'lax' }, app))
33
.use(prepareState)