more efficient public ip search
Massimo Melina committed
Oct 1, 2023 at 00:34 UTC
56bfe8a549dfae578eae90f01239a7452929e67d
6 files changed
+86
-56
admin/src/InternetPage.ts
+4
-5
@@ -2,7 +2,7 @@ import { createElement as h, useEffect, useState } from 'react'
2
import { Alert, Box, Button, Card, CardContent, CircularProgress, Divider, LinearProgress, Link } from '@mui/material'
3
import { CardMembership, HomeWorkTwoTone, Lock, PublicTwoTone, RouterTwoTone, Send } from '@mui/icons-material'
4
import { apiCall, useApiEx } from './api'
5
-import { closeDialog, DAY, formatTimestamp, GetNat, onlyTruthy, wantArray, with_ } from '@hfs/shared'
5
+import { closeDialog, DAY, formatTimestamp, GetNat, wantArray, with_ } from '@hfs/shared'
6
import { Flex, LinkBtn, manipulateConfig, isIP, Btn } from './misc'
7
import { alertDialog, confirmDialog, promptDialog, toast } from './dialog'
8
import { BoolField, Form, NumberField } from '@hfs/mui-grid-form'
@@ -24,9 +24,8 @@ export default function InternetPage() {
24
h && s ? 'error' : h || s ? 'warning' : 'success')
25
const { data: nat, reload: reloadNat, error, loading, element } = useApiEx<GetNat>('get_nat')
26
const port = nat?.internalPort
27
- const publicIps = onlyTruthy([nat?.publicIp4, nat?.publicIp6])
27
const wrongMap = nat?.mapped && nat.mapped.private.port !== port && nat.mapped.private.port
29
- const doubleNat = nat?.externalIp && !publicIps.includes(nat.externalIp)
28
+ const doubleNat = nat?.externalIp && nat?.publicIps && !nat.publicIps.includes(nat.externalIp)
29
useEffect(() => {
30
if (!verifyAgain || !nat || loading) return
31
verify().then()
@@ -137,13 +136,13 @@ export default function InternetPage() {
136
"port ", wrongMap ? 'is wrong' : nat?.externalPort || "unknown"),
137
}),
138
h(Sep),
140
- h(Device, { name: "Internet", icon: PublicTwoTone, ip: publicIps,
139
+ h(Device, { name: "Internet", icon: PublicTwoTone, ip: nat?.publicIps,
140
color: checkResult ? 'success' : checkResult === false ? 'error' : doubleNat ? 'warning' : undefined,
141
below: checking ? h(LinearProgress, { sx: { height: '1em' } }) : h(Box, { fontSize: 'smaller' },
142
doubleNat && h(LinkBtn, { display: 'block', onClick: () => alertDialog(MSG_ISP, 'warning') }, "Double NAT"),
143
checkResult ? "Working!" : checkResult === false ? "Failed!" : '',
144
' ',
146
- publicIps.length && nat.internalPort && h(LinkBtn, { onClick: verify }, "Verify")
145
+ nat?.publicIps.length && nat.internalPort && h(LinkBtn, { onClick: verify }, "Verify")
146
)
147
}),
148
)
central.json
+7
-3
@@ -29,9 +29,13 @@
29
],
30
"publicIpServices_049": [
31
{ "url": "http://ipv6.icanhazip.com", "v": 6 },
32
+ { "url": "http://checkip.amazonaws.com", "prefers": 4 },
33
{ "url": "http://ifconfig.io/ip", "v": 6 },
33
- { "url": "http://checkip.amazonaws.com" },
34
- { "url": "http://ipinfo.io/ip" },
35
- { "url": "http://ipv4.icanhazip.com", "v": 4 }
34
+ { "url": "http://ipv4.icanhazip.com", "v": 4 },
35
+ { "url": "http://ip4only.me/api/", "v": 4, "after": "IPv4," },
36
+ { "url": "http://ip6only.me/api/", "v": 6, "after": "IPv6," },
37
+ { "url": "https://www.showmyip.com", "v": 46, "after": "IPv$V</td><td><b>" },
38
+ { "url": "https://dnschecker.org/whats-my-ip-address.php", "v": 46, "after": "ipv$V_html.+>" },
39
+ { "url": "http://ipinfo.io/ip", "prefers": 4 }
40
]
41
}
src/api.net.ts
+47
-36
@@ -11,7 +11,8 @@ import { cert, getCertObject, getIps, getServerStatus, privateKey } from './list
11
import { getProjectInfo } from './github'
12
import { httpString } from './util-http'
13
import { exec } from 'child_process'
14
-import { apiAssertTypes, DAY, debounceAsync, haveTimeout, HOUR, MINUTE, objSameKeys, onlyTruthy, repeat, Dict, isIp6, GetNat } from './misc'
14
+import { apiAssertTypes, DAY, debounceAsync, haveTimeout, HOUR, MINUTE, objSameKeys, onlyTruthy, repeat, Dict, GetNat,
15
+ parallelJobs } from './misc'
16
import acme from 'acme-client'
17
import fs from 'fs/promises'
18
import { createServer, RequestListener } from 'http'
@@ -19,6 +20,7 @@ import { Middleware } from 'koa'
20
import { lookup, Resolver } from 'dns/promises'
21
import { defineConfig } from './config'
22
import events from './events'
23
+import { isIP, isIPv6 } from 'net'
24
25
const upnpClient = new Client({ timeout: 4_000 })
26
const originalMethod = upnpClient.getGateway
@@ -30,8 +32,7 @@ export let externalIp = '' // poll external ip
32
repeat(10 * MINUTE, () => upnpClient.getPublicIp().then(v => externalIp = v))
33
34
const getNatInfo = debounceAsync(async () => {
33
- const gettingIp4 = getPublicIp(4) // don't wait, do it in parallel
34
- const gettingIp6 = getPublicIp(6)
35
+ const gettingIps = getPublicIps() // don't wait, do it in parallel
36
const res = await upnpClient.getGateway().catch(() => null)
37
const status = await getServerStatus()
38
const mappings = res && await haveTimeout(5_000, upnpClient.getMappings()).catch(() => null)
@@ -40,40 +41,50 @@ const getNatInfo = debounceAsync(async () => {
41
const localIp = res?.address || (await getIps())[0]
42
const internalPort = status?.https?.listening && status.https.port || status?.http?.listening && status.http.port || undefined
43
const mapped = _.find(mappings, x => x.private.host === localIp && x.private.port === internalPort)
43
- const ret: GetNat = { // i'd like to use 'satisfies' but my ide is not supporting it
44
+ return {
45
upnp: Boolean(res),
46
localIp,
47
gatewayIp,
47
- publicIp4: await gettingIp4,
48
- publicIp6: await gettingIp6,
48
+ publicIps: Object.values(await gettingIps),
49
externalIp,
50
mapped,
51
internalPort,
52
externalPort: mapped?.public.port,
53
- }
54
- return ret
53
+ } satisfies GetNat
54
})
55
57
-async function getPublicIp(version?: number) {
56
+async function getPublicIps() {
57
+ const ret: any = {}
58
+ const attemptsMade: any = { 4: 0, 6: 0 }
59
const prjInfo = await getProjectInfo()
59
- const services = prjInfo.publicIpServices_049?.filter((x: any) => !x.v || version === x.v)
60
- for (const chunk of _.chunk(_.shuffle(services), 2)) // small parallelization
61
- try {
62
- return await Promise.any(chunk.map(service => httpString(service.url).then(res => {
63
- if (service.after) {
64
- const i = res.match(new RegExp(service.after, 'i'))?.index
65
- if (i == null) throw Error("cannot match: " + res)
66
- res = res.slice(i)
67
- }
68
- const ip = res.trim()
69
- if (!/[.:0-9a-fA-F]/.test(ip))
70
- throw Error("bad result: " + ip)
71
- if (version && version === 6 !== isIp6(ip)) // enforce result format
72
- throw Error("bad version: " + ip)
73
- return ip
74
- })))
60
+ // small parallelization. The logic is a bit complex because considers many things
61
+ await parallelJobs(2, prjInfo.publicIpServices_049.map((s: any) => async () => {
62
+ if (ret[4] && ret[6]) return // no more to do
63
+ if (s.v && s.v !== 46 && ret[s.v] || s.prefers && ret[s.prefers]) return // no good
64
+ const expected = String(s.v||'').split('')
65
+ const checkAttempt = expected.length === 1 ? expected[0]! : ret[4] ? 6 : ret[6] ? 4 : 0
66
+ if (attemptsMade[checkAttempt] > 1) return // give up on this (you may not have this version, after all)
67
+ console.debug("trying service", s.url)
68
+ for (const k of expected) attemptsMade[k]++
69
+ const res = await httpString(s.url).catch(() => null)
70
+ if (!res) return
71
+ for (const afterV of s.v && s.after && expected.length ? expected : ['']) {
72
+ if (ret[afterV]) continue
73
+ let workOn = res
74
+ if (s.after) {
75
+ const found = workOn.match(new RegExp(s.after.replace('$V', afterV), 'i'))
76
+ workOn = !found ? '' : workOn.slice(found.index! + found[0].length)
77
+ }
78
+ const ip = workOn.match(/[.:0-9a-fA-F]+/)?.[0]?.trim()
79
+ if (ip && isIP(ip)) {
80
+ ret[isIPv6(ip) ? 6 : 4] = ip
81
+ continue
82
+ }
83
+ if (s.v === afterV)
84
+ console.debug("bad reply", s.url)
85
}
76
- catch (e: any) { console.debug(String(e)) }
86
+ }))
87
+ return ret
88
}
89
90
function findGateway(): Promise<string | undefined> {
@@ -114,12 +125,12 @@ async function checkDomain(domain: string) {
125
const domainIps = _.uniq(onlyTruthy(settled.map(x => x.status === 'fulfilled' && x.value)).flat())
126
if (!domainIps.length)
127
throw new ApiError(HTTP_FAILED_DEPENDENCY, "domain not working")
117
- const { publicIp4, publicIp6 } = await getNatInfo() // do this before stopping the server
128
+ const { publicIps } = await getNatInfo() // do this before stopping the server
129
for (const v6 of [false, true]) {
119
- const domainIpsThisVersion = domainIps.filter(x => isIp6(x) === v6)
120
- const ipThisVersion = v6 ? publicIp6 : publicIp4
121
- if (domainIpsThisVersion.length && ipThisVersion && !domainIpsThisVersion.includes(ipThisVersion))
122
- throw new ApiError(HTTP_FAILED_DEPENDENCY, `please configure your domain to point to ${publicIp4} or (currently on ${domainIps[0]}) --- a change can take hours to be effective`)
130
+ const domainIpsThisVersion = domainIps.filter(x => isIPv6(x) === v6)
131
+ const ipsThisVersion = publicIps.filter(x => isIPv6(x) === v6)
132
+ if (domainIpsThisVersion.length && ipsThisVersion.length && !_.intersection(domainIpsThisVersion, ipsThisVersion).length)
133
+ throw new ApiError(HTTP_FAILED_DEPENDENCY, `configure your domain to point to ${ipsThisVersion} (currently on ${domainIpsThisVersion[0]}) – a change can take hours to be effective`)
134
}
135
}
136
@@ -231,15 +242,15 @@ const apis: ApiHandlers = {
242
},
243
244
async check_server({ port }) {
234
- const { publicIp4, publicIp6, internalPort, externalPort } = await getNatInfo()
235
- const publicIp = publicIp4 || publicIp6
236
- if (!publicIp)
245
+ const { publicIps, internalPort, externalPort } = await getNatInfo()
246
+ const ip = publicIps[0]
247
+ if (!ip)
248
return new ApiError(HTTP_SERVICE_UNAVAILABLE, 'cannot detect public ip')
249
if (!internalPort)
250
return new ApiError(HTTP_FAILED_DEPENDENCY, 'no internal port')
251
port ||= externalPort || internalPort
241
- console.log(`checking server ${publicIp}:${port}`)
242
- return await checkPort(publicIp, port)
252
+ console.log(`checking server ${ip}:${port}`)
253
+ return await checkPort(ip, port)
254
|| new ApiError(HTTP_SERVICE_UNAVAILABLE)
255
},
256
src/cross.ts
+22
-8
@@ -1,6 +1,7 @@
1
// This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
2
// all content here is shared between client and server
3
import _ from 'lodash'
4
+import { isIPv6 } from 'net'
5
6
export const REPO_URL = 'https://github.com/rejetto/hfs/'
7
export const WIKI_URL = REPO_URL + 'wiki/'
@@ -28,8 +29,7 @@ export interface GetNat {
29
upnp: boolean,
30
localIp?: string
31
gatewayIp?: string
31
- publicIp4?: string
32
- publicIp6?: string
32
+ publicIps: string[]
33
externalIp: string,
34
mapped?: Mapping
35
internalPort?: number
@@ -286,15 +286,29 @@ export function ipLocalHost(ip: string) {
286
return ip === '::1' || ip.endsWith('127.0.0.1')
287
}
288
289
-export function isIp6(ip: string) {
290
- return ip.includes(':')
291
-}
292
-
289
export function ipForUrl(ip: string) {
294
- return isIp6(ip) ? '[' + ip + ']' : ip
290
+ return isIPv6(ip) ? '[' + ip + ']' : ip
291
}
292
293
export function escapeHTML(text: string) {
294
return text.replace(/[\u0000-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u00FF]/g,
295
c => '&#' + ('000' + c.charCodeAt(0)).slice(-4) + ';')
300
-}
\ No newline at end of file
296
+}
297
+
298
+export async function parallelJobs<T>(parallelization: number, promiseReturners: (() => Promise<T>)[]) {
299
+ const now = promiseReturners.slice(0, parallelization).map(x => x())
300
+ let i = now.length
301
+ while (now.length) {
302
+ now.splice(await raceIndex(now), 1)
303
+ const next = promiseReturners[i++]?.()
304
+ if (next)
305
+ now.push(next)
306
+ }
307
+
308
+ async function raceIndex(promises: Promise<unknown>[]) {
309
+ let ret: number
310
+ promises.forEach((p,i) => p.finally(() => ret ??= i))
311
+ await Promise.race(promises)
312
+ return ret!
313
+ }
314
+}
src/listen.ts
+3
-2
@@ -8,7 +8,7 @@ import { watchLoad } from './watchLoad'
8
import { networkInterfaces } from 'os';
9
import { newConnection } from './connections'
10
import open from 'open'
11
-import { debounceAsync, ipForUrl, isIp6, objSameKeys, onlyTruthy, wait, waitFor } from './misc'
11
+import { debounceAsync, ipForUrl, objSameKeys, onlyTruthy, wait, waitFor } from './misc'
12
import { ADMIN_URI, argv, DEV, IS_WINDOWS } from './const'
13
import findProcess from 'find-process'
14
import { anyAccountCanLoginAdmin } from './adminApis'
@@ -16,6 +16,7 @@ import _ from 'lodash'
16
import { X509Certificate } from 'crypto'
17
import { externalIp } from './api.net'
18
import events from './events'
19
+import { isIPv6 } from 'net'
20
21
interface ServerExtra { name: string, error?: string, busy?: Promise<string> }
22
let httpSrv: undefined | http.Server & ServerExtra
@@ -249,7 +250,7 @@ export async function getIps(external=true) {
250
.filter((x,i,a) => a.length > 1 || !x.startsWith('169.254')) // 169.254 = dhcp failure on the interface, but keep it if it's our only one
251
252
function v4first(a: string[]) {
252
- return _.sortBy(a, isIp6) // works because `false` comes first
253
+ return _.sortBy(a, isIPv6) // works because `false` comes first
254
}
255
}
256
src/misc.ts
+3
-2
@@ -15,7 +15,8 @@ import { matcher } from 'micromatch'
15
import { SocketAddress, BlockList } from 'node:net'
16
import { ApiError } from './apiMiddleware'
17
import { HTTP_BAD_REQUEST } from './const'
18
-import { ipLocalHost, isIp6 } from './cross'
18
+import { ipLocalHost } from './cross'
19
+import { isIPv6 } from 'net'
20
21
type ProcessExitHandler = (signal:string) => any
22
const cbs = new Set<ProcessExitHandler>()
@@ -89,7 +90,7 @@ export function makeNetMatcher(mask: string, emptyMaskReturns=false) {
90
}
91
92
function parseAddress(s: string) {
92
- return new SocketAddress({ address: s, family: isIp6(s) ? 'ipv6' : 'ipv4' })
93
+ return new SocketAddress({ address: s, family: isIPv6(s) ? 'ipv6' : 'ipv4' })
94
}
95
96
export function makeMatcher(mask: string, emptyMaskReturns=false) {