| 1 | // This file is part of HFS - Copyright 2021-2023, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt |
| 2 | |
| 3 | import { urlToHttpOptions } from 'node:url' |
| 4 | import https from 'node:https' |
| 5 | import http, { IncomingMessage } from 'node:http' |
| 6 | import { Readable } from 'node:stream' |
| 7 | import _ from 'lodash' |
| 8 | import { text as stream2string, buffer } from 'node:stream/consumers' |
| 9 | import * as tls from 'node:tls' |
| 10 | import { enforceStarting } from './cross' |
| 11 | export { stream2string } |
| 12 | |
| 13 | export async function httpString(url: string, options?: XRequestOptions): Promise<string> { |
| 14 | return await stream2string(await httpStream(url, options)) |
| 15 | } |
| 16 | |
| 17 | export async function httpWithBody(url: string, options?: XRequestOptions): Promise<IncomingMessage & { ok: boolean, body: Buffer | undefined }> { |
| 18 | const req = await httpStream(url, options) |
| 19 | return Object.assign(req, { |
| 20 | ok: _.inRange(req.statusCode!, 200, 300), |
| 21 | body: req.statusCode ? await buffer(req) : undefined, |
| 22 | }) |
| 23 | } |
| 24 | |
| 25 | export interface XRequestOptions extends https.RequestOptions { |
| 26 | body?: string | Buffer | Readable |
| 27 | proxy?: string // url format |
| 28 | // very basic cookie store |
| 29 | jar?: { [host: string]: { [cookieName: string]: string } } |
| 30 | noRedirect?: boolean |
| 31 | // throw for http-level errors. Default is true. |
| 32 | httpThrow?: boolean |
| 33 | } |
| 34 | |
| 35 | export declare namespace httpStream { |
| 36 | let defaultProxy: string | undefined |
| 37 | let defaultUA: string | undefined |
| 38 | } |
| 39 | export function httpStream(url: string, { body, proxy, jar, noRedirect, httpThrow=true, ...options }: XRequestOptions ={}, redirected: string[]=[]) { |
| 40 | const controller = new AbortController() |
| 41 | options.signal ??= controller.signal |
| 42 | return Object.assign(new Promise<IncomingMessage>(async (resolve, reject) => { |
| 43 | proxy ??= httpStream.defaultProxy |
| 44 | options.headers ??= {} |
| 45 | if (httpStream.defaultUA && !Object.keys(options.headers).find(k => k.toLowerCase() === 'user-agent')) |
| 46 | options.headers['user-agent'] = httpStream.defaultUA |
| 47 | if (body) { |
| 48 | options.method ||= 'POST' |
| 49 | if (_.isPlainObject(body)) { |
| 50 | options.headers['content-type'] ??= 'application/json' |
| 51 | body = JSON.stringify(body) |
| 52 | } |
| 53 | if (!(body instanceof Readable)) |
| 54 | options.headers['content-length'] ??= Buffer.byteLength(body) |
| 55 | } |
| 56 | const { auth, ...parsed } = parseHttpUrl(url) |
| 57 | const hostJar = jar && (jar[parsed.hostname || ''] ||= {}) |
| 58 | if (hostJar) { |
| 59 | options.headers.cookie = _.map(hostJar, (v,k) => `${k}=${v}; `).join('') |
| 60 | + (options.headers.cookie || '') // preserve parameter |
| 61 | } |
| 62 | const proxyParsed = proxy ? parseHttpUrl(proxy) : null |
| 63 | Object.assign(options, _.pick(proxyParsed || parsed, ['hostname', 'port', 'path', 'protocol'])) |
| 64 | if (auth) { |
| 65 | options.auth = auth |
| 66 | if (proxy) |
| 67 | url = parsed.protocol + '//' + parsed.host + parsed.path // rewrite without authentication part |
| 68 | } |
| 69 | if (proxy) { |
| 70 | options.path = url // full url as path |
| 71 | options.headers.host ??= parsed.host || undefined // keep original host header |
| 72 | } |
| 73 | // this needs the prefix "proxy-" |
| 74 | const proxyAuth = proxyParsed?.auth ? { 'proxy-authorization': `Basic ${Buffer.from(proxyParsed.auth, 'utf8').toString('base64')}` } : undefined |
| 75 | |
| 76 | // https through proxy is better with CONNECT |
| 77 | if (!proxy || parsed.protocol === 'http:' || !await connect()) |
| 78 | Object.assign(options.headers, proxyAuth) |
| 79 | |
| 80 | const proto = options.protocol === 'https:' ? https : http |
| 81 | const req = proto.request(options, res => { |
| 82 | console.debug("HTTP responded", res.statusCode, "to", url) |
| 83 | if (hostJar) for (const entry of res.headers['set-cookie'] || []) { |
| 84 | const [, k, v] = /(.+?)=([^;]+)/.exec(entry) || [] |
| 85 | if (!k) continue |
| 86 | if (v) hostJar[k] = v |
| 87 | else delete hostJar[k] |
| 88 | } |
| 89 | if (!res.statusCode || httpThrow && res.statusCode >= 400) |
| 90 | return reject(Error(String(res.statusCode), { cause: res })) |
| 91 | let r = res.headers.location |
| 92 | if (r && !noRedirect) { |
| 93 | const dest = new URL(r, url) // rewrite in case r is just a path, and thus relative to the current url |
| 94 | r = dest.toString() |
| 95 | const src = new URL(url) |
| 96 | const sameOrigin = src.protocol === dest.protocol && src.host === dest.host |
| 97 | return redirected.includes(r) ? reject(Error('endless http redirection')) |
| 98 | : redirected.length > 20 ? reject(Error('excessive http redirection')) |
| 99 | : resolve(httpStream(r, { |
| 100 | httpThrow, jar, proxy, |
| 101 | ..._.pick(options, ['agent', 'rejectUnauthorized', 'timeout']), |
| 102 | // forward some headers and exclude authorization if it's cross-origin |
| 103 | headers: options.headers && _.omit(options.headers, ['content-length', 'host', 'connection', 'transfer-encoding', sameOrigin ? '' : 'authorization']), |
| 104 | }, [...redirected, r])) |
| 105 | } |
| 106 | resolve(res) |
| 107 | }).on('error', (e: any) => { |
| 108 | if (proxy && e?.code === 'ECONNREFUSED') |
| 109 | console.debug("Cannot connect to proxy ", proxy) |
| 110 | e.cause ??= req // enrich the error |
| 111 | reject(e) |
| 112 | }) |
| 113 | if (options.timeout) // node only emits the timeout event, so destroy the request to unblock callers waiting for the body |
| 114 | req.setTimeout(options.timeout, () => req.destroy(Object.assign(Error('timeout'), { code: 'ETIMEDOUT' }))) |
| 115 | if (body && body instanceof Readable) |
| 116 | body.pipe(req).on('end', () => req.end()) |
| 117 | else |
| 118 | req.end(body) |
| 119 | |
| 120 | function connect() { |
| 121 | return proxyParsed && new Promise<boolean>(resolve => { |
| 122 | const path = `${parsed.hostname}:${parsed.port || 443}` |
| 123 | ;(proxyParsed.protocol === 'https:' ? https : http).request({ |
| 124 | ...proxyParsed, |
| 125 | auth: undefined, // void proxyParsed.auth |
| 126 | method: 'CONNECT', |
| 127 | path, |
| 128 | headers: { Host: path, ...proxyAuth } |
| 129 | }).on('connect', (res, socket) => { |
| 130 | if (res.statusCode !== 200) |
| 131 | return resolve(false) |
| 132 | // we are creating a TLS for every request, very inefficient. Consider optimizing in the future, especially for reading plugins from github, which makes tens of requests. |
| 133 | options.createConnection = () => tls.connect({ socket, servername: parsed.hostname || undefined }) |
| 134 | resolve(true) |
| 135 | }).on('response', res => { |
| 136 | console.debug("Proxy CONNECT response", res.statusCode, res.statusMessage) |
| 137 | resolve(false) |
| 138 | }).on('error', reject) |
| 139 | .end() |
| 140 | }) |
| 141 | } |
| 142 | |
| 143 | }), { |
| 144 | abort() { controller.abort() } |
| 145 | }) |
| 146 | } |
| 147 | |
| 148 | // works the same way as the now deprecated url.parse() |
| 149 | export function parseHttpUrl(url: string) { |
| 150 | const parsed = new URL(url) |
| 151 | const options = urlToHttpOptions(parsed) |
| 152 | const withoutHash = url.split('#', 1)[0]! |
| 153 | const authority = /^[a-z][a-z\d+.-]*:\/\/[^/?#]*/i.exec(withoutHash)?.[0] |
| 154 | const unresolvedPath = !authority ? '/' : enforceStarting('/', withoutHash.slice(authority.length)) |
| 155 | return { |
| 156 | ...options, |
| 157 | host: parsed.host, |
| 158 | path: unresolvedPath.replace(/[\u0000-\u0020\u0100-\u{10FFFF}]/gu, encodeURIComponent), // keep unresolved paths for tests, but escape characters that Node request rejects as unescaped |
| 159 | } |
| 160 | } |