import * as http from 'node:http'; import * as https from 'node:https'; import CacheableLookup from 'cacheable-lookup'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import * as undici from 'undici'; import { LookupFunction } from 'node:net'; import { TransformStream } from 'node:stream/web'; import * as dns from 'node:dns'; export type IpChecker = (ip: string) => boolean; @Injectable() export class HttpRequestService { /** * Get http non-proxy agent (undici) */ private agent: undici.Agent; /** * Get http proxy or non-proxy agent (undici) */ public proxiedAgent: undici.ProxyAgent | undici.Agent; /** * Get http non-proxy agent */ private http: http.Agent; /** * Get https non-proxy agent */ private https: https.Agent; /** * Get http proxy or non-proxy agent */ public httpAgent: http.Agent; /** * Get https proxy or non-proxy agent */ public httpsAgent: https.Agent; public readonly dnsCache: CacheableLookup; public readonly clientDefaults: undici.Agent.Options; constructor( @Inject(DI.config) private config: Config, ) { this.dnsCache = new CacheableLookup({ maxTtl: 3600, // 1hours errorTtl: 30, // 30secs lookup: false, // nativeのdns.lookupにfallbackしない }); this.clientDefaults = { keepAliveTimeout: 4 * 1000, keepAliveMaxTimeout: 10 * 60 * 1000, keepAliveTimeoutThreshold: 1 * 1000, strictContentLength: true, connect: { maxCachedSessions: 100, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 }, } this.agent = new undici.Agent({ ...this.clientDefaults, }); this.http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, lookup: this.dnsCache.lookup, } as http.AgentOptions); this.https = new https.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, lookup: this.dnsCache.lookup, } as https.AgentOptions); const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); this.proxiedAgent = config.proxy ? new undici.ProxyAgent({ ...this.clientDefaults, connections: maxSockets, uri: config.proxy, }) : this.agent; this.httpAgent = config.proxy ? new HttpProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, }) : this.http; this.httpsAgent = config.proxy ? new HttpsProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, }) : this.https; } /** * Get agent by URL * @param url URL * @param bypassProxy Allways bypass proxy */ @bindThis public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { return this.agent; } else { return this.proxiedAgent; } } /** * Get agent by URL * @param url URL * @param bypassProxy Allways bypass proxy */ @bindThis public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { return url.protocol === 'http:' ? this.http : this.https; } else { return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent; } } /** * check ip */ @bindThis public checkIp(url: URL, fn: IpChecker): Promise { const lookup = this.dnsCache.lookup as LookupFunction || dns.lookup; return new Promise((resolve, reject) => { lookup(url.hostname, {}, (err, ip) => { if (err) { resolve(false); } else { resolve(fn(ip)); } }); }); } @bindThis public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { const res = await this.fetch({ url, headers: Object.assign({ 'User-Agent': this.config.userAgent, Accept: accept, }, headers ?? {}), timeout, size: 1024 * 256, }); return await res.json(); } @bindThis public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { const res = await this.fetch({ url, headers: Object.assign({ 'User-Agent': this.config.userAgent, Accept: accept, }, headers ?? {}), timeout, }); return await res.text(); } @bindThis public async fetch(args: { url: string, method?: string, body?: string, headers?: Record, timeout?: number, size?: number, redirect?: RequestRedirect | undefined, dispatcher?: undici.Dispatcher, ipCheckers?: { type: 'black' | 'white', fn: IpChecker, }[], noOkError?: boolean, }): Promise { const url = new URL(args.url); if (args.ipCheckers) { for (const check of args.ipCheckers) { const result = await this.checkIp(url, check.fn); if ( (check.type === 'black' && result === true) || (check.type === 'white' && result === false) ) { throw new StatusError('IP is not allowed', 403, 'IP is not allowed'); } } } const timeout = args.timeout ?? 10 * 1000; const controller = new AbortController(); setTimeout(() => { controller.abort(); }, timeout * 6); const res = await Promise.race([ undici.fetch(args.url, { method: args.method ?? 'GET', headers: args.headers, body: args.body, redirect: args.redirect, dispatcher: args.dispatcher ?? this.getAgentByUrl(url), keepalive: true, signal: controller.signal, }), new Promise((res) => setTimeout(() => res(null))) ]); if (res == null) { throw new StatusError(`Request Timeout`, 408, 'Request Timeout'); } if (!res.ok && !args.noOkError) { throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); } return ({ ...res, body: this.fetchLimiter(res, args.size), }); } /** * Fetch body limiter * @param res undici.Response * @param size number of Max size (Bytes) (default: 10MiB) * @returns ReadableStream (provided by node:stream/web) */ @bindThis private fetchLimiter(res: undici.Response, size: number = 10 * 1024 * 1024) { if (res.body == null) return null; let total = 0; return res.body.pipeThrough(new TransformStream({ start() {}, transform(chunk, controller) { // TypeScirptグローバルの定義はUnit8ArrayだがundiciはReadableStreamを渡してくるので一応変換 const uint8 = new Uint8Array(chunk); total += uint8.length; if (total > size) { controller.error(new StatusError(`Payload Too Large`, 413, 'Payload Too Large')); } else { controller.enqueue(uint8); } }, flush() {}, })); } }