From 238f923b419b487337fbcaa5ba415bf2f823f27e Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 24 Jan 2023 08:31:02 +0900 Subject: [PATCH 1/2] =?UTF-8?q?refactor(server):=20httpRequestService?= =?UTF-8?q?=E3=81=AEUndiciFetcher=E4=BE=9D=E5=AD=98=E3=82=92=E3=81=AA?= =?UTF-8?q?=E3=82=8B=E3=81=B9=E3=81=8F=E3=82=AB=E3=83=97=E3=82=BB=E3=83=AB?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/DownloadService.ts | 33 ++++++------ .../backend/src/core/HttpRequestService.ts | 53 ++++++++++--------- .../src/core/activitypub/ApResolverService.ts | 10 ++-- 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index a3078bff45..e4632f857a 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -6,6 +6,7 @@ import IPCIDR from 'ip-cidr'; import PrivateIp from 'private-ip'; import got, * as Got from 'got'; import chalk from 'chalk'; +import { buildConnector } from 'undici'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; @@ -13,7 +14,6 @@ import { createTemp } from '@/misc/create-temp.js'; import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; -import { buildConnector } from 'undici'; const pipeline = util.promisify(stream.pipeline); import { bindThis } from '@/decorators.js'; @@ -32,23 +32,20 @@ export class DownloadService { ) { this.logger = this.loggerService.getLogger('download'); - this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption( - { - connect: process.env.NODE_ENV === 'development' ? - this.httpRequestService.clientDefaults.connect - : - this.httpRequestService.getConnectorWithIpCheck( - buildConnector({ - ...this.httpRequestService.clientDefaults.connect, - }), - (ip) => !this.isPrivateIp(ip) - ), - bodyTimeout: 30 * 1000, - }, - { - connect: this.httpRequestService.clientDefaults.connect, - } - ), this.logger); + this.undiciFetcher = this.httpRequestService.createFetcher({ + connect: process.env.NODE_ENV === 'development' ? + this.httpRequestService.clientDefaults.connect + : + this.httpRequestService.getConnectorWithIpCheck( + buildConnector({ + ...this.httpRequestService.clientDefaults.connect, + }), + (ip) => !this.isPrivateIp(ip), + ), + bodyTimeout: 30 * 1000, + }, { + connect: this.httpRequestService.clientDefaults.connect, + }, this.logger); } @bindThis diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 2864ad4405..3a70ac77f9 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -1,14 +1,14 @@ import * as http from 'node:http'; import * as https from 'node:https'; +import { LookupFunction } from 'node:net'; import CacheableLookup from 'cacheable-lookup'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; +import * as undici from 'undici'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; -import * as undici from 'undici'; -import { LookupFunction } from 'node:net'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; @@ -62,7 +62,7 @@ export class UndiciFetcher { undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => { this.logger?.debug('Socket connector called', socket); if (err) { - this.logger?.debug(`Socket error`, err); + this.logger?.debug('Socket error', err); cb(new Error(`Error while socket connecting\n${err}`), null); return; } @@ -79,20 +79,20 @@ export class UndiciFetcher { uri: args.proxy.uri, - connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function') + connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy.options?.connect ?? args.agentOptions.connect) !== 'function') ? (options, cb) => { // Custom connector for debug undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => { this.logger?.debug('Socket connector called (secure)', socket); if (err) { - this.logger?.debug(`Socket error`, err); + this.logger?.debug('Socket error', err); cb(new Error(`Error while socket connecting\n${err}`), null); return; } this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`); cb(null, socket); }); - } : (args.proxy?.options?.connect ?? args.agentOptions.connect), + } : (args.proxy.options?.connect ?? args.agentOptions.connect), }) : this.nonProxiedAgent; } @@ -115,7 +115,7 @@ export class UndiciFetcher { public async fetch( url: string | URL, options: undici.RequestInit = {}, - privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false } + privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }, ): Promise { const res = await undici.fetch(url, { dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), @@ -142,7 +142,7 @@ export class UndiciFetcher { headers: Object.assign({ Accept: accept, }, headers ?? {}), - } + }, ); return await res.json() as T; @@ -156,7 +156,7 @@ export class UndiciFetcher { headers: Object.assign({ Accept: accept, }, headers ?? {}), - } + }, ); return await res.text(); @@ -219,18 +219,18 @@ export class HttpRequestService { maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 }, - } + }; this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128); - this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger); + this.defaultFetcher = this.createFetcher({}, {}, this.logger); this.fetch = this.defaultFetcher.fetch; this.getHtml = this.defaultFetcher.getHtml; - this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({ + this.defaultJsonFetcher = this.createFetcher({ maxResponseSize: 1024 * 256, - }), this.logger); + }, {}, this.logger); this.getJson = this.defaultJsonFetcher.getJson; @@ -272,23 +272,28 @@ export class HttpRequestService { } @bindThis - public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) { + private getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) { return { agentOptions: { ...this.clientDefaults, ...opts, }, ...(this.config.proxy ? { - proxy: { - uri: this.config.proxy, - options: { - connections: this.maxSockets, - ...proxyOpts, - } - } + proxy: { + uri: this.config.proxy, + options: { + connections: this.maxSockets, + ...proxyOpts, + }, + }, } : {}), userAgent: this.config.userAgent, - } + }; + } + + @bindThis + public createFetcher(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}, logger: Logger) { + return new UndiciFetcher(this.getStandardUndiciFetcherOption(opts, proxyOpts), logger); } /** @@ -314,13 +319,13 @@ export class HttpRequestService { connector(options, (err, socket) => { this.logger.debug('Socket connector (with ip checker) called', socket); if (err) { - this.logger.error(`Socket error`, err) + this.logger.error('Socket error', err); cb(new Error(`Error while socket connecting\n${err}`), null); return; } if (socket.remoteAddress == undefined) { - this.logger.error(`Socket error: remoteAddress is undefined`); + this.logger.error('Socket error: remoteAddress is undefined'); cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null); return; } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index e51ae37954..94bb6d6c55 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -8,13 +8,13 @@ import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js' import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; import { isCollectionOrOrderedCollection } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; -import { LoggerService } from '@/core/LoggerService.js'; import type { IObject, ICollection, IOrderedCollection } from './type.js'; -import type Logger from '@/logger.js'; export class Resolver { private history: Set; @@ -39,10 +39,10 @@ export class Resolver { private recursionLimit = 100, ) { this.history = new Set(); - this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる - this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ + this.logger = this.loggerService.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる + this.undiciFetcher = this.httpRequestService.createFetcher({ maxRedirections: 0, - }), this.logger); + }, {}, this.logger); } @bindThis From 869854eae736cf5c910f533191ea0d1f7c9755dd Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 24 Jan 2023 08:32:17 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=E3=82=B3=E3=83=9F=E3=83=83=E3=83=88?= =?UTF-8?q?=E6=BC=8F=E3=82=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/DownloadService.ts | 1 - .../backend/src/core/activitypub/ApRequestService.ts | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index e4632f857a..c5b2bcaef4 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -4,7 +4,6 @@ import * as util from 'node:util'; import { Inject, Injectable } from '@nestjs/common'; import IPCIDR from 'ip-cidr'; import PrivateIp from 'private-ip'; -import got, * as Got from 'got'; import chalk from 'chalk'; import { buildConnector } from 'undici'; import { DI } from '@/di-symbols.js'; diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index ab22a0c411..fd7319702e 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -41,10 +41,10 @@ export class ApRequestService { private httpRequestService: HttpRequestService, private loggerService: LoggerService, ) { - this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる - this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ + this.logger = this.loggerService.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる + this.undiciFetcher = this.httpRequestService.createFetcher({ maxRedirections: 0, - }), this.logger ); + }, {}, this.logger); } @bindThis @@ -169,7 +169,7 @@ export class ApRequestService { method: req.request.method, headers: req.request.headers, body, - } + }, ); } @@ -197,7 +197,7 @@ export class ApRequestService { { method: req.request.method, headers: req.request.headers, - } + }, ); return await res.json();