Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into serve-stream

Browse source
This commit is contained in:
tamaina 2023-01-24 01:03:55 +00:00
parent 6f6f257d8d 869854eae7
commit cb2823dc22
4 changed files with 54 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
packages/backend/src/core/DownloadService.ts
View file

@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common';
import IPCIDR from 'ip-cidr'; import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip'; import PrivateIp from 'private-ip';
import chalk from 'chalk'; import chalk from 'chalk';
import { buildConnector, Response } from 'undici';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
@ -12,8 +13,6 @@ import { createTemp } from '@/misc/create-temp.js';
import { StatusError } from '@/misc/status-error.js'; import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js'; import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { buildConnector } from 'undici';
import type { Response } from 'undici';
const pipeline = util.promisify(stream.pipeline); const pipeline = util.promisify(stream.pipeline);
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
@ -37,23 +36,20 @@ export class DownloadService {
) { ) {
this.logger = this.loggerService.getLogger('download'); this.logger = this.loggerService.getLogger('download');
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption( this.undiciFetcher = this.httpRequestService.createFetcher({
{ connect: process.env.NODE_ENV === 'development' ?
connect: process.env.NODE_ENV === 'development' ? this.httpRequestService.clientDefaults.connect
this.httpRequestService.clientDefaults.connect :
: this.httpRequestService.getConnectorWithIpCheck(
this.httpRequestService.getConnectorWithIpCheck( buildConnector({
buildConnector({ ...this.httpRequestService.clientDefaults.connect,
...this.httpRequestService.clientDefaults.connect, }),
}), (ip) => !this.isPrivateIp(ip),
(ip) => !this.isPrivateIp(ip) ),
), bodyTimeout: 30 * 1000,
bodyTimeout: 30 * 1000, }, {
}, connect: this.httpRequestService.clientDefaults.connect,
{ }, this.logger);
connect: this.httpRequestService.clientDefaults.connect,
}
), this.logger);
} }
@bindThis @bindThis

53
packages/backend/src/core/HttpRequestService.ts
View file

@ -1,14 +1,14 @@
import * as http from 'node:http'; import * as http from 'node:http';
import * as https from 'node:https'; import * as https from 'node:https';
import { LookupFunction } from 'node:net';
import CacheableLookup from 'cacheable-lookup'; import CacheableLookup from 'cacheable-lookup';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import * as undici from 'undici';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { StatusError } from '@/misc/status-error.js'; import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import * as undici from 'undici';
import { LookupFunction } from 'node:net';
import { LoggerService } from '@/core/LoggerService.js'; import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
@ -62,7 +62,7 @@ export class UndiciFetcher {
undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => { undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called', socket); this.logger?.debug('Socket connector called', socket);
if (err) { if (err) {
this.logger?.debug(`Socket error`, err); this.logger?.debug('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null); cb(new Error(`Error while socket connecting\n${err}`), null);
return; return;
} }
@ -79,20 +79,20 @@ export class UndiciFetcher {
uri: args.proxy.uri, uri: args.proxy.uri,
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function') connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy.options?.connect ?? args.agentOptions.connect) !== 'function')
? (options, cb) => { ? (options, cb) => {
// Custom connector for debug // Custom connector for debug
undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => { undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called (secure)', socket); this.logger?.debug('Socket connector called (secure)', socket);
if (err) { if (err) {
this.logger?.debug(`Socket error`, err); this.logger?.debug('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null); cb(new Error(`Error while socket connecting\n${err}`), null);
return; return;
} }
this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`); this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket); cb(null, socket);
}); });
} : (args.proxy?.options?.connect ?? args.agentOptions.connect), } : (args.proxy.options?.connect ?? args.agentOptions.connect),
}) })
: this.nonProxiedAgent; : this.nonProxiedAgent;
} }
@ -115,7 +115,7 @@ export class UndiciFetcher {
public async fetch( public async fetch(
url: string | URL, url: string | URL,
options: undici.RequestInit = {}, options: undici.RequestInit = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false } privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
): Promise<undici.Response> { ): Promise<undici.Response> {
const res = await undici.fetch(url, { const res = await undici.fetch(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
@ -142,7 +142,7 @@ export class UndiciFetcher {
headers: Object.assign({ headers: Object.assign({
Accept: accept, Accept: accept,
}, headers ?? {}), }, headers ?? {}),
} },
); );
return await res.json() as T; return await res.json() as T;
@ -156,7 +156,7 @@ export class UndiciFetcher {
headers: Object.assign({ headers: Object.assign({
Accept: accept, Accept: accept,
}, headers ?? {}), }, headers ?? {}),
} },
); );
return await res.text(); return await res.text();
@ -219,18 +219,18 @@ export class HttpRequestService {
maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
}, },
} };
this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128); this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger); this.defaultFetcher = this.createFetcher({}, {}, this.logger);
this.fetch = this.defaultFetcher.fetch; this.fetch = this.defaultFetcher.fetch;
this.getHtml = this.defaultFetcher.getHtml; this.getHtml = this.defaultFetcher.getHtml;
this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({ this.defaultJsonFetcher = this.createFetcher({
maxResponseSize: 1024 * 256, maxResponseSize: 1024 * 256,
}), this.logger); }, {}, this.logger);
this.getJson = this.defaultJsonFetcher.getJson; this.getJson = this.defaultJsonFetcher.getJson;
@ -272,23 +272,28 @@ export class HttpRequestService {
} }
@bindThis @bindThis
public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) { private getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
return { return {
agentOptions: { agentOptions: {
...this.clientDefaults, ...this.clientDefaults,
...opts, ...opts,
}, },
...(this.config.proxy ? { ...(this.config.proxy ? {
proxy: { proxy: {
uri: this.config.proxy, uri: this.config.proxy,
options: { options: {
connections: this.maxSockets, connections: this.maxSockets,
...proxyOpts, ...proxyOpts,
} },
} },
} : {}), } : {}),
userAgent: this.config.userAgent, userAgent: this.config.userAgent,
} };
}
@bindThis
public createFetcher(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}, logger: Logger) {
return new UndiciFetcher(this.getStandardUndiciFetcherOption(opts, proxyOpts), logger);
} }
/** /**
@ -314,13 +319,13 @@ export class HttpRequestService {
connector(options, (err, socket) => { connector(options, (err, socket) => {
this.logger.debug('Socket connector (with ip checker) called', socket); this.logger.debug('Socket connector (with ip checker) called', socket);
if (err) { if (err) {
this.logger.error(`Socket error`, err) this.logger.error('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null); cb(new Error(`Error while socket connecting\n${err}`), null);
return; return;
} }
if (socket.remoteAddress == undefined) { if (socket.remoteAddress == undefined) {
this.logger.error(`Socket error: remoteAddress is undefined`); this.logger.error('Socket error: remoteAddress is undefined');
cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null); cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
return; return;
} }

10
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -41,10 +41,10 @@ export class ApRequestService {
private httpRequestService: HttpRequestService, private httpRequestService: HttpRequestService,
private loggerService: LoggerService, private loggerService: LoggerService,
) { ) {
this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる this.logger = this.loggerService.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ this.undiciFetcher = this.httpRequestService.createFetcher({
maxRedirections: 0, maxRedirections: 0,
}), this.logger ); }, {}, this.logger);
} }
@bindThis @bindThis
@ -169,7 +169,7 @@ export class ApRequestService {
method: req.request.method, method: req.request.method,
headers: req.request.headers, headers: req.request.headers,
body, body,
} },
); );
} }
@ -197,7 +197,7 @@ export class ApRequestService {
{ {
method: req.request.method, method: req.request.method,
headers: req.request.headers, headers: req.request.headers,
} },
); );
return await res.json(); return await res.json();

10
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -8,13 +8,13 @@ import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { isCollectionOrOrderedCollection } from './type.js'; import { isCollectionOrOrderedCollection } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js'; import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js'; import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js'; import { ApRequestService } from './ApRequestService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js'; import type { IObject, ICollection, IOrderedCollection } from './type.js';
import type Logger from '@/logger.js';
export class Resolver { export class Resolver {
private history: Set<string>; private history: Set<string>;
@ -39,10 +39,10 @@ export class Resolver {
private recursionLimit = 100, private recursionLimit = 100,
) { ) {
this.history = new Set(); this.history = new Set();
this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる this.logger = this.loggerService.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ this.undiciFetcher = this.httpRequestService.createFetcher({
maxRedirections: 0, maxRedirections: 0,
}), this.logger); }, {}, this.logger);
} }
@bindThis @bindThis