Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into serve-stream

Browse source
This commit is contained in:
tamaina 2023-01-24 01:03:55 +00:00
parent 6f6f257d8d 869854eae7
commit cb2823dc22
4 changed files with 54 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
packages/backend/src/core/DownloadService.ts
View file

@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common';
import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip';
import chalk from 'chalk';
import { buildConnector, Response } from 'undici';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
@ -12,8 +13,6 @@ import { createTemp } from '@/misc/create-temp.js';
import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { buildConnector } from 'undici';
import type { Response } from 'undici';
const pipeline = util.promisify(stream.pipeline);
import { bindThis } from '@/decorators.js';
@ -37,8 +36,7 @@ export class DownloadService {
) {
this.logger = this.loggerService.getLogger('download');
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
{
this.undiciFetcher = this.httpRequestService.createFetcher({
connect: process.env.NODE_ENV === 'development' ?
this.httpRequestService.clientDefaults.connect
:
@ -46,14 +44,12 @@ export class DownloadService {
buildConnector({
...this.httpRequestService.clientDefaults.connect,
}),
(ip) => !this.isPrivateIp(ip)
(ip) => !this.isPrivateIp(ip),
),
bodyTimeout: 30 * 1000,
},
{
}, {
connect: this.httpRequestService.clientDefaults.connect,
}
), this.logger);
}, this.logger);
}
@bindThis

41
packages/backend/src/core/HttpRequestService.ts
View file

@ -1,14 +1,14 @@
import * as http from 'node:http';
import * as https from 'node:https';
import { LookupFunction } from 'node:net';
import CacheableLookup from 'cacheable-lookup';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import * as undici from 'undici';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import * as undici from 'undici';
import { LookupFunction } from 'node:net';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
@ -62,7 +62,7 @@ export class UndiciFetcher {
undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called', socket);
if (err) {
this.logger?.debug(`Socket error`, err);
this.logger?.debug('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
@ -79,20 +79,20 @@ export class UndiciFetcher {
uri: args.proxy.uri,
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy.options?.connect ?? args.agentOptions.connect) !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called (secure)', socket);
if (err) {
this.logger?.debug(`Socket error`, err);
this.logger?.debug('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : (args.proxy?.options?.connect ?? args.agentOptions.connect),
} : (args.proxy.options?.connect ?? args.agentOptions.connect),
})
: this.nonProxiedAgent;
}
@ -115,7 +115,7 @@ export class UndiciFetcher {
public async fetch(
url: string | URL,
options: undici.RequestInit = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
): Promise<undici.Response> {
const res = await undici.fetch(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
@ -142,7 +142,7 @@ export class UndiciFetcher {
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
}
},
);
return await res.json() as T;
@ -156,7 +156,7 @@ export class UndiciFetcher {
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
}
},
);
return await res.text();
@ -219,18 +219,18 @@ export class HttpRequestService {
maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
},
}
};
this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
this.defaultFetcher = this.createFetcher({}, {}, this.logger);
this.fetch = this.defaultFetcher.fetch;
this.getHtml = this.defaultFetcher.getHtml;
this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
this.defaultJsonFetcher = this.createFetcher({
maxResponseSize: 1024 * 256,
}), this.logger);
}, {}, this.logger);
this.getJson = this.defaultJsonFetcher.getJson;
@ -272,7 +272,7 @@ export class HttpRequestService {
}
@bindThis
public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
private getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
return {
agentOptions: {
...this.clientDefaults,
@ -284,11 +284,16 @@ export class HttpRequestService {
options: {
connections: this.maxSockets,
...proxyOpts,
}
}
},
},
} : {}),
userAgent: this.config.userAgent,
};
}
@bindThis
public createFetcher(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}, logger: Logger) {
return new UndiciFetcher(this.getStandardUndiciFetcherOption(opts, proxyOpts), logger);
}
/**
@ -314,13 +319,13 @@ export class HttpRequestService {
connector(options, (err, socket) => {
this.logger.debug('Socket connector (with ip checker) called', socket);
if (err) {
this.logger.error(`Socket error`, err)
this.logger.error('Socket error', err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
if (socket.remoteAddress == undefined) {
this.logger.error(`Socket error: remoteAddress is undefined`);
this.logger.error('Socket error: remoteAddress is undefined');
cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
return;
}

10
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -41,10 +41,10 @@ export class ApRequestService {
private httpRequestService: HttpRequestService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
this.logger = this.loggerService.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = this.httpRequestService.createFetcher({
maxRedirections: 0,
}), this.logger );
}, {}, this.logger);
}
@bindThis
@ -169,7 +169,7 @@ export class ApRequestService {
method: req.request.method,
headers: req.request.headers,
body,
}
},
);
}
@ -197,7 +197,7 @@ export class ApRequestService {
{
method: req.request.method,
headers: req.request.headers,
}
},
);
return await res.json();

10
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -8,13 +8,13 @@ import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { isCollectionOrOrderedCollection } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js';
import type Logger from '@/logger.js';
export class Resolver {
private history: Set<string>;
@ -39,10 +39,10 @@ export class Resolver {
private recursionLimit = 100,
) {
this.history = new Set();
this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
this.logger = this.loggerService.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = this.httpRequestService.createFetcher({
maxRedirections: 0,
}), this.logger);
}, {}, this.logger);
}
@bindThis