Merge branch 'fetch' into serve-stream

packages/backend/src/core/CaptchaService.ts

@@ -27,16 +27,19 @@ export class CaptchaService {
 			response,
 		});
 	
-		const res = await fetch(url, {
-			method: 'POST',
-			body: params,
-			headers: {
-				'User-Agent': this.config.userAgent,
+		const res = await this.httpRequestService.fetch(
+			url,
+			{
+				method: 'POST',
+				body: params,
+				headers: {
+					'User-Agent': this.config.userAgent,
+				},
 			},
-			// TODO
-			//timeout: 10 * 1000,
-			agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy),
-		}).catch(err => {
+			{
+				noOkError: true,
+			}
+		).catch(err => {
 			throw `${err.message ?? err}`;
 		});
 	

packages/backend/src/core/DownloadService.ts

@@ -8,11 +8,12 @@ import got, * as Got from 'got';
 import chalk from 'chalk';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { StatusError } from '@/misc/status-error.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import type Logger from '@/logger.js';
+import { buildConnector } from 'undici';
 
 const pipeline = util.promisify(stream.pipeline);
 import { bindThis } from '@/decorators.js';
@@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js';
 @Injectable()
 export class DownloadService {
 	private logger: Logger;
+	private undiciFetcher: UndiciFetcher;
 
 	constructor(
 		@Inject(DI.config)
@@ -29,69 +31,58 @@ export class DownloadService {
 		private loggerService: LoggerService,
 	) {
 		this.logger = this.loggerService.getLogger('download');
+
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
+			{
+				connect: process.env.NODE_ENV === 'development' ?
+					this.httpRequestService.clientDefaults.connect
+					:
+					this.httpRequestService.getConnectorWithIpCheck(
+						buildConnector({
+							...this.httpRequestService.clientDefaults.connect,
+						}),
+						(ip) => !this.isPrivateIp(ip)
+					),
+				bodyTimeout: 30 * 1000,
+			},
+			{
+				connect: this.httpRequestService.clientDefaults.connect,
+			}
+		), this.logger);
 	}
 
 	@bindThis
-	public gotUrl(url: string): Got.Request {
+	public fetchUrl(url: string): any {
 		this.logger.info(`Downloading ${chalk.cyan(url)} ...`);
 	
 		const timeout = 30 * 1000;
 		const operationTimeout = 60 * 1000;
 		const maxSize = this.config.maxFileSize ?? 262144000;
-	
-		const req = got.stream(url, {
-			headers: {
-				'User-Agent': this.config.userAgent,
-			},
-			timeout: {
-				lookup: timeout,
-				connect: timeout,
-				secureConnect: timeout,
-				socket: timeout,	// read timeout
-				response: timeout,
-				send: timeout,
-				request: operationTimeout,	// whole operation timeout
-			},
-			agent: {
-				http: this.httpRequestService.httpAgent,
-				https: this.httpRequestService.httpsAgent,
-			},
-			http2: false,	// default
-			retry: {
-				limit: 0,
-			},
-		}).on('response', (res: Got.Response) => {
-			if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
-				if (this.isPrivateIp(res.ip)) {
-					this.logger.warn(`Blocked address: ${res.ip}`);
-					req.destroy();
-				}
-			}
-	
-			const contentLength = res.headers['content-length'];
-			if (contentLength != null) {
-				const size = Number(contentLength);
-				if (size > maxSize) {
-					this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
-					req.destroy();
-				}
-			}
-		}).on('downloadProgress', (progress: Got.Progress) => {
-			if (progress.transferred > maxSize) {
-				this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
-				req.destroy();
-			}
-		});
 
-		return req;
+		const response = await this.undiciFetcher.fetch(
+			url,
+			{
+				method: 'GET',
+				headers: {
+					'User-Agent': this.config.userAgent,
+				},
+			}
+		);
+
+		if (response.body === null) {
+			throw new StatusError('No body', 400, 'No body');
+		}
+	
+		this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
+
+		return response;
 	}
 
 	@bindThis
-	public async pipeRequestToFile(req: Got.Request, path: string): Promise<void> {
-		const copied = req.pipe(new stream.PassThrough());
+	public async pipeRequestToFile(response: any, path: string): Promise<void> {
 		try {
 			this.logger.info(`Saving File to ${chalk.cyanBright(path)} from downloading ...`);
-			await pipeline(copied, fs.createWriteStream(path));
+			await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
 		} catch (e) {
 			if (e instanceof Got.HTTPError) {
 				throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
@@ -125,7 +116,7 @@ export class DownloadService {
 			cleanup();
 		}
 	}
-	
+
 	@bindThis
 	private isPrivateIp(ip: string): boolean {
 		for (const net of this.config.allowedPrivateNetworks ?? []) {
@@ -135,6 +126,6 @@ export class DownloadService {
 			}
 		}
 
-		return PrivateIp(ip);
+		return PrivateIp(ip) ?? false;
 	}
 }

packages/backend/src/core/FederatedInstanceService.ts

@@ -22,7 +22,7 @@ export class FederatedInstanceService {
 	}
 
 	@bindThis
-	public async registerOrFetchInstanceDoc(host: string): Promise<Instance> {
+	public async fetch(host: string): Promise<Instance> {
 		host = this.utilityService.toPuny(host);
 	
 		const cached = this.cache.get(host);
@@ -35,7 +35,6 @@ export class FederatedInstanceService {
 				id: this.idService.genId(),
 				host,
 				caughtAt: new Date(),
-				lastCommunicatedAt: new Date(),
 			}).then(x => this.instancesRepository.findOneByOrFail(x.identifiers[0]));
 	
 			this.cache.set(host, i);
@@ -45,4 +44,17 @@ export class FederatedInstanceService {
 			return index;
 		}
 	}
+
+	@bindThis
+	public async updateCachePartial(host: string, data: Partial<Instance>): Promise<void> {
+		host = this.utilityService.toPuny(host);
+	
+		const cached = this.cache.get(host);
+		if (cached == null) return;
+	
+		this.cache.set(host, {
+			...cached,
+			...data,
+		});
+	}
 }

packages/backend/src/core/FetchInstanceMetadataService.ts

@@ -1,7 +1,6 @@
 import { URL } from 'node:url';
 import { Inject, Injectable } from '@nestjs/common';
 import { JSDOM } from 'jsdom';
-import fetch from 'node-fetch';
 import tinycolor from 'tinycolor2';
 import type { Instance } from '@/models/entities/Instance.js';
 import type { InstancesRepository } from '@/models/index.js';
@@ -191,11 +190,7 @@ export class FetchInstanceMetadataService {
 	
 		const faviconUrl = url + '/favicon.ico';
 	
-		const favicon = await fetch(faviconUrl, {
-			// TODO
-			//timeout: 10000,
-			agent: url => this.httpRequestService.getAgentByUrl(url),
-		});
+		const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true });
 	
 		if (favicon.ok) {
 			return faviconUrl;

packages/backend/src/core/HttpRequestService.ts

@@ -1,67 +1,255 @@
 import * as http from 'node:http';
 import * as https from 'node:https';
 import CacheableLookup from 'cacheable-lookup';
-import fetch from 'node-fetch';
 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import { Inject, Injectable } from '@nestjs/common';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { StatusError } from '@/misc/status-error.js';
 import { bindThis } from '@/decorators.js';
-import type { Response } from 'node-fetch';
-import type { URL } from 'node:url';
+import * as undici from 'undici';
+import { LookupFunction } from 'node:net';
+import { LoggerService } from '@/core/LoggerService.js';
+import type Logger from '@/logger.js';
+
+// true to allow, false to deny
+export type IpChecker = (ip: string) => boolean;
+
+/* 
+ *  Child class to create and save Agent for fetch.
+ *  You should construct this when you want
+ *  to change timeout, size limit, socket connect function, etc.
+ */
+export class UndiciFetcher {
+	/**
+	 * Get http non-proxy agent (undici)
+	 */
+	public nonProxiedAgent: undici.Agent;
+
+	/**
+	 * Get http proxy or non-proxy agent (undici)
+	 */
+	public agent: undici.ProxyAgent | undici.Agent;
+
+	private proxyBypassHosts: string[];
+	private userAgent: string | undefined;
+
+	private logger: Logger | undefined;
+
+	constructor(
+		args: {
+			agentOptions: undici.Agent.Options;
+			proxy?: {
+				uri: string;
+				options?: undici.Agent.Options; // Override of agentOptions
+			},
+			proxyBypassHosts?: string[];
+			userAgent?: string;
+		},
+		logger?: Logger,
+	) {
+		this.logger = logger;
+		this.logger?.debug('UndiciFetcher constructor', args);
+
+		this.proxyBypassHosts = args.proxyBypassHosts ?? [];
+		this.userAgent = args.userAgent;
+
+		this.nonProxiedAgent = new undici.Agent({
+			...args.agentOptions,
+			connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
+				? (options, cb) => {
+					// Custom connector for debug
+					undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
+						this.logger?.debug('Socket connector called', socket);
+						if (err) {
+							this.logger?.debug(`Socket error`, err);
+							cb(new Error(`Error while socket connecting\n${err}`), null);
+							return;
+						}
+						this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
+						cb(null, socket);
+					});
+				} : args.agentOptions.connect,
+		});
+
+		this.agent = args.proxy
+			? new undici.ProxyAgent({
+				...args.agentOptions,
+				...args.proxy.options,
+
+				uri: args.proxy.uri,
+
+				connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
+					? (options, cb) => {
+						// Custom connector for debug
+						undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
+							this.logger?.debug('Socket connector called (secure)', socket);
+							if (err) {
+								this.logger?.debug(`Socket error`, err);
+								cb(new Error(`Error while socket connecting\n${err}`), null);
+								return;
+							}
+							this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
+							cb(null, socket);
+						});
+					} : (args.proxy?.options?.connect ?? args.agentOptions.connect),
+			})
+			: this.nonProxiedAgent;
+	}
+
+	/**
+	 * Get agent by URL
+	 * @param url URL
+	 * @param bypassProxy Allways bypass proxy
+	 */
+	@bindThis
+	public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
+		if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
+			return this.nonProxiedAgent;
+		} else {
+			return this.agent;
+		}
+	}
+
+	@bindThis
+	public async fetch(
+		url: string | URL,
+		options: undici.RequestInit = {},
+		privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
+	): Promise<undici.Response> {
+		const res = await undici.fetch(url, {
+			dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
+			...options,
+		}).catch((err) => {
+			this.logger?.error('fetch error', err);
+			throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
+		});
+		if (!res.ok && !privateOptions.noOkError) {
+			throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
+		}
+		return res;
+	}
+
+	@bindThis
+	public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
+		const res = await this.fetch(
+			url,
+			{
+				headers: Object.assign({
+					'User-Agent': this.userAgent,
+					Accept: accept,
+				}, headers ?? {}),
+			}
+		);
+
+		return await res.json() as T;
+	}
+
+	@bindThis
+	public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
+		const res = await this.fetch(
+			url,
+			{
+				headers: Object.assign({
+					'User-Agent': this.userAgent,
+					Accept: accept,
+				}, headers ?? {}),
+			}
+		);
+
+		return await res.text();
+	}
+}
 
 @Injectable()
 export class HttpRequestService {
-	/**
-	 * Get http non-proxy agent
-	 */
+	public defaultFetcher: UndiciFetcher;
+	public fetch: UndiciFetcher['fetch'];
+	public getHtml: UndiciFetcher['getHtml'];
+	public defaultJsonFetcher: UndiciFetcher;
+	public getJson: UndiciFetcher['getJson'];
+
+	//#region for old http/https, only used in S3Service
+	// http non-proxy agent
 	private http: http.Agent;
 
-	/**
-	 * Get https non-proxy agent
-	 */
+	// https non-proxy agent
 	private https: https.Agent;
 
-	/**
-	 * Get http proxy or non-proxy agent
-	 */
+	// http proxy or non-proxy agent
 	public httpAgent: http.Agent;
 
-	/**
-	 * Get https proxy or non-proxy agent
-	 */
+	// https proxy or non-proxy agent
 	public httpsAgent: https.Agent;
+	//#endregion
+
+	public readonly dnsCache: CacheableLookup;
+	public readonly clientDefaults: undici.Agent.Options;
+	private maxSockets: number;
+
+	private logger: Logger;
 
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
+		private loggerService: LoggerService,
 	) {
-		const cache = new CacheableLookup({
+		this.logger = this.loggerService.getLogger('http-request');
+
+		this.dnsCache = new CacheableLookup({
 			maxTtl: 3600,	// 1hours
 			errorTtl: 30,	// 30secs
 			lookup: false,	// nativeのdns.lookupにfallbackしない
 		});
-		
+
+		this.clientDefaults = {
+			keepAliveTimeout: 30 * 1000,
+			keepAliveMaxTimeout: 10 * 60 * 1000,
+			keepAliveTimeoutThreshold: 1 * 1000,
+			strictContentLength: true,
+			headersTimeout: 10 * 1000,
+			bodyTimeout: 10 * 1000,
+			maxHeaderSize: 16364, // default
+			maxResponseSize: 10 * 1024 * 1024,
+			maxRedirections: 3,
+			connect: {
+				timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
+				maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
+				lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
+			},
+		}
+
+		this.maxSockets = Math.max(256, this.config.deliverJobConcurrency ?? 128);
+
+		this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
+
+		this.fetch = this.defaultFetcher.fetch;
+		this.getHtml = this.defaultFetcher.getHtml;
+
+		this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
+			maxResponseSize: 1024 * 256,
+		}), this.logger);
+
+		this.getJson = this.defaultJsonFetcher.getJson;
+
+		//#region for old http/https, only used in S3Service
 		this.http = new http.Agent({
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
-			lookup: cache.lookup,
+			lookup: this.dnsCache.lookup,
 		} as http.AgentOptions);
 		
 		this.https = new https.Agent({
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
-			lookup: cache.lookup,
+			lookup: this.dnsCache.lookup,
 		} as https.AgentOptions);
-		
-		const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
-		
+
 		this.httpAgent = config.proxy
 			? new HttpProxyAgent({
 				keepAlive: true,
 				keepAliveMsecs: 30 * 1000,
-				maxSockets,
+				maxSockets: this.maxSockets,
 				maxFreeSockets: 256,
 				scheduling: 'lifo',
 				proxy: config.proxy,
@@ -72,21 +260,46 @@ export class HttpRequestService {
 			? new HttpsProxyAgent({
 				keepAlive: true,
 				keepAliveMsecs: 30 * 1000,
-				maxSockets,
+				maxSockets: this.maxSockets,
 				maxFreeSockets: 256,
 				scheduling: 'lifo',
 				proxy: config.proxy,
 			})
 			: this.https;
+		//#endregion
 	}
 
 	/**
-	 * Get agent by URL
+	 * Get http agent by URL
 	 * @param url URL
 	 * @param bypassProxy Allways bypass proxy
 	 */
 	@bindThis
-	public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
+	public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
+		return {
+			agentOptions: {
+				...this.clientDefaults,
+				...opts,
+			},
+			...(this.config.proxy ? {
+			proxy: {
+				uri: this.config.proxy,
+				options: {
+					connections: this.maxSockets,
+					...proxyOpts,
+				}
+			}
+			} : {}),
+		}
+	}
+
+	/**
+	 * Get http agent by URL
+	 * @param url URL
+	 * @param bypassProxy Allways bypass proxy
+	 */
+	@bindThis
+	public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
 		if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
 			return url.protocol === 'http:' ? this.http : this.https;
 		} else {
@@ -94,67 +307,37 @@ export class HttpRequestService {
 		}
 	}
 
+	/**
+	 * check ip
+	 */
 	@bindThis
-	public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<unknown> {
-		const res = await this.getResponse({
-			url,
-			method: 'GET',
-			headers: Object.assign({
-				'User-Agent': this.config.userAgent,
-				Accept: accept,
-			}, headers ?? {}),
-			timeout,
-			size: 1024 * 256,
-		});
+	public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
+		return (options, cb) => {
+			connector(options, (err, socket) => {
+				this.logger.debug('Socket connector (with ip checker) called', socket);
+				if (err) {
+					this.logger.error(`Socket error`, err)
+					cb(new Error(`Error while socket connecting\n${err}`), null);
+					return;
+				}
 
-		return await res.json();
-	}
+				if (socket.remoteAddress == undefined) {
+					this.logger.error(`Socket error: remoteAddress is undefined`);
+					cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
+					return;
+				}
 
-	@bindThis
-	public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>): Promise<string> {
-		const res = await this.getResponse({
-			url,
-			method: 'GET',
-			headers: Object.assign({
-				'User-Agent': this.config.userAgent,
-				Accept: accept,
-			}, headers ?? {}),
-			timeout,
-		});
+				// allow
+				if (checkIp(socket.remoteAddress)) {
+					this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
+					cb(null, socket);
+					return;
+				}
 
-		return await res.text();
-	}
-
-	@bindThis
-	public async getResponse(args: {
-		url: string,
-		method: string,
-		body?: string,
-		headers: Record<string, string>,
-		timeout?: number,
-		size?: number,
-	}): Promise<Response> {
-		const timeout = args.timeout ?? 10 * 1000;
-
-		const controller = new AbortController();
-		setTimeout(() => {
-			controller.abort();
-		}, timeout * 6);
-
-		const res = await fetch(args.url, {
-			method: args.method,
-			headers: args.headers,
-			body: args.body,
-			timeout,
-			size: args.size ?? 10 * 1024 * 1024,
-			agent: (url) => this.getAgentByUrl(url),
-			signal: controller.signal,
-		});
-
-		if (!res.ok) {
-			throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
-		}
-
-		return res;
+				this.logger.error('IP is not allowed', socket);
+				cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
+				socket.destroy();
+			});
+		};
 	}
 }

packages/backend/src/core/NoteCreateService.ts

@@ -428,7 +428,7 @@ export class NoteCreateService {
 
 		// Register host
 		if (this.userEntityService.isRemoteUser(user)) {
-			this.federatedInstanceService.registerOrFetchInstanceDoc(user.host).then(i => {
+			this.federatedInstanceService.fetch(user.host).then(i => {
 				this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
 				this.instanceChart.updateNote(i.host, note, true);
 			});

packages/backend/src/core/NoteDeleteService.ts

@@ -100,7 +100,7 @@ export class NoteDeleteService {
 			this.perUserNotesChart.update(user, note, false);
 
 			if (this.userEntityService.isRemoteUser(user)) {
-				this.federatedInstanceService.registerOrFetchInstanceDoc(user.host).then(i => {
+				this.federatedInstanceService.fetch(user.host).then(i => {
 					this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1);
 					this.instanceChart.updateNote(i.host, note, false);
 				});

packages/backend/src/core/S3Service.ts

@@ -33,7 +33,7 @@ export class S3Service {
 				? false
 				: meta.objectStorageS3ForcePathStyle,
 			httpOptions: {
-				agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
+				agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
 			},
 		});
 	}

packages/backend/src/core/UserFollowingService.ts

@@ -205,12 +205,12 @@ export class UserFollowingService {
 	
 		//#region Update instance stats
 		if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) {
-			this.federatedInstanceService.registerOrFetchInstanceDoc(follower.host).then(i => {
+			this.federatedInstanceService.fetch(follower.host).then(i => {
 				this.instancesRepository.increment({ id: i.id }, 'followingCount', 1);
 				this.instanceChart.updateFollowing(i.host, true);
 			});
 		} else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) {
-			this.federatedInstanceService.registerOrFetchInstanceDoc(followee.host).then(i => {
+			this.federatedInstanceService.fetch(followee.host).then(i => {
 				this.instancesRepository.increment({ id: i.id }, 'followersCount', 1);
 				this.instanceChart.updateFollowers(i.host, true);
 			});
@@ -323,12 +323,12 @@ export class UserFollowingService {
 	
 		//#region Update instance stats
 		if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) {
-			this.federatedInstanceService.registerOrFetchInstanceDoc(follower.host).then(i => {
+			this.federatedInstanceService.fetch(follower.host).then(i => {
 				this.instancesRepository.decrement({ id: i.id }, 'followingCount', 1);
 				this.instanceChart.updateFollowing(i.host, false);
 			});
 		} else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) {
-			this.federatedInstanceService.registerOrFetchInstanceDoc(followee.host).then(i => {
+			this.federatedInstanceService.fetch(followee.host).then(i => {
 				this.instancesRepository.decrement({ id: i.id }, 'followersCount', 1);
 				this.instanceChart.updateFollowers(i.host, false);
 			});

packages/backend/src/core/WebfingerService.ts

@@ -30,7 +30,7 @@ export class WebfingerService {
 	public async webfinger(query: string): Promise<IWebFinger> {
 		const url = this.genUrl(query);
 
-		return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger;
+		return await this.httpRequestService.getJson<IWebFinger>(url, 'application/jrd+json, application/json');
 	}
 
 	@bindThis

packages/backend/src/core/activitypub/ApRequestService.ts

@@ -5,7 +5,7 @@ import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import type { User } from '@/models/entities/User.js';
 import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
 import { bindThis } from '@/decorators.js';
 
 type Request = {
@@ -28,6 +28,8 @@ type PrivateKey = {
 
 @Injectable()
 export class ApRequestService {
+	private undiciFetcher: UndiciFetcher;
+
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
@@ -35,6 +37,9 @@ export class ApRequestService {
 		private userKeypairStoreService: UserKeypairStoreService,
 		private httpRequestService: HttpRequestService,
 	) {
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
+			maxRedirections: 0,
+		}));
 	}
 
 	@bindThis
@@ -152,12 +157,14 @@ export class ApRequestService {
 			},
 		});
 
-		await this.httpRequestService.getResponse({
+		await this.undiciFetcher.fetch(
 			url,
-			method: req.request.method,
-			headers: req.request.headers,
-			body,
-		});
+			{
+				method: req.request.method,
+				headers: req.request.headers,
+				body,
+			}
+		);
 	}
 
 	/**
@@ -180,11 +187,13 @@ export class ApRequestService {
 			},
 		});
 
-		const res = await this.httpRequestService.getResponse({
+		const res = await this.httpRequestService.fetch(
 			url,
-			method: req.request.method,
-			headers: req.request.headers,
-		});
+			{
+				method: req.request.method,
+				headers: req.request.headers,
+			}
+		);
 
 		return await res.json();
 	}

packages/backend/src/core/activitypub/ApResolverService.ts

@@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js';
 import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
 import type { Config } from '@/config.js';
 import { MetaService } from '@/core/MetaService.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
 import { DI } from '@/di-symbols.js';
 import { UtilityService } from '@/core/UtilityService.js';
 import { bindThis } from '@/decorators.js';
@@ -17,6 +17,7 @@ import type { IObject, ICollection, IOrderedCollection } from './type.js';
 export class Resolver {
 	private history: Set<string>;
 	private user?: ILocalUser;
+	private undiciFetcher: UndiciFetcher;
 
 	constructor(
 		private config: Config,
@@ -34,6 +35,9 @@ export class Resolver {
 		private recursionLimit = 100,
 	) {
 		this.history = new Set();
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
+			maxRedirections: 0,
+		}));
 	}
 
 	@bindThis
@@ -96,8 +100,8 @@ export class Resolver {
 		}
 
 		const object = (this.user
-			? await this.apRequestService.signedGet(value, this.user)
-			: await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject;
+			? await this.apRequestService.signedGet(value, this.user) as IObject
+			: await this.undiciFetcher.getJson<IObject>(value, 'application/activity+json, application/ld+json'));
 
 		if (object == null || (
 			Array.isArray(object['@context']) ?

packages/backend/src/core/activitypub/LdSignatureService.ts

@@ -1,6 +1,5 @@
 import * as crypto from 'node:crypto';
 import { Inject, Injectable } from '@nestjs/common';
-import fetch from 'node-fetch';
 import { HttpRequestService } from '@/core/HttpRequestService.js';
 import { bindThis } from '@/decorators.js';
 import { CONTEXTS } from './misc/contexts.js';
@@ -116,14 +115,19 @@ class LdSignature {
 
 	@bindThis
 	private async fetchDocument(url: string) {
-		const json = await fetch(url, {
-			headers: {
-				Accept: 'application/ld+json, application/json',
+		const json = await this.httpRequestService.fetch(
+			url,
+			{
+				headers: {
+					Accept: 'application/ld+json, application/json',
+				},
+				// TODO
+				//timeout: this.loderTimeout,
 			},
-			// TODO
-			//timeout: this.loderTimeout,
-			agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent,
-		}).then(res => {
+			{
+				noOkError: true,
+			}
+		).then(res => {
 			if (!res.ok) {
 				throw `${res.status} ${res.statusText}`;
 			} else {

packages/backend/src/core/activitypub/models/ApPersonService.ts

@@ -348,7 +348,7 @@ export class ApPersonService implements OnModuleInit {
 		}
 
 		// Register host
-		this.federatedInstanceService.registerOrFetchInstanceDoc(host).then(i => {
+		this.federatedInstanceService.fetch(host).then(i => {
 			this.instancesRepository.increment({ id: i.id }, 'usersCount', 1);
 			this.instanceChart.newUser(i.host);
 			this.fetchInstanceMetadataService.fetchInstanceMetadata(i);

packages/backend/src/core/chart/charts/federation.ts

@@ -86,7 +86,7 @@ export default class FederationChart extends Chart<typeof schema> {
 				.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
 				.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts })
 				.andWhere('instance.isSuspended = false')
-				.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) })
+				.andWhere('instance.isNotResponding = false')
 				.getRawOne()
 				.then(x => parseInt(x.count, 10)),
 			this.instancesRepository.createQueryBuilder('instance')
@@ -94,7 +94,7 @@ export default class FederationChart extends Chart<typeof schema> {
 				.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
 				.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts })
 				.andWhere('instance.isSuspended = false')
-				.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) })
+				.andWhere('instance.isNotResponding = false')
 				.getRawOne()
 				.then(x => parseInt(x.count, 10)),
 		]);

packages/backend/src/core/entities/InstanceEntityService.ts

@@ -7,8 +7,8 @@ import type { } from '@/models/entities/Blocking.js';
 import type { User } from '@/models/entities/User.js';
 import type { Instance } from '@/models/entities/Instance.js';
 import { MetaService } from '@/core/MetaService.js';
-import { UserEntityService } from './UserEntityService.js';
 import { bindThis } from '@/decorators.js';
+import { UserEntityService } from './UserEntityService.js';
 
 @Injectable()
 export class InstanceEntityService {
@@ -33,8 +33,6 @@ export class InstanceEntityService {
 			notesCount: instance.notesCount,
 			followingCount: instance.followingCount,
 			followersCount: instance.followersCount,
-			latestRequestSentAt: instance.latestRequestSentAt ? instance.latestRequestSentAt.toISOString() : null,
-			lastCommunicatedAt: instance.lastCommunicatedAt.toISOString(),
 			isNotResponding: instance.isNotResponding,
 			isSuspended: instance.isSuspended,
 			isBlocked: meta.blockedHosts.includes(instance.host),

packages/backend/src/models/entities/Instance.ts

@@ -59,22 +59,6 @@ export class Instance {
 	})
 	public followersCount: number;
 
-	/**
-	 * 直近のリクエスト送信日時
-	 */
-	@Column('timestamp with time zone', {
-		nullable: true,
-	})
-	public latestRequestSentAt: Date | null;
-
-	/**
-	 * 直近のリクエスト送信時のHTTPステータスコード
-	 */
-	@Column('integer', {
-		nullable: true,
-	})
-	public latestStatus: number | null;
-
 	/**
 	 * 直近のリクエスト受信日時
 	 */
@@ -83,12 +67,6 @@ export class Instance {
 	})
 	public latestRequestReceivedAt: Date | null;
 
-	/**
-	 * このインスタンスと最後にやり取りした日時
-	 */
-	@Column('timestamp with time zone')
-	public lastCommunicatedAt: Date;
-
 	/**
 	 * このインスタンスと不通かどうか
 	 */

packages/backend/src/models/schema/federation-instance.ts

@@ -32,16 +32,6 @@ export const packedFederationInstanceSchema = {
 			type: 'number',
 			optional: false, nullable: false,
 		},
-		latestRequestSentAt: {
-			type: 'string',
-			optional: false, nullable: true,
-			format: 'date-time',
-		},
-		lastCommunicatedAt: {
-			type: 'string',
-			optional: false, nullable: false,
-			format: 'date-time',
-		},
 		isNotResponding: {
 			type: 'boolean',
 			optional: false, nullable: false,

packages/backend/src/queue/processors/DeliverProcessorService.ts

@@ -15,10 +15,10 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js';
 import FederationChart from '@/core/chart/charts/federation.js';
 import { StatusError } from '@/misc/status-error.js';
 import { UtilityService } from '@/core/UtilityService.js';
+import { bindThis } from '@/decorators.js';
 import { QueueLoggerService } from '../QueueLoggerService.js';
 import type Bull from 'bull';
 import type { DeliverJobData } from '../types.js';
-import { bindThis } from '@/decorators.js';
 
 @Injectable()
 export class DeliverProcessorService {
@@ -48,7 +48,6 @@ export class DeliverProcessorService {
 	) {
 		this.logger = this.queueLoggerService.logger.createSubLogger('deliver');
 		this.suspendedHostsCache = new Cache<Instance[]>(1000 * 60 * 60);
-		this.latest = null;
 	}
 
 	@bindThis
@@ -76,20 +75,18 @@ export class DeliverProcessorService {
 		}
 
 		try {
-			if (this.latest !== (this.latest = JSON.stringify(job.data.content, null, 2))) {
-				this.logger.debug(`delivering ${this.latest}`);
-			}
-
 			await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content);
 
 			// Update stats
-			this.federatedInstanceService.registerOrFetchInstanceDoc(host).then(i => {
-				this.instancesRepository.update(i.id, {
-					latestRequestSentAt: new Date(),
-					latestStatus: 200,
-					lastCommunicatedAt: new Date(),
-					isNotResponding: false,
-				});
+			this.federatedInstanceService.fetch(host).then(i => {
+				if (i.isNotResponding) {
+					this.instancesRepository.update(i.id, {
+						isNotResponding: false,
+					});
+					this.federatedInstanceService.updateCachePartial(host, {
+						isNotResponding: false,
+					});
+				}
 
 				this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
 
@@ -100,13 +97,16 @@ export class DeliverProcessorService {
 
 			return 'Success';
 		} catch (res) {
-		// Update stats
-			this.federatedInstanceService.registerOrFetchInstanceDoc(host).then(i => {
-				this.instancesRepository.update(i.id, {
-					latestRequestSentAt: new Date(),
-					latestStatus: res instanceof StatusError ? res.statusCode : null,
-					isNotResponding: true,
-				});
+			// Update stats
+			this.federatedInstanceService.fetch(host).then(i => {
+				if (!i.isNotResponding) {
+					this.instancesRepository.update(i.id, {
+						isNotResponding: true,
+					});
+					this.federatedInstanceService.updateCachePartial(host, {
+						isNotResponding: true,
+					});
+				}
 
 				this.instanceChart.requestSent(i.host, false);
 				this.apRequestChart.deliverFail();
@@ -114,17 +114,17 @@ export class DeliverProcessorService {
 			});
 
 			if (res instanceof StatusError) {
-			// 4xx
+				// 4xx
 				if (res.isClientError) {
-				// HTTPステータスコード4xxはクライアントエラーであり、それはつまり
-				// 何回再送しても成功することはないということなのでエラーにはしないでおく
+					// HTTPステータスコード4xxはクライアントエラーであり、それはつまり
+					// 何回再送しても成功することはないということなのでエラーにはしないでおく
 					return `${res.statusCode} ${res.statusMessage}`;
 				}
 
 				// 5xx etc.
 				throw `${res.statusCode} ${res.statusMessage}`;
 			} else {
-			// DNS error, socket error, timeout ...
+				// DNS error, socket error, timeout ...
 				throw res;
 			}
 		}

packages/backend/src/queue/processors/InboxProcessorService.ts

@@ -176,10 +176,12 @@ export class InboxProcessorService {
 		}
 
 		// Update stats
-		this.federatedInstanceService.registerOrFetchInstanceDoc(authUser.user.host).then(i => {
+		this.federatedInstanceService.fetch(authUser.user.host).then(i => {
 			this.instancesRepository.update(i.id, {
 				latestRequestReceivedAt: new Date(),
-				lastCommunicatedAt: new Date(),
+				isNotResponding: false,
+			});
+			this.federatedInstanceService.updateCachePartial(host, {
 				isNotResponding: false,
 			});
 

packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts

@@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService {
 		try {
 			this.logger.debug(`delivering ${job.data.webhookId}`);
 	
-			const res = await this.httpRequestService.getResponse({
-				url: job.data.to,
-				method: 'POST',
-				headers: {
-					'User-Agent': 'Misskey-Hooks',
-					'X-Misskey-Host': this.config.host,
-					'X-Misskey-Hook-Id': job.data.webhookId,
-					'X-Misskey-Hook-Secret': job.data.secret,
-				},
-				body: JSON.stringify({
-					hookId: job.data.webhookId,
-					userId: job.data.userId,
-					eventId: job.data.eventId,
-					createdAt: job.data.createdAt,
-					type: job.data.type,
-					body: job.data.content,
-				}),
-			});
+			const res = await this.httpRequestService.fetch(
+				job.data.to,
+				{
+					method: 'POST',
+					headers: {
+						'User-Agent': 'Misskey-Hooks',
+						'X-Misskey-Host': this.config.host,
+						'X-Misskey-Hook-Id': job.data.webhookId,
+						'X-Misskey-Hook-Secret': job.data.secret,
+					},
+					body: JSON.stringify({
+						hookId: job.data.webhookId,
+						userId: job.data.userId,
+						eventId: job.data.eventId,
+						createdAt: job.data.createdAt,
+						type: job.data.type,
+						body: job.data.content,
+					}),
+				}
+			);
 	
 			this.webhooksRepository.update({ id: job.data.webhookId }, {
 				latestSentAt: new Date(),

packages/backend/src/server/api/endpoints/federation/instances.ts

@@ -64,8 +64,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 				case '-followers': query.orderBy('instance.followersCount', 'ASC'); break;
 				case '+caughtAt': query.orderBy('instance.caughtAt', 'DESC'); break;
 				case '-caughtAt': query.orderBy('instance.caughtAt', 'ASC'); break;
-				case '+lastCommunicatedAt': query.orderBy('instance.lastCommunicatedAt', 'DESC'); break;
-				case '-lastCommunicatedAt': query.orderBy('instance.lastCommunicatedAt', 'ASC'); break;
+				case '+latestRequestReceivedAt': query.orderBy('instance.latestRequestReceivedAt', 'DESC'); break;
+				case '-latestRequestReceivedAt': query.orderBy('instance.latestRequestReceivedAt', 'ASC'); break;
 
 				default: query.orderBy('instance.id', 'DESC'); break;
 			}

packages/backend/src/server/api/endpoints/fetch-rss.ts

@@ -33,15 +33,17 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 		private httpRequestService: HttpRequestService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			const res = await this.httpRequestService.getResponse({
-				url: ps.url,
-				method: 'GET',
-				headers: Object.assign({
-					'User-Agent': config.userAgent,
-					Accept: 'application/rss+xml, */*',
-				}),
-				timeout: 5000,
-			});
+			const res = await this.httpRequestService.fetch(
+				ps.url,
+				{
+					method: 'GET',
+					headers: Object.assign({
+						'User-Agent': config.userAgent,
+						Accept: 'application/rss+xml, */*',
+					}),
+					// timeout: 5000,
+				}
+			);
 
 			const text = await res.text();
 

packages/backend/src/server/api/endpoints/notes/translate.ts

@@ -1,5 +1,4 @@
 import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
 import { Inject, Injectable } from '@nestjs/common';
 import type { NotesRepository } from '@/models/index.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
@@ -84,25 +83,28 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 
 			const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
 
-			const res = await fetch(endpoint, {
-				method: 'POST',
-				headers: {
-					'Content-Type': 'application/x-www-form-urlencoded',
-					'User-Agent': config.userAgent,
-					Accept: 'application/json, */*',
+			const res = await this.httpRequestService.fetch(
+				endpoint,
+				{
+					method: 'POST',
+					headers: {
+						'Content-Type': 'application/x-www-form-urlencoded',
+						'User-Agent': config.userAgent,
+						Accept: 'application/json, */*',
+					},
+					body: params.toString(),
 				},
-				body: params,
-				// TODO
-				//timeout: 10000,
-				agent: (url) => this.httpRequestService.getAgentByUrl(url),
-			});
+				{
+					noOkError: false,
+				}
+			);
 
 			const json = (await res.json()) as {
-		translations: {
-			detected_source_language: string;
-			text: string;
-		}[];
-	};
+				translations: {
+					detected_source_language: string;
+					text: string;
+				}[];
+			};
 
 			return {
 				sourceLang: json.translations[0].detected_source_language,

packages/backend/src/server/api/endpoints/users/show.ts

@@ -139,10 +139,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 					throw new ApiError(meta.errors.noSuchUser);
 				}
 
-				if (me == null && ip != null) {
-					this.perUserPvChart.commitByVisitor(user, ip);
-				} else if (me && me.id !== user.id) {
-					this.perUserPvChart.commitByUser(user, me.id);
+				if (user.host == null) {
+					if (me == null && ip != null) {
+						this.perUserPvChart.commitByVisitor(user, ip);
+					} else if (me && me.id !== user.id) {
+						this.perUserPvChart.commitByUser(user, me.id);
+					}
 				}
 
 				return await this.userEntityService.pack(user, me, {

packages/backend/src/server/api/integration/DiscordServerService.ts

@@ -181,7 +181,7 @@ export class DiscordServerService {
 						}
 					}));
 
-				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
+				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
 					'Authorization': `Bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 
@@ -249,7 +249,7 @@ export class DiscordServerService {
 						}
 					}));
 
-				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
+				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
 					'Authorization': `Bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 				if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') {

packages/backend/src/server/api/integration/GithubServerService.ts

@@ -174,7 +174,7 @@ export class GithubServerService {
 						}
 					}));
 
-				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
 					'Authorization': `bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 				if (typeof login !== 'string' || typeof id !== 'string') {
@@ -223,7 +223,7 @@ export class GithubServerService {
 							}
 						}));
 
-				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
 					'Authorization': `bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 

packages/backend/src/server/web/UrlPreviewService.ts

@@ -63,9 +63,8 @@ export class UrlPreviewService {
 		this.logger.info(meta.summalyProxy
 			? `(Proxy) Getting preview of ${url}@${lang} ...`
 			: `Getting preview of ${url}@${lang} ...`);
-	
 		try {
-			const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({
+			const summary = meta.summalyProxy ? await this.httpRequestService.getJson<ReturnType<typeof summaly.default>>(`${meta.summalyProxy}?${query({
 				url: url,
 				lang: lang ?? 'ja-JP',
 			})}`) : await summaly.default(url, {