Merge branch 'develop' into re-ed25519

This commit is contained in:
tamaina 2024-08-18 19:21:31 +09:00
commit d01cd153d1
38 changed files with 611 additions and 157 deletions

View file

@ -29,7 +29,7 @@ export class AvatarDecorationService implements OnApplicationShutdown {
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
) {
this.cache = new MemorySingleCache<MiAvatarDecoration[]>(1000 * 60 * 30);
this.cache = new MemorySingleCache<MiAvatarDecoration[]>(1000 * 60 * 30); // 30s
this.redisForSub.on('message', this.onMessage);
}

View file

@ -56,10 +56,10 @@ export class CacheService implements OnApplicationShutdown {
) {
//this.onMessage = this.onMessage.bind(this);
this.userByIdCache = new MemoryKVCache<MiUser>(Infinity);
this.localUserByNativeTokenCache = new MemoryKVCache<MiLocalUser | null>(Infinity);
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(Infinity);
this.uriPersonCache = new MemoryKVCache<MiUser | null>(Infinity);
this.userByIdCache = new MemoryKVCache<MiUser>(1000 * 60 * 5); // 5m
this.localUserByNativeTokenCache = new MemoryKVCache<MiLocalUser | null>(1000 * 60 * 5); // 5m
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 5); // 5m
this.uriPersonCache = new MemoryKVCache<MiUser | null>(1000 * 60 * 5); // 5m
this.userProfileCache = new RedisKVCache<MiUserProfile>(this.redisClient, 'userProfile', {
lifetime: 1000 * 60 * 30, // 30m
@ -135,14 +135,14 @@ export class CacheService implements OnApplicationShutdown {
if (user == null) {
this.userByIdCache.delete(body.id);
this.localUserByIdCache.delete(body.id);
for (const [k, v] of this.uriPersonCache.cache.entries()) {
for (const [k, v] of this.uriPersonCache.entries) {
if (v.value?.id === body.id) {
this.uriPersonCache.delete(k);
}
}
} else {
this.userByIdCache.set(user.id, user);
for (const [k, v] of this.uriPersonCache.cache.entries()) {
for (const [k, v] of this.uriPersonCache.entries) {
if (v.value?.id === user.id) {
this.uriPersonCache.set(k, user);
}

View file

@ -24,7 +24,7 @@ const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/;
@Injectable()
export class CustomEmojiService implements OnApplicationShutdown {
private cache: MemoryKVCache<MiEmoji | null>;
private emojisCache: MemoryKVCache<MiEmoji | null>;
public localEmojisCache: RedisSingleCache<Map<string, MiEmoji>>;
constructor(
@ -40,7 +40,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
) {
this.cache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12);
this.emojisCache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12); // 12h
this.localEmojisCache = new RedisSingleCache<Map<string, MiEmoji>>(this.redisClient, 'localEmojis', {
lifetime: 1000 * 60 * 30, // 30m
@ -334,7 +334,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
host,
})) ?? null;
const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull);
const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull);
if (emoji == null) return null;
return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のためpublicUrlはstringなので??はだめ)
@ -361,7 +361,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
*/
@bindThis
public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> {
const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null);
const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null);
const emojisQuery: any[] = [];
const hosts = new Set(notCachedEmojis.map(e => e.host));
for (const host of hosts) {
@ -376,7 +376,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
select: ['name', 'host', 'originalUrl', 'publicUrl'],
}) : [];
for (const emoji of _emojis) {
this.cache.set(`${emoji.name} ${emoji.host}`, emoji);
this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji);
}
}
@ -401,7 +401,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
@bindThis
public dispose(): void {
this.cache.dispose();
this.emojisCache.dispose();
}
@bindThis

View file

@ -38,7 +38,7 @@ export class RelayService {
private apRendererService: ApRendererService,
private userKeypairService: UserKeypairService,
) {
this.relaysCache = new MemorySingleCache<MiRelay[]>(1000 * 60 * 10);
this.relaysCache = new MemorySingleCache<MiRelay[]>(1000 * 60 * 10); // 10m
}
@bindThis

View file

@ -127,10 +127,8 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
private moderationLogService: ModerationLogService,
private fanoutTimelineService: FanoutTimelineService,
) {
//this.onMessage = this.onMessage.bind(this);
this.rolesCache = new MemorySingleCache<MiRole[]>(1000 * 60 * 60 * 1);
this.roleAssignmentByUserIdCache = new MemoryKVCache<MiRoleAssignment[]>(1000 * 60 * 60 * 1);
this.rolesCache = new MemorySingleCache<MiRole[]>(1000 * 60 * 60); // 1h
this.roleAssignmentByUserIdCache = new MemoryKVCache<MiRoleAssignment[]>(1000 * 60 * 5); // 5m
this.redisForSub.on('message', this.onMessage);
}

View file

@ -34,7 +34,7 @@ export class UserKeypairService implements OnApplicationShutdown {
) {
this.keypairEntityCache = new RedisKVCache<MiUserKeypair>(this.redisClient, 'userKeypair', {
lifetime: 1000 * 60 * 60 * 24, // 24h
memoryCacheLifetime: Infinity,
memoryCacheLifetime: 1000 * 60 * 60, // 1h
fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),

View file

@ -64,7 +64,7 @@ export class ApDbResolverService implements OnApplicationShutdown {
private apLoggerService: ApLoggerService,
private utilityService: UtilityService,
) {
this.publicKeyByUserIdCache = new MemoryKVCache<MiUserPublickey[] | null>(Infinity);
this.publicKeyByUserIdCache = new MemoryKVCache<MiUserPublickey[] | null>(1000 * 60 * 60 * 12); // 12h
this.logger = this.apLoggerService.logger.createSubLogger('db-resolver');
this.redisForSub.on('message', this.onMessage);
}

View file

@ -6,6 +6,7 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { genRFC3230DigestHeader, signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures';
import { Window } from 'happy-dom';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { MiUser } from '@/models/User.js';
@ -126,7 +127,9 @@ export class ApRequestService {
* @param url URL to fetch
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise<unknown> {
public async signedGet(url: string, user: { id: MiUser['id'] }, level: string, followAlternate?: boolean): Promise<unknown> {
const _followAlternate = followAlternate ?? true;
const key = await this.userKeypairService.getLocalUserPrivateKey(user.id, level);
const req = await createSignedGet({
level,
@ -152,9 +155,29 @@ export class ApRequestService {
headers: req.request.headers,
}, {
throwErrorWhenResponseNotOk: true,
validators: [validateContentTypeSetAsActivityPub],
});
//#region リクエスト先がhtmlかつactivity+jsonへのalternate linkタグがあるとき
const contentType = res.headers.get('content-type');
if ((contentType ?? '').split(';')[0].trimEnd().toLowerCase() === 'text/html' && _followAlternate === true) {
const html = await res.text();
const window = new Window();
const document = window.document;
document.documentElement.innerHTML = html;
const alternate = document.querySelector('head > link[rel="alternate"][type="application/activity+json"]');
if (alternate) {
const href = alternate.getAttribute('href');
if (href) {
return await this.signedGet(href, user, level, false);
}
}
}
//#endregion
validateContentTypeSetAsActivityPub(res);
return await res.json();
}
}

View file

@ -7,23 +7,23 @@ import * as Redis from 'ioredis';
import { bindThis } from '@/decorators.js';
export class RedisKVCache<T> {
private redisClient: Redis.Redis;
private name: string;
private lifetime: number;
private memoryCache: MemoryKVCache<T>;
private fetcher: (key: string) => Promise<T>;
private toRedisConverter: (value: T) => string;
private fromRedisConverter: (value: string) => T | undefined;
private readonly lifetime: number;
private readonly memoryCache: MemoryKVCache<T>;
private readonly fetcher: (key: string) => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
constructor(redisClient: RedisKVCache<T>['redisClient'], name: RedisKVCache<T>['name'], opts: {
lifetime: RedisKVCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisKVCache<T>['fetcher'];
toRedisConverter: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter: RedisKVCache<T>['fromRedisConverter'];
}) {
this.redisClient = redisClient;
this.name = name;
constructor(
private redisClient: Redis.Redis,
private name: string,
opts: {
lifetime: RedisKVCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisKVCache<T>['fetcher'];
toRedisConverter: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter: RedisKVCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
@ -55,7 +55,13 @@ export class RedisKVCache<T> {
const cached = await this.redisClient.get(`kvcache:${this.name}:${key}`);
if (cached == null) return undefined;
return this.fromRedisConverter(cached);
const value = this.fromRedisConverter(cached);
if (value !== undefined) {
this.memoryCache.set(key, value);
}
return value;
}
@bindThis
@ -66,6 +72,10 @@ export class RedisKVCache<T> {
/**
* fetcherを呼び出して結果をキャッシュ&
* This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons:
* * Other code uses this to synchronize changes between worker processes. A failed write can internally de-sync the cluster.
* * Without an `await`, consecutive calls could race. An unlucky race could result in the older write overwriting the newer value.
* * Not awaiting here makes the entire cache non-consistent. The prevents many possible uses.
*/
@bindThis
public async fetch(key: string): Promise<T> {
@ -77,14 +87,14 @@ export class RedisKVCache<T> {
// Cache MISS
const value = await this.fetcher(key);
this.set(key, value);
await this.set(key, value);
return value;
}
@bindThis
public async refresh(key: string) {
const value = await this.fetcher(key);
this.set(key, value);
await this.set(key, value);
// TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする
}
@ -101,23 +111,23 @@ export class RedisKVCache<T> {
}
export class RedisSingleCache<T> {
private redisClient: Redis.Redis;
private name: string;
private lifetime: number;
private memoryCache: MemorySingleCache<T>;
private fetcher: () => Promise<T>;
private toRedisConverter: (value: T) => string;
private fromRedisConverter: (value: string) => T | undefined;
private readonly lifetime: number;
private readonly memoryCache: MemorySingleCache<T>;
private readonly fetcher: () => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
constructor(redisClient: RedisSingleCache<T>['redisClient'], name: RedisSingleCache<T>['name'], opts: {
lifetime: RedisSingleCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisSingleCache<T>['fetcher'];
toRedisConverter: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter: RedisSingleCache<T>['fromRedisConverter'];
}) {
this.redisClient = redisClient;
this.name = name;
constructor(
private redisClient: Redis.Redis,
private name: string,
opts: {
lifetime: number;
memoryCacheLifetime: number;
fetcher: RedisSingleCache<T>['fetcher'];
toRedisConverter: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter: RedisSingleCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
@ -149,7 +159,18 @@ export class RedisSingleCache<T> {
const cached = await this.redisClient.get(`singlecache:${this.name}`);
if (cached == null) return undefined;
return this.fromRedisConverter(cached);
const value = this.fromRedisConverter(cached);
if (value !== undefined) {
this.memoryCache.set(value);
}
return value;
}
@bindThis
public getRaw(): T | undefined {
return this.memoryCache.get();
}
@bindThis
@ -160,6 +181,10 @@ export class RedisSingleCache<T> {
/**
* fetcherを呼び出して結果をキャッシュ&
* This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons:
* * Other code uses this to synchronize changes between worker processes. A failed write can internally de-sync the cluster.
* * Without an `await`, consecutive calls could race. An unlucky race could result in the older write overwriting the newer value.
* * Not awaiting here makes the entire cache non-consistent. The prevents many possible uses.
*/
@bindThis
public async fetch(): Promise<T> {
@ -171,14 +196,14 @@ export class RedisSingleCache<T> {
// Cache MISS
const value = await this.fetcher();
this.set(value);
await this.set(value);
return value;
}
@bindThis
public async refresh() {
const value = await this.fetcher();
this.set(value);
await this.set(value);
// TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする
}
@ -187,25 +212,12 @@ export class RedisSingleCache<T> {
// TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする?
export class MemoryKVCache<T> {
/**
*
* @deprecated
*/
public cache: Map<string, { date: number; value: T; }>;
private lifetime: number;
private gcIntervalHandle: NodeJS.Timeout;
private readonly cache = new Map<string, { date: number; value: T; }>();
private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); // 3m
/**
* @param lifetime (ms)
*/
constructor(lifetime: MemoryKVCache<never>['lifetime']) {
this.cache = new Map();
this.lifetime = lifetime;
this.gcIntervalHandle = setInterval(() => {
this.gc();
}, 1000 * 60 * 3);
}
constructor(
private readonly lifetime: number,
) {}
@bindThis
/**
@ -290,10 +302,14 @@ export class MemoryKVCache<T> {
@bindThis
public gc(): void {
const now = Date.now();
for (const [key, { date }] of this.cache.entries()) {
if ((now - date) > this.lifetime) {
this.cache.delete(key);
}
// The map is ordered from oldest to youngest.
// We can stop once we find an entry that's still active, because all following entries must *also* be active.
const age = now - date;
if (age < this.lifetime) break;
this.cache.delete(key);
}
}
@ -301,16 +317,19 @@ export class MemoryKVCache<T> {
public dispose(): void {
clearInterval(this.gcIntervalHandle);
}
public get entries() {
return this.cache.entries();
}
}
export class MemorySingleCache<T> {
private cachedAt: number | null = null;
private value: T | undefined;
private lifetime: number;
constructor(lifetime: MemorySingleCache<never>['lifetime']) {
this.lifetime = lifetime;
}
constructor(
private lifetime: number,
) {}
@bindThis
public set(value: T): void {

View file

@ -45,7 +45,7 @@ export class DeliverProcessorService {
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('deliver');
this.suspendedHostsCache = new MemorySingleCache<MiInstance[]>(1000 * 60 * 60);
this.suspendedHostsCache = new MemorySingleCache<MiInstance[]>(1000 * 60 * 60); // 1h
}
@bindThis

View file

@ -141,7 +141,7 @@ export class NodeinfoServerService {
return document;
};
const cache = new MemorySingleCache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10);
const cache = new MemorySingleCache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10); // 10m
fastify.get(nodeinfo2_1path, async (request, reply) => {
const base = await cache.fetch(() => nodeinfo2(21));

View file

@ -37,7 +37,7 @@ export class AuthenticateService implements OnApplicationShutdown {
private cacheService: CacheService,
) {
this.appCache = new MemoryKVCache<MiApp>(Infinity);
this.appCache = new MemoryKVCache<MiApp>(1000 * 60 * 60 * 24 * 7); // 1w
}
@bindThis

View file

@ -4,9 +4,11 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import type { FlashsRepository } from '@/models/_.js';
import type { FlashsRepository, UsersRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -44,17 +46,35 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
constructor(
@Inject(DI.flashsRepository)
private flashsRepository: FlashsRepository,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private moderationLogService: ModerationLogService,
private roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
const flash = await this.flashsRepository.findOneBy({ id: ps.flashId });
if (flash == null) {
throw new ApiError(meta.errors.noSuchFlash);
}
if (flash.userId !== me.id) {
if (!await this.roleService.isModerator(me) && flash.userId !== me.id) {
throw new ApiError(meta.errors.accessDenied);
}
await this.flashsRepository.delete(flash.id);
if (flash.userId !== me.id) {
const user = await this.usersRepository.findOneByOrFail({ id: flash.userId });
this.moderationLogService.log(me, 'deleteFlash', {
flashId: flash.id,
flashUserId: flash.userId,
flashUserUsername: user.username,
flash,
});
}
});
}
}

View file

@ -5,8 +5,10 @@
import { Inject, Injectable } from '@nestjs/common';
import { Endpoint } from '@/server/api/endpoint-base.js';
import type { GalleryPostsRepository } from '@/models/_.js';
import type { GalleryPostsRepository, UsersRepository } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '../../../error.js';
export const meta = {
@ -22,6 +24,12 @@ export const meta = {
code: 'NO_SUCH_POST',
id: 'ae52f367-4bd7-4ecd-afc6-5672fff427f5',
},
accessDenied: {
message: 'Access denied.',
code: 'ACCESS_DENIED',
id: 'c86e09de-1c48-43ac-a435-1c7e42ed4496',
},
},
} as const;
@ -38,18 +46,35 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
constructor(
@Inject(DI.galleryPostsRepository)
private galleryPostsRepository: GalleryPostsRepository,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private moderationLogService: ModerationLogService,
private roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
const post = await this.galleryPostsRepository.findOneBy({
id: ps.postId,
userId: me.id,
});
const post = await this.galleryPostsRepository.findOneBy({ id: ps.postId });
if (post == null) {
throw new ApiError(meta.errors.noSuchPost);
}
if (!await this.roleService.isModerator(me) && post.userId !== me.id) {
throw new ApiError(meta.errors.accessDenied);
}
await this.galleryPostsRepository.delete(post.id);
if (post.userId !== me.id) {
const user = await this.usersRepository.findOneByOrFail({ id: post.userId });
this.moderationLogService.log(me, 'deleteGalleryPost', {
postId: post.id,
postUserId: post.userId,
postUserUsername: user.username,
post,
});
}
});
}
}

View file

@ -4,9 +4,11 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import type { PagesRepository } from '@/models/_.js';
import type { PagesRepository, UsersRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -44,17 +46,35 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
constructor(
@Inject(DI.pagesRepository)
private pagesRepository: PagesRepository,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private moderationLogService: ModerationLogService,
private roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
const page = await this.pagesRepository.findOneBy({ id: ps.pageId });
if (page == null) {
throw new ApiError(meta.errors.noSuchPage);
}
if (page.userId !== me.id) {
if (!await this.roleService.isModerator(me) && page.userId !== me.id) {
throw new ApiError(meta.errors.accessDenied);
}
await this.pagesRepository.delete(page.id);
if (page.userId !== me.id) {
const user = await this.usersRepository.findOneByOrFail({ id: page.userId });
this.moderationLogService.log(me, 'deletePage', {
pageId: page.id,
pageUserId: page.userId,
pageUserUsername: user.username,
page,
});
}
});
}
}

View file

@ -97,6 +97,9 @@ export const moderationLogTypes = [
'updateAbuseReportNotificationRecipient',
'deleteAbuseReportNotificationRecipient',
'deleteAccount',
'deletePage',
'deleteFlash',
'deleteGalleryPost',
] as const;
export type ModerationLogPayloads = {
@ -320,6 +323,24 @@ export type ModerationLogPayloads = {
userUsername: string;
userHost: string | null;
};
deletePage: {
pageId: string;
pageUserId: string;
pageUserUsername: string;
page: any;
};
deleteFlash: {
flashId: string;
flashUserId: string;
flashUserUsername: string;
flash: any;
};
deleteGalleryPost: {
postId: string;
postUserId: string;
postUserUsername: string;
post: any;
};
};
export type Serialized<T> = {