Mizah/Sharkey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge: upstream

Browse source
This commit is contained in:
Marie 2023-12-23 02:09:23 +01:00
commit 5db583a3eb
701 changed files with 50809 additions and 13660 deletions
Download patch file Download diff file Expand all files Collapse all files

26
packages/backend/src/core/AntennaService.ts
View file

@ -16,7 +16,7 @@ import type { AntennasRepository, UserListMembershipsRepository } from '@/models
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FunoutTimelineService } from '@/core/FunoutTimelineService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -39,7 +39,7 @@ export class AntennaService implements OnApplicationShutdown {
private utilityService: UtilityService,
private globalEventService: GlobalEventService,
private funoutTimelineService: FunoutTimelineService,
private fanoutTimelineService: FanoutTimelineService,
) {
this.antennasFetched = false;
this.antennas = [];
@ -60,11 +60,21 @@ export class AntennaService implements OnApplicationShutdown {
lastUsedAt: new Date(body.lastUsedAt),
});
break;
case 'antennaUpdated':
this.antennas[this.antennas.findIndex(a => a.id === body.id)] = {
...body,
lastUsedAt: new Date(body.lastUsedAt),
};
case 'antennaUpdated': {
const idx = this.antennas.findIndex(a => a.id === body.id);
if (idx >= 0) {
this.antennas[idx] = {
...body,
lastUsedAt: new Date(body.lastUsedAt),
};
} else {
// サーバ起動時にactiveじゃなかった場合、リストに持っていないので追加する必要あり
this.antennas.push({
...body,
lastUsedAt: new Date(body.lastUsedAt),
});
}
}
break;
case 'antennaDeleted':
this.antennas = this.antennas.filter(a => a.id !== body.id);
@ -84,7 +94,7 @@ export class AntennaService implements OnApplicationShutdown {
const redisPipeline = this.redisForTimelines.pipeline();
for (const antenna of matchedAntennas) {
this.funoutTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline);
this.fanoutTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline);
this.globalEventService.publishAntennaStream(antenna.id, 'note', note);
}

18
packages/backend/src/core/CoreModule.ts
View file

@ -4,6 +4,7 @@
*/
import { Module } from '@nestjs/common';
import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js';
import { AccountMoveService } from './AccountMoveService.js';
import { AccountUpdateService } from './AccountUpdateService.js';
import { AnnouncementService } from './AnnouncementService.js';
@ -62,7 +63,7 @@ import { FileInfoService } from './FileInfoService.js';
import { SearchService } from './SearchService.js';
import { ClipService } from './ClipService.js';
import { FeaturedService } from './FeaturedService.js';
import { FunoutTimelineService } from './FunoutTimelineService.js';
import { FanoutTimelineService } from './FanoutTimelineService.js';
import { ChannelFollowingService } from './ChannelFollowingService.js';
import { RegistryApiService } from './RegistryApiService.js';
import { ChartLoggerService } from './chart/ChartLoggerService.js';
@ -194,7 +195,8 @@ const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: Fi
const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService };
const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService };
const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService };
const $FunoutTimelineService: Provider = { provide: 'FunoutTimelineService', useExisting: FunoutTimelineService };
const $FanoutTimelineService: Provider = { provide: 'FanoutTimelineService', useExisting: FanoutTimelineService };
const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpointService', useExisting: FanoutTimelineEndpointService };
const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService };
const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService };
@ -330,7 +332,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
SearchService,
ClipService,
FeaturedService,
FunoutTimelineService,
FanoutTimelineService,
FanoutTimelineEndpointService,
ChannelFollowingService,
RegistryApiService,
ChartLoggerService,
@ -459,7 +462,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$SearchService,
$ClipService,
$FeaturedService,
$FunoutTimelineService,
$FanoutTimelineService,
$FanoutTimelineEndpointService,
$ChannelFollowingService,
$RegistryApiService,
$ChartLoggerService,
@ -589,7 +593,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
SearchService,
ClipService,
FeaturedService,
FunoutTimelineService,
FanoutTimelineService,
FanoutTimelineEndpointService,
ChannelFollowingService,
RegistryApiService,
FederationChart,
@ -717,7 +722,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$SearchService,
$ClipService,
$FeaturedService,
$FunoutTimelineService,
$FanoutTimelineService,
$FanoutTimelineEndpointService,
$ChannelFollowingService,
$RegistryApiService,
$FederationChart,

92
packages/backend/src/core/EmailService.ts
View file

@ -3,9 +3,11 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { URLSearchParams } from 'node:url';
import * as nodemailer from 'nodemailer';
import { Inject, Injectable } from '@nestjs/common';
import { validate as validateEmail } from 'deep-email-validator';
import { SubOutputFormat } from 'deep-email-validator/dist/output/output.js';
import { MetaService } from '@/core/MetaService.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
@ -13,6 +15,7 @@ import type Logger from '@/logger.js';
import type { UserProfilesRepository } from '@/models/_.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
@Injectable()
export class EmailService {
@ -27,6 +30,7 @@ export class EmailService {
private metaService: MetaService,
private loggerService: LoggerService,
private httpRequestService: HttpRequestService,
) {
this.logger = this.loggerService.getLogger('email');
}
@ -160,14 +164,25 @@ export class EmailService {
email: emailAddress,
});
const validated = meta.enableActiveEmailValidation ? await validateEmail({
email: emailAddress,
validateRegex: true,
validateMx: true,
validateTypo: false, // TLDを見ているみたいだけどclubとか弾かれるので
validateDisposable: true, // 捨てアドかどうかチェック
validateSMTP: false, // 日本だと25ポートが殆どのプロバイダーで塞がれていてタイムアウトになるので
}) : { valid: true, reason: null };
const verifymailApi = meta.enableVerifymailApi && meta.verifymailAuthKey != null;
let validated;
if (meta.enableActiveEmailValidation && meta.verifymailAuthKey) {
if (verifymailApi) {
validated = await this.verifyMail(emailAddress, meta.verifymailAuthKey);
} else {
validated = meta.enableActiveEmailValidation ? await validateEmail({
email: emailAddress,
validateRegex: true,
validateMx: true,
validateTypo: false, // TLDを見ているみたいだけどclubとか弾かれるので
validateDisposable: true, // 捨てアドかどうかチェック
validateSMTP: false, // 日本だと25ポートが殆どのプロバイダーで塞がれていてタイムアウトになるので
}) : { valid: true, reason: null };
}
} else {
validated = { valid: true, reason: null };
}
const available = exist === 0 && validated.valid;
@ -182,4 +197,65 @@ export class EmailService {
null,
};
}
private async verifyMail(emailAddress: string, verifymailAuthKey: string): Promise<{
valid: boolean;
reason: 'used' | 'format' | 'disposable' | 'mx' | 'smtp' | null;
}> {
const endpoint = 'https://verifymail.io/api/' + emailAddress + '?key=' + verifymailAuthKey;
const res = await this.httpRequestService.send(endpoint, {
method: 'GET',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/json, */*',
},
});
const json = (await res.json()) as {
block: boolean;
catch_all: boolean;
deliverable_email: boolean;
disposable: boolean;
domain: string;
email_address: string;
email_provider: string;
mx: boolean;
mx_fallback: boolean;
mx_host: string[];
mx_ip: string[];
mx_priority: { [key: string]: number };
privacy: boolean;
related_domains: string[];
};
if (json.email_address === undefined) {
return {
valid: false,
reason: 'format',
};
}
if (json.deliverable_email !== undefined && !json.deliverable_email) {
return {
valid: false,
reason: 'smtp',
};
}
if (json.disposable) {
return {
valid: false,
reason: 'disposable',
};
}
if (json.mx !== undefined && !json.mx) {
return {
valid: false,
reason: 'mx',
};
}
return {
valid: true,
reason: null,
};
}
}

192
packages/backend/src/core/FanoutTimelineEndpointService.ts Normal file
View file

@ -0,0 +1,192 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import type { MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { Packed } from '@/misc/json-schema.js';
import type { NotesRepository } from '@/models/_.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { FanoutTimelineName, FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { isPureRenote } from '@/misc/is-pure-renote.js';
import { CacheService } from '@/core/CacheService.js';
import { isReply } from '@/misc/is-reply.js';
import { isInstanceMuted } from '@/misc/is-instance-muted.js';
type TimelineOptions = {
untilId: string | null,
sinceId: string | null,
limit: number,
allowPartial: boolean,
me?: { id: MiUser['id'] } | undefined | null,
useDbFallback: boolean,
redisTimelines: FanoutTimelineName[],
noteFilter?: (note: MiNote) => boolean,
alwaysIncludeMyNotes?: boolean;
ignoreAuthorFromBlock?: boolean;
ignoreAuthorFromMute?: boolean;
excludeNoFiles?: boolean;
excludeReplies?: boolean;
excludeBots?: boolean;
excludePureRenotes: boolean;
dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise<MiNote[]>,
};
@Injectable()
export class FanoutTimelineEndpointService {
constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private noteEntityService: NoteEntityService,
private cacheService: CacheService,
private fanoutTimelineService: FanoutTimelineService,
) {
}
@bindThis
async timeline(ps: TimelineOptions): Promise<Packed<'Note'>[]> {
return await this.noteEntityService.packMany(await this.getMiNotes(ps), ps.me);
}
@bindThis
private async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> {
let noteIds: string[];
let shouldFallbackToDb = false;
// 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える
if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]);
const shouldPrepend = ps.sinceId && !ps.untilId;
const idCompare: (a: string, b: string) => number = shouldPrepend ? (a, b) => a < b ? -1 : 1 : (a, b) => a > b ? -1 : 1;
const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId);
// TODO: いい感じにgetMulti内でソート済だからuniqするときにredisResultが全てソート済なのを利用して再ソートを避けたい
const redisResultIds = Array.from(new Set(redisResult.flat(1)));
redisResultIds.sort(idCompare);
noteIds = redisResultIds.slice(0, ps.limit);
shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0);
if (!shouldFallbackToDb) {
let filter = ps.noteFilter ?? (_note => true);
if (ps.alwaysIncludeMyNotes && ps.me) {
const me = ps.me;
const parentFilter = filter;
filter = (note) => note.userId === me.id || parentFilter(note);
}
if (ps.excludeNoFiles) {
const parentFilter = filter;
filter = (note) => note.fileIds.length !== 0 && parentFilter(note);
}
if (ps.excludeReplies) {
const parentFilter = filter;
filter = (note) => !isReply(note, ps.me?.id) && parentFilter(note);
}
if (ps.excludeBots) {
const parentFilter = filter;
filter = (note) => !note.user?.isBot && parentFilter(note);
}
if (ps.excludePureRenotes) {
const parentFilter = filter;
filter = (note) => !isPureRenote(note) && parentFilter(note);
}
if (ps.me) {
const me = ps.me;
const [
userIdsWhoMeMuting,
userIdsWhoMeMutingRenotes,
userIdsWhoBlockingMe,
userMutedInstances,
] = await Promise.all([
this.cacheService.userMutingsCache.fetch(ps.me.id),
this.cacheService.renoteMutingsCache.fetch(ps.me.id),
this.cacheService.userBlockedCache.fetch(ps.me.id),
this.cacheService.userProfileCache.fetch(me.id).then(p => new Set(p.mutedInstances)),
]);
const parentFilter = filter;
filter = (note) => {
if (isUserRelated(note, userIdsWhoBlockingMe, ps.ignoreAuthorFromBlock)) return false;
if (isUserRelated(note, userIdsWhoMeMuting, ps.ignoreAuthorFromMute)) return false;
if (isPureRenote(note) && isUserRelated(note, userIdsWhoMeMutingRenotes, ps.ignoreAuthorFromMute)) return false;
if (isInstanceMuted(note, userMutedInstances)) return false;
return parentFilter(note);
};
}
const redisTimeline: MiNote[] = [];
let readFromRedis = 0;
let lastSuccessfulRate = 1; // rateをキャッシュする？
while ((redisResultIds.length - readFromRedis) !== 0) {
const remainingToRead = ps.limit - redisTimeline.length;
// DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで
const countToGet = Math.ceil(remainingToRead * Math.min(1.1 / lastSuccessfulRate, 3));
noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet);
readFromRedis += noteIds.length;
const gotFromDb = await this.getAndFilterFromDb(noteIds, filter, idCompare);
redisTimeline.push(...gotFromDb);
lastSuccessfulRate = gotFromDb.length / noteIds.length;
if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) {
// 十分Redisからとれた
const result = redisTimeline.slice(0, ps.limit);
if (shouldPrepend) result.reverse();
return result;
}
}
// まだ足りない分はDBにフォールバック
const remainingToRead = ps.limit - redisTimeline.length;
let dbUntil: string | null;
let dbSince: string | null;
if (shouldPrepend) {
redisTimeline.reverse();
dbUntil = ps.untilId;
dbSince = noteIds[noteIds.length - 1];
} else {
dbUntil = noteIds[noteIds.length - 1];
dbSince = ps.sinceId;
}
const gotFromDb = await ps.dbFallback(dbUntil, dbSince, remainingToRead);
return shouldPrepend ? [...gotFromDb, ...redisTimeline] : [...redisTimeline, ...gotFromDb];
}
return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit);
}
private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean, idCompare: (a: string, b: string) => number): Promise<MiNote[]> {
const query = this.notesRepository.createQueryBuilder('note')
.where('note.id IN (:...noteIds)', { noteIds: noteIds })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
const notes = (await query.getMany()).filter(noteFilter);
notes.sort((a, b) => idCompare(a.id, b.id));
return notes;
}
}

39
packages/backend/src/core/FunoutTimelineService.ts → packages/backend/src/core/FanoutTimelineService.ts
View file

@ -9,8 +9,37 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
export type FanoutTimelineName =
// home timeline
| `homeTimeline:${string}`
| `homeTimelineWithFiles:${string}` // only notes with files are included
// local timeline
| `localTimeline` // replies are not included
| `localTimelineWithFiles` // only non-reply notes with files are included
| `localTimelineWithReplies` // only replies are included
| `localTimelineWithReplyTo:${string}` // Only replies to specific local user are included. Parameter is reply user id.
// antenna
| `antennaTimeline:${string}`
// user timeline
| `userTimeline:${string}` // replies are not included
| `userTimelineWithFiles:${string}` // only non-reply notes with files are included
| `userTimelineWithReplies:${string}` // only replies are included
| `userTimelineWithChannel:${string}` // only channel notes are included, replies are included
// user list timelines
| `userListTimeline:${string}`
| `userListTimelineWithFiles:${string}` // only notes with files are included
// channel timelines
| `channelTimeline:${string}` // replies are included
// role timelines
| `roleTimeline:${string}` // any notes are included
@Injectable()
export class FunoutTimelineService {
export class FanoutTimelineService {
constructor(
@Inject(DI.redisForTimelines)
private redisForTimelines: Redis.Redis,
@ -20,7 +49,7 @@ export class FunoutTimelineService {
}
@bindThis
public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) {
public push(tl: FanoutTimelineName, id: string, maxlen: number, pipeline: Redis.ChainableCommander) {
// リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、
// 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する
if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) {
@ -41,7 +70,7 @@ export class FunoutTimelineService {
}
@bindThis
public get(name: string, untilId?: string | null, sinceId?: string | null) {
public get(name: FanoutTimelineName, untilId?: string | null, sinceId?: string | null) {
if (untilId && sinceId) {
return this.redisForTimelines.lrange('list:' + name, 0, -1)
.then(ids => ids.filter(id => id < untilId && id > sinceId).sort((a, b) => a > b ? -1 : 1));
@ -58,7 +87,7 @@ export class FunoutTimelineService {
}
@bindThis
public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise<string[][]> {
public getMulti(name: FanoutTimelineName[], untilId?: string | null, sinceId?: string | null): Promise<string[][]> {
const pipeline = this.redisForTimelines.pipeline();
for (const n of name) {
pipeline.lrange('list:' + n, 0, -1);
@ -79,7 +108,7 @@ export class FunoutTimelineService {
}
@bindThis
public purge(name: string) {
public purge(name: FanoutTimelineName) {
return this.redisForTimelines.del('list:' + name);
}
}

33
packages/backend/src/core/FeaturedService.ts
View file

@ -5,14 +5,17 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import type { MiNote, MiUser } from '@/models/_.js';
import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと
const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと
const featuredEpoc = new Date('2023-01-01T00:00:00Z').getTime();
@Injectable()
export class FeaturedService {
constructor(
@ -23,7 +26,7 @@ export class FeaturedService {
@bindThis
private getCurrentWindow(windowRange: number): number {
const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime();
const passed = new Date().getTime() - featuredEpoc;
return Math.floor(passed / windowRange);
}
@ -74,11 +77,27 @@ export class FeaturedService {
return Array.from(ranking.keys());
}
@bindThis
private async removeFromRanking(name: string, windowRange: number, element: string): Promise<void> {
const currentWindow = this.getCurrentWindow(windowRange);
const previousWindow = currentWindow - 1;
const redisPipeline = this.redisClient.pipeline();
redisPipeline.zrem(`${name}:${currentWindow}`, element);
redisPipeline.zrem(`${name}:${previousWindow}`, element);
await redisPipeline.exec();
}
@bindThis
public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
}
@bindThis
public updateGalleryPostsRanking(galleryPostId: MiGalleryPost['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPostId, score);
}
@bindThis
public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
@ -99,6 +118,11 @@ export class FeaturedService {
return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, threshold);
}
@bindThis
public getGalleryPostsRanking(threshold: number): Promise<MiGalleryPost['id'][]> {
return this.getRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, threshold);
}
@bindThis
public getInChannelNotesRanking(channelId: MiNote['channelId'], threshold: number): Promise<MiNote['id'][]> {
return this.getRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, threshold);
@ -113,4 +137,9 @@ export class FeaturedService {
public getHashtagsRanking(threshold: number): Promise<string[]> {
return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, threshold);
}
@bindThis
public removeHashtagsFromRanking(hashtag: string): Promise<void> {
return this.removeFromRanking('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag);
}
}

23
packages/backend/src/core/IdService.ts
View file

@ -7,11 +7,11 @@ import { Inject, Injectable } from '@nestjs/common';
import { ulid } from 'ulid';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { genAid, parseAid } from '@/misc/id/aid.js';
import { genAidx, parseAidx } from '@/misc/id/aidx.js';
import { genMeid, parseMeid } from '@/misc/id/meid.js';
import { genMeidg, parseMeidg } from '@/misc/id/meidg.js';
import { genObjectId, parseObjectId } from '@/misc/id/object-id.js';
import { genAid, isSafeAidT, parseAid } from '@/misc/id/aid.js';
import { genAidx, isSafeAidxT, parseAidx } from '@/misc/id/aidx.js';
import { genMeid, isSafeMeidT, parseMeid } from '@/misc/id/meid.js';
import { genMeidg, isSafeMeidgT, parseMeidg } from '@/misc/id/meidg.js';
import { genObjectId, isSafeObjectIdT, parseObjectId } from '@/misc/id/object-id.js';
import { bindThis } from '@/decorators.js';
import { parseUlid } from '@/misc/id/ulid.js';
@ -26,6 +26,19 @@ export class IdService {
this.method = config.id.toLowerCase();
}
@bindThis
public isSafeT(t: number): boolean {
switch (this.method) {
case 'aid': return isSafeAidT(t);
case 'aidx': return isSafeAidxT(t);
case 'meid': return isSafeMeidT(t);
case 'meidg': return isSafeMeidgT(t);
case 'ulid': return t > 0;
case 'objectid': return isSafeObjectIdT(t);
default: throw new Error('unrecognized id generation method');
}
}
/**
* 時間を元にIDを生成します(省略時は現在日時)
* @param time 日時

25
packages/backend/src/core/MetaService.ts
View file

@ -11,6 +11,7 @@ import { MiMeta } from '@/models/Meta.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -25,6 +26,7 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
//this.onMessage = this.onMessage.bind(this);
@ -95,6 +97,8 @@ export class MetaService implements OnApplicationShutdown {
@bindThis
public async update(data: Partial<MiMeta>): Promise<MiMeta> {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
@ -102,10 +106,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
const meta = metas[0];
before = metas[0];
if (meta) {
await transactionalEntityManager.update(MiMeta, meta.id, data);
if (before) {
await transactionalEntityManager.update(MiMeta, before.id, data);
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
@ -119,6 +123,21 @@ export class MetaService implements OnApplicationShutdown {
}
});
if (data.hiddenTags) {
process.nextTick(() => {
const hiddenTags = new Set<string>(data.hiddenTags);
if (before) {
for (const previousHiddenTag of before.hiddenTags) {
hiddenTags.delete(previousHiddenTag);
}
}
for (const hiddenTag of hiddenTags) {
this.featuredService.removeHashtagsFromRanking(hiddenTag);
}
});
}
this.globalEventService.publishInternalEvent('metaUpdated', updated);
return updated;

79
packages/backend/src/core/MfmService.ts
View file

@ -250,6 +250,12 @@ export class MfmService {
}
}
function fnDefault(node: mfm.MfmFn) {
const el = doc.createElement('i');
appendChildren(node.children, el);
return el;
}
const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any } = {
bold: (node) => {
const el = doc.createElement('b');
@ -276,17 +282,68 @@ export class MfmService {
},
fn: (node) => {
if (node.props.name === 'unixtime') {
const text = node.children[0]!.type === 'text' ? node.children[0].props.text : '';
const date = new Date(parseInt(text, 10) * 1000);
const el = doc.createElement('time');
el.setAttribute('datetime', date.toISOString());
el.textContent = date.toISOString();
return el;
} else {
const el = doc.createElement('i');
appendChildren(node.children, el);
return el;
switch (node.props.name) {
case 'unixtime': {
const text = node.children[0].type === 'text' ? node.children[0].props.text : '';
try {
const date = new Date(parseInt(text, 10) * 1000);
const el = doc.createElement('time');
el.setAttribute('datetime', date.toISOString());
el.textContent = date.toISOString();
return el;
} catch (err) {
return fnDefault(node);
}
}
case 'ruby': {
if (node.children.length === 1) {
const child = node.children[0];
const text = child.type === 'text' ? child.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
return rubyEl;
} else {
const rt = node.children.at(-1);
if (!rt) {
return fnDefault(node);
}
const text = rt.type === 'text' ? rt.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
rtEl.appendChild(doc.createTextNode(text.trim()));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
return rubyEl;
}
}
default: {
return fnDefault(node);
}
}
},

58
packages/backend/src/core/NoteCreateService.ts
View file

@ -54,9 +54,10 @@ import { RoleService } from '@/core/RoleService.js';
import { MetaService } from '@/core/MetaService.js';
import { SearchService } from '@/core/SearchService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { FunoutTimelineService } from '@/core/FunoutTimelineService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { isReply } from '@/misc/is-reply.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -194,7 +195,7 @@ export class NoteCreateService implements OnApplicationShutdown {
private idService: IdService,
private globalEventService: GlobalEventService,
private queueService: QueueService,
private funoutTimelineService: FunoutTimelineService,
private fanoutTimelineService: FanoutTimelineService,
private noteReadService: NoteReadService,
private notificationService: NotificationService,
private relayService: RelayService,
@ -461,7 +462,7 @@ export class NoteCreateService implements OnApplicationShutdown {
}
// Check blocking
if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) {
if (this.isQuote(data)) {
if (data.renote.userHost === null) {
if (data.renote.userId !== user.id) {
const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
@ -804,7 +805,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// If it is renote
if (data.renote) {
const type = data.text ? 'quote' : 'renote';
const type = this.isQuote(data) ? 'quote' : 'renote';
// Notify
if (data.renote.userHost === null) {
@ -1010,6 +1011,12 @@ export class NoteCreateService implements OnApplicationShutdown {
return false;
}
@bindThis
private isQuote(note: Option): note is Option & { renote: MiNote } {
// sync with misc/is-quote.ts
return !!note.renote && (!!note.text || !!note.cw || (!!note.files && !!note.files.length) || !!note.poll);
}
@bindThis
private incRenoteCount(renote: MiNote) {
this.notesRepository.createQueryBuilder().update()
@ -1075,7 +1082,7 @@ export class NoteCreateService implements OnApplicationShutdown {
private async renderNoteOrRenoteActivity(data: Option, note: MiNote) {
if (data.localOnly) return null;
const content = data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)
const content = data.renote && !this.isQuote(data)
? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note)
: this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, false), note);
@ -1125,9 +1132,9 @@ export class NoteCreateService implements OnApplicationShutdown {
const r = this.redisForTimelines.pipeline();
if (note.channelId) {
this.funoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r);
this.fanoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r);
this.funoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
this.fanoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
const channelFollowings = await this.channelFollowingsRepository.find({
where: {
@ -1137,9 +1144,9 @@ export class NoteCreateService implements OnApplicationShutdown {
});
for (const channelFollowing of channelFollowings) {
this.funoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
this.fanoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
} else {
@ -1173,13 +1180,13 @@ export class NoteCreateService implements OnApplicationShutdown {
if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue;
// 「自分自身への返信 or そのフォロワーへの返信」のどちらでもない場合
if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === following.followerId)) {
if (isReply(note, following.followerId)) {
if (!following.withReplies) continue;
}
this.funoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
this.fanoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
@ -1191,40 +1198,43 @@ export class NoteCreateService implements OnApplicationShutdown {
) continue;
// 「自分自身への返信 or そのリストの作成者への返信」のどちらでもない場合
if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === userListMembership.userListUserId)) {
if (isReply(note, userListMembership.userListUserId)) {
if (!userListMembership.withReplies) continue;
}
this.funoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r);
this.fanoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r);
}
}
if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL
this.funoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r);
this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
// 自分自身以外への返信
if (note.replyId && note.replyUserId !== note.userId) {
this.funoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
if (isReply(note)) {
this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
if (note.visibility === 'public' && note.userHost == null) {
this.funoutTimelineService.push('localTimelineWithReplies', note.id, 300, r);
this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r);
if (note.replyUserHost == null) {
this.fanoutTimelineService.push(`localTimelineWithReplyTo:${note.replyUserId}`, note.id, 300 / 10, r);
}
}
} else {
this.funoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r);
}
if (note.visibility === 'public' && note.userHost == null) {
this.funoutTimelineService.push('localTimeline', note.id, 1000, r);
this.fanoutTimelineService.push('localTimeline', note.id, 1000, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push('localTimelineWithFiles', note.id, 500, r);
this.fanoutTimelineService.push('localTimelineWithFiles', note.id, 500, r);
}
}
}

36
packages/backend/src/core/NoteEditService.ts
View file

@ -44,7 +44,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { RoleService } from '@/core/RoleService.js';
import { MetaService } from '@/core/MetaService.js';
import { SearchService } from '@/core/SearchService.js';
import { FunoutTimelineService } from '@/core/FunoutTimelineService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { UtilityService } from '@/core/UtilityService.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -191,7 +191,7 @@ export class NoteEditService implements OnApplicationShutdown {
private idService: IdService,
private globalEventService: GlobalEventService,
private queueService: QueueService,
private funoutTimelineService: FunoutTimelineService,
private fanoutTimelineService: FanoutTimelineService,
private noteReadService: NoteReadService,
private notificationService: NotificationService,
private relayService: RelayService,
@ -786,9 +786,9 @@ export class NoteEditService implements OnApplicationShutdown {
const r = this.redisForTimelines.pipeline();
if (note.channelId) {
this.funoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r);
this.fanoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r);
this.funoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
this.fanoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
const channelFollowings = await this.channelFollowingsRepository.find({
where: {
@ -798,9 +798,9 @@ export class NoteEditService implements OnApplicationShutdown {
});
for (const channelFollowing of channelFollowings) {
this.funoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
this.fanoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
} else {
@ -838,9 +838,9 @@ export class NoteEditService implements OnApplicationShutdown {
if (!following.withReplies) continue;
}
this.funoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
this.fanoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
@ -856,36 +856,36 @@ export class NoteEditService implements OnApplicationShutdown {
if (!userListMembership.withReplies) continue;
}
this.funoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r);
this.fanoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r);
}
}
if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL
this.funoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r);
this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
// 自分自身以外への返信
if (note.replyId && note.replyUserId !== note.userId) {
this.funoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
if (note.visibility === 'public' && note.userHost == null) {
this.funoutTimelineService.push('localTimelineWithReplies', note.id, 300, r);
this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r);
}
} else {
this.funoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r);
this.fanoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r);
}
if (note.visibility === 'public' && note.userHost == null) {
this.funoutTimelineService.push('localTimeline', note.id, 1000, r);
this.fanoutTimelineService.push('localTimeline', note.id, 1000, r);
if (note.fileIds.length > 0) {
this.funoutTimelineService.push('localTimelineWithFiles', note.id, 500, r);
this.fanoutTimelineService.push('localTimelineWithFiles', note.id, 500, r);
}
}
}

4
packages/backend/src/core/NotePiningService.ts
View file

@ -77,7 +77,7 @@ export class NotePiningService {
} as MiUserNotePining);
// Deliver to remote followers
if (this.userEntityService.isLocalUser(user)) {
if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) {
this.deliverPinnedChange(user.id, note.id, true);
}
}
@ -105,7 +105,7 @@ export class NotePiningService {
});
// Deliver to remote followers
if (this.userEntityService.isLocalUser(user)) {
if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) {
this.deliverPinnedChange(user.id, noteId, false);
}
}

43
packages/backend/src/core/RoleService.ts
View file

@ -6,7 +6,14 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { In } from 'typeorm';
import type { MiRole, MiRoleAssignment, RoleAssignmentsRepository, RolesRepository, UsersRepository } from '@/models/_.js';
import { ModuleRef } from '@nestjs/core';
import type {
MiRole,
MiRoleAssignment,
RoleAssignmentsRepository,
RolesRepository,
UsersRepository,
} from '@/models/_.js';
import { MemoryKVCache, MemorySingleCache } from '@/misc/cache.js';
import type { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
@ -16,12 +23,13 @@ import { CacheService } from '@/core/CacheService.js';
import type { RoleCondFormulaValue } from '@/models/Role.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import type { Packed } from '@/misc/json-schema.js';
import { FunoutTimelineService } from '@/core/FunoutTimelineService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { NotificationService } from '@/core/NotificationService.js';
import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
export type RolePolicies = {
gtlAvailable: boolean;
@ -49,6 +57,7 @@ export type RolePolicies = {
userEachUserListsLimit: number;
rateLimitFactor: number;
canImportNotes: boolean;
avatarDecorationLimit: number;
};
export const DEFAULT_POLICIES: RolePolicies = {
@ -77,20 +86,27 @@ export const DEFAULT_POLICIES: RolePolicies = {
userEachUserListsLimit: 50,
rateLimitFactor: 1,
canImportNotes: true,
avatarDecorationLimit: 1,
};
@Injectable()
export class RoleService implements OnApplicationShutdown {
export class RoleService implements OnApplicationShutdown, OnModuleInit {
private rolesCache: MemorySingleCache<MiRole[]>;
private roleAssignmentByUserIdCache: MemoryKVCache<MiRoleAssignment[]>;
private notificationService: NotificationService;
public static AlreadyAssignedError = class extends Error {};
public static NotAssignedError = class extends Error {};
constructor(
private moduleRef: ModuleRef,
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForTimelines)
private redisForTimelines: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@ -109,7 +125,7 @@ export class RoleService implements OnApplicationShutdown {
private globalEventService: GlobalEventService,
private idService: IdService,
private moderationLogService: ModerationLogService,
private funoutTimelineService: FunoutTimelineService,
private fanoutTimelineService: FanoutTimelineService,
) {
//this.onMessage = this.onMessage.bind(this);
@ -119,6 +135,10 @@ export class RoleService implements OnApplicationShutdown {
this.redisForSub.on('message', this.onMessage);
}
async onModuleInit() {
this.notificationService = this.moduleRef.get(NotificationService.name);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
@ -329,6 +349,7 @@ export class RoleService implements OnApplicationShutdown {
userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)),
rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)),
canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)),
avatarDecorationLimit: calc('avatarDecorationLimit', vs => Math.max(...vs)),
};
}
@ -427,6 +448,12 @@ export class RoleService implements OnApplicationShutdown {
this.globalEventService.publishInternalEvent('userRoleAssigned', created);
if (role.isPublic) {
this.notificationService.createNotification(userId, 'roleAssigned', {
roleId: roleId,
});
}
if (moderator) {
const user = await this.usersRepository.findOneByOrFail({ id: userId });
this.moderationLogService.log(moderator, 'assignRole', {
@ -482,10 +509,10 @@ export class RoleService implements OnApplicationShutdown {
public async addNoteToRoleTimeline(note: Packed<'Note'>): Promise<void> {
const roles = await this.getUserRoles(note.userId);
const redisPipeline = this.redisClient.pipeline();
const redisPipeline = this.redisForTimelines.pipeline();
for (const role of roles) {
this.funoutTimelineService.push(`roleTimeline:${role.id}`, note.id, 1000, redisPipeline);
this.fanoutTimelineService.push(`roleTimeline:${role.id}`, note.id, 1000, redisPipeline);
this.globalEventService.publishRoleTimelineStream(role.id, 'note', note);
}

16
packages/backend/src/core/SearchService.ts
View file

@ -12,6 +12,8 @@ import { MiNote } from '@/models/Note.js';
import { MiUser } from '@/models/_.js';
import type { NotesRepository } from '@/models/_.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { CacheService } from '@/core/CacheService.js';
import { QueryService } from '@/core/QueryService.js';
import { IdService } from '@/core/IdService.js';
import type { Index, MeiliSearch } from 'meilisearch';
@ -74,6 +76,7 @@ export class SearchService {
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private cacheService: CacheService,
private queryService: QueryService,
private idService: IdService,
) {
@ -230,8 +233,19 @@ export class SearchService {
limit: pagination.limit,
});
if (res.hits.length === 0) return [];
const notes = await this.notesRepository.findBy({
const [
userIdsWhoMeMuting,
userIdsWhoBlockingMe,
] = me ? await Promise.all([
this.cacheService.userMutingsCache.fetch(me.id),
this.cacheService.userBlockedCache.fetch(me.id),
]) : [new Set<string>(), new Set<string>()];
const notes = (await this.notesRepository.findBy({
id: In(res.hits.map(x => x.id)),
})).filter(note => {
if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
return true;
});
return notes.sort((a, b) => a.id > b.id ? -1 : 1);
} else {

8
packages/backend/src/core/UserFollowingService.ts
View file

@ -29,7 +29,7 @@ import { CacheService } from '@/core/CacheService.js';
import type { Config } from '@/config.js';
import { AccountMoveService } from '@/core/AccountMoveService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { FunoutTimelineService } from '@/core/FunoutTimelineService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import Logger from '../logger.js';
const logger = new Logger('following/create');
@ -84,7 +84,7 @@ export class UserFollowingService implements OnModuleInit {
private webhookService: WebhookService,
private apRendererService: ApRendererService,
private accountMoveService: AccountMoveService,
private funoutTimelineService: FunoutTimelineService,
private fanoutTimelineService: FanoutTimelineService,
private perUserFollowingChart: PerUserFollowingChart,
private instanceChart: InstanceChart,
) {
@ -304,8 +304,6 @@ export class UserFollowingService implements OnModuleInit {
});
}
});
this.funoutTimelineService.purge(`homeTimeline:${follower.id}`);
}
// Publish followed event
@ -373,8 +371,6 @@ export class UserFollowingService implements OnModuleInit {
});
}
});
this.funoutTimelineService.purge(`homeTimeline:${follower.id}`);
}
if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) {

17
packages/backend/src/core/UserListService.ts
View file

@ -3,30 +3,34 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import { Inject, Injectable, OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
import * as Redis from 'ioredis';
import { ModuleRef } from '@nestjs/core';
import type { UserListMembershipsRepository } from '@/models/_.js';
import type { MiUser } from '@/models/User.js';
import type { MiUserList } from '@/models/UserList.js';
import type { MiUserListMembership } from '@/models/UserListMembership.js';
import { IdService } from '@/core/IdService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ProxyAccountService } from '@/core/ProxyAccountService.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { QueueService } from '@/core/QueueService.js';
import { RedisKVCache } from '@/misc/cache.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { RoleService } from '@/core/RoleService.js';
@Injectable()
export class UserListService implements OnApplicationShutdown {
export class UserListService implements OnApplicationShutdown, OnModuleInit {
public static TooManyUsersError = class extends Error {};
public membersCache: RedisKVCache<Set<string>>;
private roleService: RoleService;
constructor(
private moduleRef: ModuleRef,
@Inject(DI.redis)
private redisClient: Redis.Redis,
@ -38,7 +42,6 @@ export class UserListService implements OnApplicationShutdown {
private userEntityService: UserEntityService,
private idService: IdService,
private roleService: RoleService,
private globalEventService: GlobalEventService,
private proxyAccountService: ProxyAccountService,
private queueService: QueueService,
@ -54,6 +57,10 @@ export class UserListService implements OnApplicationShutdown {
this.redisForSub.on('message', this.onMessage);
}
async onModuleInit() {
this.roleService = this.moduleRef.get(RoleService.name);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);

8
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -306,9 +306,15 @@ export class ApInboxService {
this.logger.info(`Creating the (Re)Note: ${uri}`);
const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc);
const createdAt = activity.published ? new Date(activity.published) : null;
if (createdAt && createdAt < this.idService.parse(renote.id).date) {
this.logger.warn('skip: malformed createdAt');
return;
}
await this.noteCreateService.create(actor, {
createdAt: activity.published ? new Date(activity.published) : null,
createdAt,
renote,
visibility: activityAudience.visibility,
visibleUsers: activityAudience.visibleUsers,

4
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -97,6 +97,10 @@ export class ApNoteService {
return new Error(`invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`);
}
if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) {
return new Error('invalid Note: published timestamp is malformed');
}
return null;
}

30
packages/backend/src/core/entities/NotificationEntityService.ts
View file

@ -15,8 +15,8 @@ import type { Packed } from '@/misc/json-schema.js';
import { bindThis } from '@/decorators.js';
import { isNotNull } from '@/misc/is-not-null.js';
import { FilterUnionByProperty, notificationTypes } from '@/types.js';
import { RoleEntityService } from './RoleEntityService.js';
import type { OnModuleInit } from '@nestjs/common';
import type { CustomEmojiService } from '../CustomEmojiService.js';
import type { UserEntityService } from './UserEntityService.js';
import type { NoteEntityService } from './NoteEntityService.js';
@ -27,7 +27,7 @@ const NOTE_REQUIRED_GROUPED_NOTIFICATION_TYPES = new Set(['note', 'mention', 're
export class NotificationEntityService implements OnModuleInit {
private userEntityService: UserEntityService;
private noteEntityService: NoteEntityService;
private customEmojiService: CustomEmojiService;
private roleEntityService: RoleEntityService;
constructor(
private moduleRef: ModuleRef,
@ -43,14 +43,13 @@ export class NotificationEntityService implements OnModuleInit {
//private userEntityService: UserEntityService,
//private noteEntityService: NoteEntityService,
//private customEmojiService: CustomEmojiService,
) {
}
onModuleInit() {
this.userEntityService = this.moduleRef.get('UserEntityService');
this.noteEntityService = this.moduleRef.get('NoteEntityService');
this.customEmojiService = this.moduleRef.get('CustomEmojiService');
this.roleEntityService = this.moduleRef.get('RoleEntityService');
}
@bindThis
@ -81,6 +80,7 @@ export class NotificationEntityService implements OnModuleInit {
detail: false,
})
) : undefined;
const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined;
return await awaitAll({
id: notification.id,
@ -92,6 +92,9 @@ export class NotificationEntityService implements OnModuleInit {
...(notification.type === 'reaction' ? {
reaction: notification.reaction,
} : {}),
...(notification.type === 'roleAssigned' ? {
role: role,
} : {}),
...(notification.type === 'achievementEarned' ? {
achievement: notification.achievement,
} : {}),
@ -198,12 +201,14 @@ export class NotificationEntityService implements OnModuleInit {
});
} else if (notification.type === 'renote:grouped') {
const users = await Promise.all(notification.userIds.map(userId => {
const user = hint?.packedUsers != null
? hint.packedUsers.get(userId)
: this.userEntityService.pack(userId!, { id: meId }, {
detail: false,
});
return user;
const packedUser = hint?.packedUsers != null ? hint.packedUsers.get(userId) : null;
if (packedUser) {
return packedUser;
}
return this.userEntityService.pack(userId, { id: meId }, {
detail: false,
});
}));
return await awaitAll({
id: notification.id,
@ -214,6 +219,8 @@ export class NotificationEntityService implements OnModuleInit {
});
}
const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined;
return await awaitAll({
id: notification.id,
createdAt: new Date(notification.createdAt).toISOString(),
@ -224,6 +231,9 @@ export class NotificationEntityService implements OnModuleInit {
...(notification.type === 'reaction' ? {
reaction: notification.reaction,
} : {}),
...(notification.type === 'roleAssigned' ? {
role: role,
} : {}),
...(notification.type === 'achievementEarned' ? {
achievement: notification.achievement,
} : {}),

14
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -361,13 +361,13 @@ export class UserEntityService implements OnModuleInit {
const mastoapi = !opts.detail ? opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null;
const followingCount = profile == null ? null :
(profile.ffVisibility === 'public') || isMe ? user.followingCount :
(profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount :
(profile.followingVisibility === 'public') || isMe ? user.followingCount :
(profile.followingVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount :
null;
const followersCount = profile == null ? null :
(profile.ffVisibility === 'public') || isMe ? user.followersCount :
(profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount :
(profile.followersVisibility === 'public') || isMe ? user.followersCount :
(profile.followersVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount :
null;
const isModerator = isMe && opts.detail ? this.roleService.isModerator(user) : null;
@ -395,6 +395,8 @@ export class UserEntityService implements OnModuleInit {
id: ud.id,
angle: ud.angle || undefined,
flipH: ud.flipH || undefined,
offsetX: ud.offsetX || undefined,
offsetY: ud.offsetY || undefined,
url: decorations.find(d => d.id === ud.id)!.url,
}))) : [],
isBot: user.isBot,
@ -452,7 +454,8 @@ export class UserEntityService implements OnModuleInit {
pinnedPageId: profile!.pinnedPageId,
pinnedPage: profile!.pinnedPageId ? this.pageEntityService.pack(profile!.pinnedPageId, me) : null,
publicReactions: profile!.publicReactions,
ffVisibility: profile!.ffVisibility,
followersVisibility: profile!.followersVisibility,
followingVisibility: profile!.followingVisibility,
twoFactorEnabled: profile!.twoFactorEnabled,
usePasswordLessLogin: profile!.usePasswordLessLogin,
securityKeys: profile!.twoFactorEnabled
@ -511,6 +514,7 @@ export class UserEntityService implements OnModuleInit {
hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id),
unreadNotificationsCount: notificationsInfo?.unreadCount,
mutedWords: profile!.mutedWords,
hardMutedWords: profile!.hardMutedWords,
mutedInstances: profile!.mutedInstances,
mutingNotificationTypes: [], // 後方互換性のため
notificationRecieveConfig: profile!.notificationRecieveConfig,