Merge remote-tracking branch 'misskey-original/develop' into develop

# Conflicts:
#	locales/index.d.ts
#	locales/ja-JP.yml
#	package.json
#	packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
#	packages/backend/src/server/api/endpoints/admin/meta.ts
#	packages/backend/src/server/api/endpoints/channels/timeline.ts
#	packages/backend/src/server/api/endpoints/notes/featured.ts
#	packages/frontend/src/components/MkButton.vue
#	packages/frontend/src/components/MkClickerGame.vue
#	packages/frontend/src/components/MkDialog.vue
#	packages/frontend/src/components/MkDrive.vue
#	packages/frontend/src/components/MkEmojiEditDialog.vue
#	packages/frontend/src/components/MkEmojiPicker.section.vue
#	packages/frontend/src/components/MkEmojiPicker.vue
#	packages/frontend/src/components/MkFollowButton.vue
#	packages/frontend/src/components/MkInstanceTicker.vue
#	packages/frontend/src/components/MkLaunchPad.vue
#	packages/frontend/src/components/MkMenu.vue
#	packages/frontend/src/components/MkNote.vue
#	packages/frontend/src/components/MkNoteSimple.vue
#	packages/frontend/src/components/MkPostForm.vue
#	packages/frontend/src/components/MkRadio.vue
#	packages/frontend/src/components/MkSignupDialog.form.vue
#	packages/frontend/src/components/MkSwitch.vue
#	packages/frontend/src/custom-emojis.ts
#	packages/frontend/src/pages/about.emojis.vue
#	packages/frontend/src/pages/about.vue
#	packages/frontend/src/pages/admin/index.vue
#	packages/frontend/src/pages/admin/other-settings.vue
#	packages/frontend/src/pages/custom-emojis-manager.vue
#	packages/frontend/src/pages/settings/general.vue
#	packages/frontend/src/pages/settings/mute-block.vue
#	packages/frontend/src/pages/timeline.vue
#	packages/frontend/src/pages/user/home.vue
#	packages/frontend/src/pages/user/index.files.vue
#	packages/frontend/src/scripts/get-note-menu.ts
#	packages/frontend/src/store.ts
#	packages/frontend/src/ui/_common_/stream-indicator.vue
#	packages/frontend/src/ui/classic.sidebar.vue
#	packages/frontend/src/ui/universal.vue
#	packages/frontend/src/ui/universal.widgets.vue
#	packages/frontend/vite.config.ts
#	packages/misskey-js/etc/misskey-js.api.md
#	packages/misskey-js/src/api.types.ts
#	packages/misskey-js/src/entities.ts
#	packages/misskey-js/src/streaming.types.ts
This commit is contained in:
mattyatea 2023-12-13 07:08:57 +09:00
commit 8c4a08c383
404 changed files with 42042 additions and 11250 deletions

View file

@ -4,6 +4,7 @@
*/
import { Module } from '@nestjs/common';
import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js';
import { AccountMoveService } from './AccountMoveService.js';
import { AccountUpdateService } from './AccountUpdateService.js';
import { AiService } from './AiService.js';
@ -195,6 +196,7 @@ const $SearchService: Provider = { provide: 'SearchService', useExisting: Search
const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService };
const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService };
const $FanoutTimelineService: Provider = { provide: 'FanoutTimelineService', useExisting: FanoutTimelineService };
const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpointService', useExisting: FanoutTimelineEndpointService };
const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService };
const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService };
@ -331,6 +333,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
ClipService,
FeaturedService,
FanoutTimelineService,
FanoutTimelineEndpointService,
ChannelFollowingService,
RegistryApiService,
ChartLoggerService,
@ -460,6 +463,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$ClipService,
$FeaturedService,
$FanoutTimelineService,
$FanoutTimelineEndpointService,
$ChannelFollowingService,
$RegistryApiService,
$ChartLoggerService,
@ -590,6 +594,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
ClipService,
FeaturedService,
FanoutTimelineService,
FanoutTimelineEndpointService,
ChannelFollowingService,
RegistryApiService,
FederationChart,
@ -718,6 +723,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$ClipService,
$FeaturedService,
$FanoutTimelineService,
$FanoutTimelineEndpointService,
$ChannelFollowingService,
$RegistryApiService,
$FederationChart,

View file

@ -0,0 +1,186 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import type { MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { Packed } from '@/misc/json-schema.js';
import type { NotesRepository } from '@/models/_.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { FanoutTimelineName, FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { isPureRenote } from '@/misc/is-pure-renote.js';
import { CacheService } from '@/core/CacheService.js';
import { isReply } from '@/misc/is-reply.js';
import { isInstanceMuted } from '@/misc/is-instance-muted.js';
type TimelineOptions = {
untilId: string | null,
sinceId: string | null,
limit: number,
allowPartial: boolean,
me?: { id: MiUser['id'] } | undefined | null,
useDbFallback: boolean,
redisTimelines: FanoutTimelineName[],
noteFilter?: (note: MiNote) => boolean,
alwaysIncludeMyNotes?: boolean;
ignoreAuthorFromBlock?: boolean;
ignoreAuthorFromMute?: boolean;
excludeNoFiles?: boolean;
excludeReplies?: boolean;
excludePureRenotes: boolean;
dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise<MiNote[]>,
};
@Injectable()
export class FanoutTimelineEndpointService {
constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private noteEntityService: NoteEntityService,
private cacheService: CacheService,
private fanoutTimelineService: FanoutTimelineService,
) {
}
@bindThis
async timeline(ps: TimelineOptions): Promise<Packed<'Note'>[]> {
return await this.noteEntityService.packMany(await this.getMiNotes(ps), ps.me);
}
@bindThis
private async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> {
let noteIds: string[];
let shouldFallbackToDb = false;
// 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える
if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]);
const shouldPrepend = ps.sinceId && !ps.untilId;
const idCompare: (a: string, b: string) => number = shouldPrepend ? (a, b) => a < b ? -1 : 1 : (a, b) => a > b ? -1 : 1;
const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId);
// TODO: いい感じにgetMulti内でソート済だからuniqするときにredisResultが全てソート済なのを利用して再ソートを避けたい
const redisResultIds = Array.from(new Set(redisResult.flat(1)));
redisResultIds.sort(idCompare);
noteIds = redisResultIds.slice(0, ps.limit);
shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0);
if (!shouldFallbackToDb) {
let filter = ps.noteFilter ?? (_note => true);
if (ps.alwaysIncludeMyNotes && ps.me) {
const me = ps.me;
const parentFilter = filter;
filter = (note) => note.userId === me.id || parentFilter(note);
}
if (ps.excludeNoFiles) {
const parentFilter = filter;
filter = (note) => note.fileIds.length !== 0 && parentFilter(note);
}
if (ps.excludeReplies) {
const parentFilter = filter;
filter = (note) => !isReply(note, ps.me?.id) && parentFilter(note);
}
if (ps.excludePureRenotes) {
const parentFilter = filter;
filter = (note) => !isPureRenote(note) && parentFilter(note);
}
if (ps.me) {
const me = ps.me;
const [
userIdsWhoMeMuting,
userIdsWhoMeMutingRenotes,
userIdsWhoBlockingMe,
userMutedInstances,
] = await Promise.all([
this.cacheService.userMutingsCache.fetch(ps.me.id),
this.cacheService.renoteMutingsCache.fetch(ps.me.id),
this.cacheService.userBlockedCache.fetch(ps.me.id),
this.cacheService.userProfileCache.fetch(me.id).then(p => new Set(p.mutedInstances)),
]);
const parentFilter = filter;
filter = (note) => {
if (isUserRelated(note, userIdsWhoBlockingMe, ps.ignoreAuthorFromBlock)) return false;
if (isUserRelated(note, userIdsWhoMeMuting, ps.ignoreAuthorFromMute)) return false;
if (isPureRenote(note) && isUserRelated(note, userIdsWhoMeMutingRenotes, ps.ignoreAuthorFromMute)) return false;
if (isInstanceMuted(note, userMutedInstances)) return false;
return parentFilter(note);
};
}
const redisTimeline: MiNote[] = [];
let readFromRedis = 0;
let lastSuccessfulRate = 1; // rateをキャッシュする
while ((redisResultIds.length - readFromRedis) !== 0) {
const remainingToRead = ps.limit - redisTimeline.length;
// DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで
const countToGet = Math.ceil(remainingToRead * Math.min(1.1 / lastSuccessfulRate, 3));
noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet);
readFromRedis += noteIds.length;
const gotFromDb = await this.getAndFilterFromDb(noteIds, filter, idCompare);
redisTimeline.push(...gotFromDb);
lastSuccessfulRate = gotFromDb.length / noteIds.length;
if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) {
// 十分Redisからとれた
const result = redisTimeline.slice(0, ps.limit);
if (shouldPrepend) result.reverse();
return result;
}
}
// まだ足りない分はDBにフォールバック
const remainingToRead = ps.limit - redisTimeline.length;
let dbUntil: string | null;
let dbSince: string | null;
if (shouldPrepend) {
redisTimeline.reverse();
dbUntil = ps.untilId;
dbSince = noteIds[noteIds.length - 1];
} else {
dbUntil = noteIds[noteIds.length - 1];
dbSince = ps.sinceId;
}
const gotFromDb = await ps.dbFallback(dbUntil, dbSince, remainingToRead);
return shouldPrepend ? [...gotFromDb, ...redisTimeline] : [...redisTimeline, ...gotFromDb];
}
return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit);
}
private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean, idCompare: (a: string, b: string) => number): Promise<MiNote[]> {
const query = this.notesRepository.createQueryBuilder('note')
.where('note.id IN (:...noteIds)', { noteIds: noteIds })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
const notes = (await query.getMany()).filter(noteFilter);
notes.sort((a, b) => idCompare(a.id, b.id));
return notes;
}
}

View file

@ -9,6 +9,35 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
export type FanoutTimelineName =
// home timeline
| `homeTimeline:${string}`
| `homeTimelineWithFiles:${string}` // only notes with files are included
// local timeline
| `localTimeline` // replies are not included
| `localTimelineWithFiles` // only non-reply notes with files are included
| `localTimelineWithReplies` // only replies are included
| `localTimelineWithReplyTo:${string}` // Only replies to specific local user are included. Parameter is reply user id.
// antenna
| `antennaTimeline:${string}`
// user timeline
| `userTimeline:${string}` // replies are not included
| `userTimelineWithFiles:${string}` // only non-reply notes with files are included
| `userTimelineWithReplies:${string}` // only replies are included
| `userTimelineWithChannel:${string}` // only channel notes are included, replies are included
// user list timelines
| `userListTimeline:${string}`
| `userListTimelineWithFiles:${string}` // only notes with files are included
// channel timelines
| `channelTimeline:${string}` // replies are included
// role timelines
| `roleTimeline:${string}` // any notes are included
@Injectable()
export class FanoutTimelineService {
constructor(
@ -20,7 +49,7 @@ export class FanoutTimelineService {
}
@bindThis
public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) {
public push(tl: FanoutTimelineName, id: string, maxlen: number, pipeline: Redis.ChainableCommander) {
// リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、
// 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する
if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) {
@ -41,7 +70,7 @@ export class FanoutTimelineService {
}
@bindThis
public get(name: string, untilId?: string | null, sinceId?: string | null) {
public get(name: FanoutTimelineName, untilId?: string | null, sinceId?: string | null) {
if (untilId && sinceId) {
return this.redisForTimelines.lrange('list:' + name, 0, -1)
.then(ids => ids.filter(id => id < untilId && id > sinceId).sort((a, b) => a > b ? -1 : 1));
@ -58,7 +87,7 @@ export class FanoutTimelineService {
}
@bindThis
public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise<string[][]> {
public getMulti(name: FanoutTimelineName[], untilId?: string | null, sinceId?: string | null): Promise<string[][]> {
const pipeline = this.redisForTimelines.pipeline();
for (const n of name) {
pipeline.lrange('list:' + n, 0, -1);
@ -79,7 +108,7 @@ export class FanoutTimelineService {
}
@bindThis
public purge(name: string) {
public purge(name: FanoutTimelineName) {
return this.redisForTimelines.del('list:' + name);
}
}

View file

@ -14,6 +14,8 @@ export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ご
const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと
const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと
const featuredEpoc = new Date('2023-01-01T00:00:00Z').getTime();
@Injectable()
export class FeaturedService {
constructor(
@ -24,7 +26,7 @@ export class FeaturedService {
@bindThis
private getCurrentWindow(windowRange: number): number {
const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime();
const passed = new Date().getTime() - featuredEpoc;
return Math.floor(passed / windowRange);
}

View file

@ -7,11 +7,11 @@ import { Inject, Injectable } from '@nestjs/common';
import { ulid } from 'ulid';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { genAid, parseAid } from '@/misc/id/aid.js';
import { genAidx, parseAidx } from '@/misc/id/aidx.js';
import { genMeid, parseMeid } from '@/misc/id/meid.js';
import { genMeidg, parseMeidg } from '@/misc/id/meidg.js';
import { genObjectId, parseObjectId } from '@/misc/id/object-id.js';
import { genAid, isSafeAidT, parseAid } from '@/misc/id/aid.js';
import { genAidx, isSafeAidxT, parseAidx } from '@/misc/id/aidx.js';
import { genMeid, isSafeMeidT, parseMeid } from '@/misc/id/meid.js';
import { genMeidg, isSafeMeidgT, parseMeidg } from '@/misc/id/meidg.js';
import { genObjectId, isSafeObjectIdT, parseObjectId } from '@/misc/id/object-id.js';
import { bindThis } from '@/decorators.js';
import { parseUlid } from '@/misc/id/ulid.js';
@ -26,6 +26,19 @@ export class IdService {
this.method = config.id.toLowerCase();
}
@bindThis
public isSafeT(t: number): boolean {
switch (this.method) {
case 'aid': return isSafeAidT(t);
case 'aidx': return isSafeAidxT(t);
case 'meid': return isSafeMeidT(t);
case 'meidg': return isSafeMeidgT(t);
case 'ulid': return t > 0;
case 'objectid': return isSafeObjectIdT(t);
default: throw new Error('unrecognized id generation method');
}
}
/**
* IDを生成します()
* @param time

View file

@ -54,6 +54,7 @@ import { FeaturedService } from '@/core/FeaturedService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { isReply } from '@/misc/is-reply.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -859,7 +860,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue;
// 「自分自身への返信 or そのフォロワーへの返信」のどちらでもない場合
if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === following.followerId)) {
if (isReply(note, following.followerId)) {
if (!following.withReplies) continue;
}
@ -877,7 +878,7 @@ export class NoteCreateService implements OnApplicationShutdown {
) continue;
// 「自分自身への返信 or そのリストの作成者への返信」のどちらでもない場合
if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === userListMembership.userListUserId)) {
if (isReply(note, userListMembership.userListUserId)) {
if (!userListMembership.withReplies) continue;
}
@ -895,11 +896,14 @@ export class NoteCreateService implements OnApplicationShutdown {
}
// 自分自身以外への返信
if (note.replyId && note.replyUserId !== note.userId) {
if (isReply(note)) {
this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);
if (note.visibility === 'public' && note.userHost == null) {
this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r);
if (note.replyUserHost == null) {
this.fanoutTimelineService.push(`localTimelineWithReplyTo:${note.replyUserId}`, note.id, 300 / 10, r);
}
}
} else {
this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r);

View file

@ -77,7 +77,7 @@ export class NotePiningService {
} as MiUserNotePining);
// Deliver to remote followers
if (this.userEntityService.isLocalUser(user)) {
if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) {
this.deliverPinnedChange(user.id, note.id, true);
}
}
@ -105,7 +105,7 @@ export class NotePiningService {
});
// Deliver to remote followers
if (this.userEntityService.isLocalUser(user)) {
if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) {
this.deliverPinnedChange(user.id, noteId, false);
}
}

View file

@ -12,6 +12,8 @@ import { MiNote } from '@/models/Note.js';
import { MiUser } from '@/models/_.js';
import type { NotesRepository } from '@/models/_.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { CacheService } from '@/core/CacheService.js';
import { QueryService } from '@/core/QueryService.js';
import { IdService } from '@/core/IdService.js';
import type { Index, MeiliSearch } from 'meilisearch';
@ -74,6 +76,7 @@ export class SearchService {
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private cacheService: CacheService,
private queryService: QueryService,
private idService: IdService,
) {
@ -187,8 +190,19 @@ export class SearchService {
limit: pagination.limit,
});
if (res.hits.length === 0) return [];
const notes = await this.notesRepository.findBy({
const [
userIdsWhoMeMuting,
userIdsWhoBlockingMe,
] = me ? await Promise.all([
this.cacheService.userMutingsCache.fetch(me.id),
this.cacheService.userBlockedCache.fetch(me.id),
]) : [new Set<string>(), new Set<string>()];
const notes = (await this.notesRepository.findBy({
id: In(res.hits.map(x => x.id)),
})).filter(note => {
if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
return true;
});
return notes.sort((a, b) => a.id > b.id ? -1 : 1);
} else {

View file

@ -304,8 +304,6 @@ export class UserFollowingService implements OnModuleInit {
});
}
});
this.fanoutTimelineService.purge(`homeTimeline:${follower.id}`);
}
// Publish followed event
@ -373,8 +371,6 @@ export class UserFollowingService implements OnModuleInit {
});
}
});
this.fanoutTimelineService.purge(`homeTimeline:${follower.id}`);
}
if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) {

View file

@ -306,9 +306,15 @@ export class ApInboxService {
this.logger.info(`Creating the (Re)Note: ${uri}`);
const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc);
const createdAt = activity.published ? new Date(activity.published) : null;
if (createdAt && createdAt < this.idService.parse(renote.id).date) {
this.logger.warn('skip: malformed createdAt');
return;
}
await this.noteCreateService.create(actor, {
createdAt: activity.published ? new Date(activity.published) : null,
createdAt,
renote,
visibility: activityAudience.visibility,
visibleUsers: activityAudience.visibleUsers,

View file

@ -92,6 +92,10 @@ export class ApNoteService {
return new Error(`invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`);
}
if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) {
return new Error('invalid Note: published timestamp is malformed');
}
return null;
}