Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

wip

Browse source
This commit is contained in:
syuilo 2023-10-02 02:21:18 +09:00
parent 72f7413f40
commit 167aaabf20
9 changed files with 140 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CHANGELOG.md
View file

@ -12,6 +12,13 @@
-->
## 2023.10.0
### NOTE
- muted_noteテーブルは使われなくなったため手動で削除を行ってください。
### Server
- タイムライン取得時のパフォーマンスを改善
## 2023.9.3
### General
- Enhance: ノートの翻訳機能の利用可否をロールで設定可能に

142
packages/backend/src/core/NoteCreateService.ts
View file

@ -478,16 +478,12 @@ export class NoteCreateService implements OnApplicationShutdown {
// Increment notes count (user)
this.incNotesCountOfUser(user);
if (data.reply) {
if (data.visibility === 'public' || data.visibility === 'home') {
this.pushToTl(note, user);
} else if (data.visibility === 'followers') {
this.pushToTl(note, user);
} else if (data.visibility === 'specified') {
// TODO
} else {
if (data.visibility === 'public' || data.visibility === 'home') {
this.pushToTl(note, user);
} else if (data.visibility === 'followers') {
this.pushToTl(note, user);
} else if (data.visibility === 'specified') {
// TODO
}
}
this.antennaService.addNoteToAntennas(note, user);
@ -802,78 +798,104 @@ export class NoteCreateService implements OnApplicationShutdown {
@bindThis
private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) {
// TODO: 休眠ユーザーを弾く
// TODO: チャンネルフォロー
// TODO: キャッシュ？
const followings = await this.followingsRepository.find({
where: {
followeeId: user.id,
followerHost: IsNull(),
},
select: ['followerId'],
});
let userLists = await this.userListJoiningsRepository.find({
where: {
userId: user.id,
},
select: ['userListId'],
});
const redisPipeline = this.redisClient.pipeline();
// TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする
for (const following of followings) {
if (note.replyId) {
if (note.visibility === 'public' || note.visibility === 'home') {
redisPipeline.xadd(
`userTimelineWithReplies:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
}
} else {
// TODO: 休眠ユーザーを弾く
// TODO: チャンネルフォロー
// TODO: キャッシュ？
const followings = await this.followingsRepository.find({
where: {
followeeId: user.id,
followerHost: IsNull(),
},
select: ['followerId'],
});
let userLists = await this.userListJoiningsRepository.find({
where: {
userId: user.id,
},
select: ['userListId'],
});
// TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする
for (const following of followings) {
redisPipeline.xadd(
`homeTimeline:${following.followerId}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
if (note.fileIds.length > 0) {
redisPipeline.xadd(
`homeTimelineWithFiles:${following.followerId}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
}
}
if (note.visibility === 'followers') {
// TODO: 重そうだから何とかしたい Set 使う？
userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListId));
}
for (const userList of userLists) {
redisPipeline.xadd(
`userListTimeline:${userList.userListId}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
if (note.fileIds.length > 0) {
redisPipeline.xadd(
`userListTimelineWithFiles:${userList.userListId}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
}
}
redisPipeline.xadd(
`homeTimeline:${following.followerId}`,
`homeTimeline:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
if (note.fileIds.length > 0) {
redisPipeline.xadd(
`homeTimelineWithFiles:${following.followerId}`,
`homeTimelineWithFiles:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
}
}
if (note.visibility === 'followers') {
// TODO: 重そうだから何とかしたい Set 使う？
userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListId));
}
for (const userList of userLists) {
redisPipeline.xadd(
`userListTimeline:${userList.userListId}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
if (note.fileIds.length > 0) {
if (note.visibility === 'public' || note.visibility === 'home') {
redisPipeline.xadd(
`userListTimelineWithFiles:${userList.userListId}`,
`userTimeline:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
if (note.fileIds.length > 0) {
redisPipeline.xadd(
`userTimelineWithFiles:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
}
}
}
redisPipeline.xadd(
`homeTimeline:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
if (note.fileIds.length > 0) {
redisPipeline.xadd(
`homeTimelineWithFiles:${user.id}`,
'MAXLEN', '~', '300',
'*',
'note', note.id);
}
redisPipeline.exec();
}

4
packages/backend/src/server/api/EndpointsModule.ts
View file

@ -205,7 +205,6 @@ import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
import * as ep___i_favorites from './endpoints/i/favorites.js';
import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js';
import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js';
import * as ep___i_getWordMutedNotesCount from './endpoints/i/get-word-muted-notes-count.js';
import * as ep___i_importBlocking from './endpoints/i/import-blocking.js';
import * as ep___i_importFollowing from './endpoints/i/import-following.js';
import * as ep___i_importMuting from './endpoints/i/import-muting.js';
@ -554,7 +553,6 @@ const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass:
const $i_favorites: Provider = { provide: 'ep:i/favorites', useClass: ep___i_favorites.default };
const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep___i_gallery_likes.default };
const $i_gallery_posts: Provider = { provide: 'ep:i/gallery/posts', useClass: ep___i_gallery_posts.default };
const $i_getWordMutedNotesCount: Provider = { provide: 'ep:i/get-word-muted-notes-count', useClass: ep___i_getWordMutedNotesCount.default };
const $i_importBlocking: Provider = { provide: 'ep:i/import-blocking', useClass: ep___i_importBlocking.default };
const $i_importFollowing: Provider = { provide: 'ep:i/import-following', useClass: ep___i_importFollowing.default };
const $i_importMuting: Provider = { provide: 'ep:i/import-muting', useClass: ep___i_importMuting.default };
@ -907,7 +905,6 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$i_favorites,
$i_gallery_likes,
$i_gallery_posts,
$i_getWordMutedNotesCount,
$i_importBlocking,
$i_importFollowing,
$i_importMuting,
@ -1254,7 +1251,6 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$i_favorites,
$i_gallery_likes,
$i_gallery_posts,
$i_getWordMutedNotesCount,
$i_importBlocking,
$i_importFollowing,
$i_importMuting,

2
packages/backend/src/server/api/endpoints.ts
View file

@ -205,7 +205,6 @@ import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
import * as ep___i_favorites from './endpoints/i/favorites.js';
import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js';
import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js';
import * as ep___i_getWordMutedNotesCount from './endpoints/i/get-word-muted-notes-count.js';
import * as ep___i_importBlocking from './endpoints/i/import-blocking.js';
import * as ep___i_importFollowing from './endpoints/i/import-following.js';
import * as ep___i_importMuting from './endpoints/i/import-muting.js';
@ -552,7 +551,6 @@ const eps = [
['i/favorites', ep___i_favorites],
['i/gallery/likes', ep___i_gallery_likes],
['i/gallery/posts', ep___i_gallery_posts],
['i/get-word-muted-notes-count', ep___i_getWordMutedNotesCount],
['i/import-blocking', ep___i_importBlocking],
['i/import-following', ep___i_importFollowing],
['i/import-muting', ep___i_importMuting],

8
packages/backend/src/server/api/endpoints/notes/timeline.ts
View file

@ -68,9 +68,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const [
userIdsWhoMeMuting,
userIdsWhoMeMutingRenotes,
userIdsWhoBlockingMe,
] = await Promise.all([
this.cacheService.userMutingsCache.fetch(me.id),
this.cacheService.renoteMutingsCache.fetch(me.id),
this.cacheService.userBlockedCache.fetch(me.id),
]);
let timeline: MiNote[] = [];
@ -103,15 +105,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
timeline = await query.getMany();
// ミュート等考慮
timeline = timeline.filter(note => {
// TODO: インスタンスミュートの考慮
// TODO: ブロックの考慮
if (note.userId === me.id) {
return true;
}
if (isUserRelated(note, userIdsWhoBlockingMe)) return false;
if (isUserRelated(note, userIdsWhoMeMuting)) return false;
if (note.renoteId) {
if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) {

10
packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts
View file

@ -94,9 +94,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const [
userIdsWhoMeMuting,
userIdsWhoMeMutingRenotes,
userIdsWhoBlockingMe,
] = await Promise.all([
this.cacheService.userMutingsCache.fetch(me.id),
this.cacheService.renoteMutingsCache.fetch(me.id),
this.cacheService.userBlockedCache.fetch(me.id),
]);
let timeline: MiNote[] = [];
@ -129,11 +131,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
timeline = await query.getMany();
// ミュート等考慮
timeline = timeline.filter(note => {
// TODO: インスタンスミュートの考慮
// TODO: ブロックの考慮
if (note.userId === me.id) {
return true;
}
if (isUserRelated(note, userIdsWhoBlockingMe)) return false;
if (isUserRelated(note, userIdsWhoMeMuting)) return false;
if (note.renoteId) {
if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) {

109
packages/backend/src/server/api/endpoints/users/notes.ts
View file

@ -5,12 +5,14 @@
import { Brackets } from 'typeorm';
import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/_.js';
import * as Redis from 'ioredis';
import type { MiNote, NotesRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueryService } from '@/core/QueryService.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { DI } from '@/di-symbols.js';
import { GetterService } from '@/server/api/GetterService.js';
import { CacheService } from '@/core/CacheService.js';
import { IdService } from '@/core/IdService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -50,9 +52,6 @@ export const paramDef = {
untilDate: { type: 'integer' },
includeMyRenotes: { type: 'boolean', default: true },
withFiles: { type: 'boolean', default: false },
fileType: { type: 'array', items: {
type: 'string',
} },
excludeNsfw: { type: 'boolean', default: false },
},
required: ['userId'],
@ -61,87 +60,63 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private noteEntityService: NoteEntityService,
private queryService: QueryService,
private getterService: GetterService,
private cacheService: CacheService,
private idService: IdService,
) {
super(meta, paramDef, async (ps, me) => {
// Lookup user
const user = await this.getterService.getUser(ps.userId).catch(err => {
if (err.id === '15348ddd-432d-49c2-8a5a-8069753becff') throw new ApiError(meta.errors.noSuchUser);
throw err;
});
let timeline: MiNote[] = [];
//#region Construct query
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.userId = :userId', { userId: user.id })
const limit = ps.limit + (ps.untilId ? 1 : 0); // untilIdに指定したものも含まれるため+1
let noteIdsRes: [string, string[]][] = [];
if (!ps.sinceId && !ps.sinceDate) {
noteIdsRes = await this.redisClient.xrevrange(
ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : ps.withReplies ? `userTimelineWithReplies:${ps.userId}` : `userTimeline:${ps.userId}`,
ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+',
'-',
'COUNT', limit);
}
const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId);
if (noteIds.length === 0) {
return [];
}
const isFollowing = me ? (await this.cacheService.userFollowingsCache.fetch(me.id)).has(ps.userId) : false;
const query = this.notesRepository.createQueryBuilder('note')
.where('note.id IN (:...noteIds)', { noteIds: noteIds })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
query.andWhere(new Brackets(qb => {
qb.orWhere('note.channelId IS NULL');
qb.orWhere('channel.isSensitive = false');
}));
timeline = await query.getMany();
this.queryService.generateVisibilityQuery(query, me);
if (me) {
this.queryService.generateMutedUserQuery(query, me, user);
this.queryService.generateBlockedUserQuery(query, me);
}
if (ps.withFiles) {
query.andWhere('note.fileIds != \'{}\'');
}
if (ps.fileType != null) {
query.andWhere('note.fileIds != \'{}\'');
query.andWhere(new Brackets(qb => {
for (const type of ps.fileType!) {
const i = ps.fileType!.indexOf(type);
qb.orWhere(`:type${i} = ANY(note.attachedFileTypes)`, { [`type${i}`]: type });
timeline = timeline.filter(note => {
if (note.renoteId) {
if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) {
if (ps.withRenotes === false) return false;
}
}));
if (ps.excludeNsfw) {
query.andWhere('note.cw IS NULL');
query.andWhere('0 = (SELECT COUNT(*) FROM drive_file df WHERE df.id = ANY(note."fileIds") AND df."isSensitive" = TRUE)');
}
}
if (!ps.withReplies) {
query.andWhere('note.replyId IS NULL');
}
if (note.visibility === 'followers' && !isFollowing) return false;
if (ps.withRenotes === false) {
query.andWhere(new Brackets(qb => {
qb.orWhere('note.renoteId IS NULL');
qb.orWhere(new Brackets(qb => {
qb.orWhere('note.text IS NOT NULL');
qb.orWhere('note.fileIds != \'{}\'');
}));
}));
}
return true;
});
if (ps.includeMyRenotes === false) {
query.andWhere(new Brackets(qb => {
qb.orWhere('note.userId != :userId', { userId: user.id });
qb.orWhere('note.renoteId IS NULL');
qb.orWhere('note.text IS NOT NULL');
qb.orWhere('note.fileIds != \'{}\'');
qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)');
}));
}
//#endregion
const timeline = await query.limit(ps.limit).getMany();
timeline.sort((a, b) => a.id > b.id ? -1 : 1);
return await this.noteEntityService.packMany(timeline, me);
});

4
packages/misskey-js/etc/misskey-js.api.md
View file

@ -1381,10 +1381,6 @@ export type Endpoints = {
req: TODO;
res: TODO;
};
'i/get-word-muted-notes-count': {
req: TODO;
res: TODO;
};
'i/import-following': {
req: TODO;
res: TODO;

1
packages/misskey-js/src/api.types.ts
View file

@ -371,7 +371,6 @@ export type Endpoints = {
'i/favorites': { req: { limit?: number; sinceId?: NoteFavorite['id']; untilId?: NoteFavorite['id']; }; res: NoteFavorite[]; };
'i/gallery/likes': { req: TODO; res: TODO; };
'i/gallery/posts': { req: TODO; res: TODO; };
'i/get-word-muted-notes-count': { req: TODO; res: TODO; };
'i/import-following': { req: TODO; res: TODO; };
'i/import-user-lists': { req: TODO; res: TODO; };
'i/move': { req: TODO; res: TODO; };