merge: Add filter options to following feed (resolves #726) (!671)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/671

Closes #726

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>
This commit is contained in:
Hazelnoot 2024-10-15 21:50:26 +00:00
commit de9b99c937
20 changed files with 478 additions and 104 deletions

View file

@ -0,0 +1,24 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class TrackLatestNoteType1728420772835 {
name = 'TrackLatestNoteType1728420772835'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "latest_note" DROP CONSTRAINT "PK_f619b62bfaafabe68f52fb50c9a"`);
await queryRunner.query(`ALTER TABLE "latest_note" ADD "is_public" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "latest_note" ADD "is_reply" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "latest_note" ADD "is_quote" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "latest_note" ADD CONSTRAINT "PK_a44ac8ca9cb916faeefc0912abd" PRIMARY KEY ("user_id", is_public, is_reply, is_quote)`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "latest_note" DROP CONSTRAINT "PK_a44ac8ca9cb916faeefc0912abd"`);
await queryRunner.query(`ALTER TABLE "latest_note" DROP COLUMN is_quote`);
await queryRunner.query(`ALTER TABLE "latest_note" DROP COLUMN is_reply`);
await queryRunner.query(`ALTER TABLE "latest_note" DROP COLUMN is_public`);
await queryRunner.query(`ALTER TABLE "latest_note" ADD CONSTRAINT "PK_f619b62bfaafabe68f52fb50c9a" PRIMARY KEY ("user_id")`);
}
}

View file

@ -14,7 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf
import { extractHashtags } from '@/misc/extract-hashtags.js';
import type { IMentionedRemoteUsers } from '@/models/Note.js';
import { MiNote } from '@/models/Note.js';
import { LatestNote } from '@/models/LatestNote.js';
import { SkLatestNote } from '@/models/LatestNote.js';
import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, LatestNotesRepository, MiFollowing, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
import type { MiApp } from '@/models/App.js';
@ -63,7 +63,7 @@ import { isReply } from '@/misc/is-reply.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { isPureRenote } from '@/misc/is-renote.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -1151,18 +1151,21 @@ export class NoteCreateService implements OnApplicationShutdown {
if (note.visibility === 'specified') return;
// Ignore pure renotes
if (isRenote(note) && !isQuote(note)) return;
if (isPureRenote(note)) return;
// Compute the compound key of the entry to check
const key = SkLatestNote.keyFor(note);
// Make sure that this isn't an *older* post.
// We can get older posts through replies, lookups, etc.
const currentLatest = await this.latestNotesRepository.findOneBy({ userId: note.userId });
const currentLatest = await this.latestNotesRepository.findOneBy(key);
if (currentLatest != null && currentLatest.noteId >= note.id) return;
// Record this as the latest note for the given user
const latestNote = new LatestNote({
userId: note.userId,
const latestNote = new SkLatestNote({
...key,
noteId: note.id,
});
await this.latestNotesRepository.upsert(latestNote, ['userId']);
await this.latestNotesRepository.upsert(latestNote, ['userId', 'isPublic', 'isReply', 'isQuote']);
}
}

View file

@ -6,8 +6,8 @@
import { Brackets, In, Not } from 'typeorm';
import { Injectable, Inject } from '@nestjs/common';
import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { MiNote, IMentionedRemoteUsers } from '@/models/Note.js';
import { LatestNote } from '@/models/LatestNote.js';
import { MiNote, IMentionedRemoteUsers } from '@/models/Note.js';
import { SkLatestNote } from '@/models/LatestNote.js';
import type { InstancesRepository, LatestNotesRepository, NotesRepository, UsersRepository } from '@/models/_.js';
import { RelayService } from '@/core/RelayService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
@ -25,7 +25,7 @@ import { bindThis } from '@/decorators.js';
import { MetaService } from '@/core/MetaService.js';
import { SearchService } from '@/core/SearchService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js';
@Injectable()
export class NoteDeleteService {
@ -240,8 +240,14 @@ export class NoteDeleteService {
// If it's a DM, then it can't possibly be the latest note so we can safely skip this.
if (note.visibility === 'specified') return;
// If it's a pure renote, then it can't possibly be the latest note so we can safely skip this.
if (isPureRenote(note)) return;
// Compute the compound key of the entry to check
const key = SkLatestNote.keyFor(note);
// Check if the deleted note was possibly the latest for the user
const hasLatestNote = await this.latestNotesRepository.existsBy({ userId: note.userId });
const hasLatestNote = await this.latestNotesRepository.existsBy(key);
if (hasLatestNote) return;
// Find the newest remaining note for the user.
@ -250,8 +256,16 @@ export class NoteDeleteService {
.createQueryBuilder('note')
.select()
.where({
userId: note.userId,
visibility: Not('specified'),
userId: key.userId,
visibility: key.isPublic
? 'public'
: Not('specified'),
replyId: key.isReply
? Not(null)
: null,
renoteId: key.isQuote
? Not(null)
: null,
})
.andWhere(`
(
@ -268,8 +282,8 @@ export class NoteDeleteService {
if (!nextLatest) return;
// Record it as the latest
const latestNote = new LatestNote({
userId: note.userId,
const latestNote = new SkLatestNote({
...key,
noteId: nextLatest.id,
});
@ -278,7 +292,7 @@ export class NoteDeleteService {
await this.latestNotesRepository
.createQueryBuilder('latest')
.insert()
.into(LatestNote)
.into(SkLatestNote)
.values(latestNote)
.orIgnore()
.execute();

View file

@ -23,6 +23,17 @@ type Quote =
hasPoll: true
});
type PureRenote =
Renote & {
text: null,
cw: null,
replyId: null,
hasPoll: false,
fileIds: {
length: 0,
},
};
export function isRenote(note: MiNote): note is Renote {
return note.renoteId != null;
}
@ -36,6 +47,10 @@ export function isQuote(note: Renote): note is Quote {
note.fileIds.length > 0;
}
export function isPureRenote(note: MiNote): note is PureRenote {
return isRenote(note) && !isQuote(note);
}
type PackedRenote =
Packed<'Note'> & {
renoteId: NonNullable<Packed<'Note'>['renoteId']>

View file

@ -6,6 +6,7 @@
import { PrimaryColumn, Entity, JoinColumn, Column, ManyToOne } from 'typeorm';
import { MiUser } from '@/models/User.js';
import { MiNote } from '@/models/Note.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
/**
* Maps a user to the most recent post by that user.
@ -13,7 +14,7 @@ import { MiNote } from '@/models/Note.js';
* DMs are not counted.
*/
@Entity('latest_note')
export class LatestNote {
export class SkLatestNote {
@PrimaryColumn({
name: 'user_id',
type: 'varchar' as const,
@ -21,6 +22,24 @@ export class LatestNote {
})
public userId: string;
@PrimaryColumn('boolean', {
name: 'is_public',
default: false,
})
public isPublic: boolean;
@PrimaryColumn('boolean', {
name: 'is_reply',
default: false,
})
public isReply: boolean;
@PrimaryColumn('boolean', {
name: 'is_quote',
default: false,
})
public isQuote: boolean;
@ManyToOne(() => MiUser, {
onDelete: 'CASCADE',
})
@ -44,11 +63,23 @@ export class LatestNote {
})
public note: MiNote | null;
constructor(data?: Partial<LatestNote>) {
constructor(data?: Partial<SkLatestNote>) {
if (!data) return;
for (const [k, v] of Object.entries(data)) {
(this as Record<string, unknown>)[k] = v;
}
}
/**
* Generates a compound key matching a provided note.
*/
static keyFor(note: MiNote) {
return {
userId: note.userId,
isPublic: note.visibility === 'public',
isReply: note.replyId != null,
isQuote: isRenote(note) && isQuote(note),
};
}
}

View file

@ -7,7 +7,7 @@ import type { Provider } from '@nestjs/common';
import { Module } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import {
LatestNote,
SkLatestNote,
MiAbuseReportNotificationRecipient,
MiAbuseUserReport,
MiAccessToken,
@ -121,7 +121,7 @@ const $avatarDecorationsRepository: Provider = {
const $latestNotesRepository: Provider = {
provide: DI.latestNotesRepository,
useFactory: (db: DataSource) => db.getRepository(LatestNote).extend(miRepository as MiRepository<LatestNote>),
useFactory: (db: DataSource) => db.getRepository(SkLatestNote).extend(miRepository as MiRepository<SkLatestNote>),
inject: [DI.db],
};

View file

@ -10,7 +10,7 @@ import { RelationIdLoader } from 'typeorm/query-builder/relation-id/RelationIdLo
import { RawSqlResultsToEntityTransformer } from 'typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer.js';
import { ObjectUtils } from 'typeorm/util/ObjectUtils.js';
import { OrmUtils } from 'typeorm/util/OrmUtils.js';
import { LatestNote } from '@/models/LatestNote.js';
import { SkLatestNote } from '@/models/LatestNote.js';
import { MiAbuseUserReport } from '@/models/AbuseUserReport.js';
import { MiAbuseReportNotificationRecipient } from '@/models/AbuseReportNotificationRecipient.js';
import { MiAccessToken } from '@/models/AccessToken.js';
@ -127,7 +127,7 @@ export const miRepository = {
} satisfies MiRepository<ObjectLiteral>;
export {
LatestNote,
SkLatestNote,
MiAbuseUserReport,
MiAbuseReportNotificationRecipient,
MiAccessToken,
@ -226,7 +226,7 @@ export type GalleryPostsRepository = Repository<MiGalleryPost> & MiRepository<Mi
export type HashtagsRepository = Repository<MiHashtag> & MiRepository<MiHashtag>;
export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>;
export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>;
export type LatestNotesRepository = Repository<LatestNote> & MiRepository<LatestNote>;
export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>;
export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>;
export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>;
export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>;

View file

@ -83,7 +83,7 @@ import { MiReversiGame } from '@/models/ReversiGame.js';
import { Config } from '@/config.js';
import MisskeyLogger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { LatestNote } from '@/models/LatestNote.js';
import { SkLatestNote } from '@/models/LatestNote.js';
pg.types.setTypeParser(20, Number);
@ -131,7 +131,7 @@ class MyCustomLogger implements Logger {
}
export const entities = [
LatestNote,
SkLatestNote,
MiAnnouncement,
MiAnnouncementRead,
MiMeta,

View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { LatestNote, MiFollowing } from '@/models/_.js';
import { SkLatestNote, MiFollowing } from '@/models/_.js';
import type { NotesRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
@ -33,6 +33,12 @@ export const paramDef = {
type: 'object',
properties: {
mutualsOnly: { type: 'boolean', default: false },
filesOnly: { type: 'boolean', default: false },
includeNonPublic: { type: 'boolean', default: false },
includeReplies: { type: 'boolean', default: false },
includeQuotes: { type: 'boolean', default: false },
includeBots: { type: 'boolean', default: true },
limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
sinceId: { type: 'string', format: 'misskey:id' },
untilId: { type: 'string', format: 'misskey:id' },
@ -52,12 +58,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private queryService: QueryService,
) {
super(meta, paramDef, async (ps, me) => {
let query = this.notesRepository
const query = this.notesRepository
.createQueryBuilder('note')
.setParameter('me', me.id)
// Limit to latest notes
.innerJoin(LatestNote, 'latest', 'note.id = latest.note_id')
.innerJoin(SkLatestNote, 'latest', 'note.id = latest.note_id')
// Avoid N+1 queries from the "pack" method
.innerJoinAndSelect('note.user', 'user')
@ -73,8 +79,28 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// Limit to mutuals, if requested
if (ps.mutualsOnly) {
query = query
.innerJoin(MiFollowing, 'mutuals', 'latest.user_id = mutuals."followerId" AND mutuals."followeeId" = :me');
query.innerJoin(MiFollowing, 'mutuals', 'latest.user_id = mutuals."followerId" AND mutuals."followeeId" = :me');
}
// Limit to files, if requested
if (ps.filesOnly) {
query.andWhere('note."fileIds" != \'{}\'');
}
// Match selected note types.
if (!ps.includeNonPublic) {
query.andWhere('latest.is_public');
}
if (!ps.includeReplies) {
query.andWhere('latest.is_reply = false');
}
if (!ps.includeQuotes) {
query.andWhere('latest.is_quote = false');
}
// Match selected user types.
if (!ps.includeBots) {
query.andWhere('"user"."isBot" = false');
}
// Respect blocks and mutes
@ -82,7 +108,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
this.queryService.generateMutedUserQuery(query, me);
// Support pagination
query = this.queryService
this.queryService
.makePaginationQuery(query, ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.orderBy('note.id', 'DESC')
.take(ps.limit);

View file

@ -17,6 +17,7 @@ import { MiLocalUser } from '@/models/User.js';
import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js';
import { FanoutTimelineName } from '@/core/FanoutTimelineService.js';
import { ApiError } from '@/server/api/error.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
export const meta = {
tags: ['users', 'notes'],
@ -51,7 +52,11 @@ export const paramDef = {
properties: {
userId: { type: 'string', format: 'misskey:id' },
withReplies: { type: 'boolean', default: false },
withRepliesToSelf: { type: 'boolean', default: true },
withQuotes: { type: 'boolean', default: true },
withRenotes: { type: 'boolean', default: true },
withBots: { type: 'boolean', default: true },
withNonPublic: { type: 'boolean', default: true },
withChannelNotes: { type: 'boolean', default: false },
limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
sinceId: { type: 'string', format: 'misskey:id' },
@ -103,6 +108,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
withChannelNotes: ps.withChannelNotes,
withFiles: ps.withFiles,
withRenotes: ps.withRenotes,
withQuotes: ps.withQuotes,
withBots: ps.withBots,
withNonPublic: ps.withNonPublic,
withRepliesToOthers: ps.withReplies,
withRepliesToSelf: ps.withRepliesToSelf,
}, me);
return await this.noteEntityService.packMany(timeline, me);
@ -127,11 +137,17 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
excludeReplies: ps.withChannelNotes && !ps.withReplies, // userTimelineWithChannel may include replies
excludeNoFiles: ps.withChannelNotes && ps.withFiles, // userTimelineWithChannel may include notes without files
excludePureRenotes: !ps.withRenotes,
excludeBots: !ps.withBots,
noteFilter: note => {
if (note.channel?.isSensitive && !isSelf) return false;
if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false;
if (note.visibility === 'followers' && !isFollowing && !isSelf) return false;
// These are handled by DB fallback, but we duplicate them here in case a timeline was already populated with notes
if (!ps.withRepliesToSelf && note.reply?.userId === note.userId) return false;
if (!ps.withQuotes && isRenote(note) && isQuote(note)) return false;
if (!ps.withNonPublic && note.visibility !== 'public') return false;
return true;
},
dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({
@ -142,6 +158,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
withChannelNotes: ps.withChannelNotes,
withFiles: ps.withFiles,
withRenotes: ps.withRenotes,
withQuotes: ps.withQuotes,
withBots: ps.withBots,
withNonPublic: ps.withNonPublic,
withRepliesToOthers: ps.withReplies,
withRepliesToSelf: ps.withRepliesToSelf,
}, me),
});
@ -157,6 +178,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
withChannelNotes: boolean,
withFiles: boolean,
withRenotes: boolean,
withQuotes: boolean,
withBots: boolean,
withNonPublic: boolean,
withRepliesToOthers: boolean,
withRepliesToSelf: boolean,
}, me: MiLocalUser | null) {
const isSelf = me && (me.id === ps.userId);
@ -188,7 +214,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
query.andWhere('note.fileIds != \'{}\'');
}
if (ps.withRenotes === false) {
if (!ps.withRenotes && !ps.withQuotes) {
query.andWhere('note.renoteId IS NULL');
} else if (!ps.withRenotes) {
query.andWhere(new Brackets(qb => {
qb.orWhere('note.userId != :userId', { userId: ps.userId });
qb.orWhere('note.renoteId IS NULL');
@ -196,6 +224,35 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
qb.orWhere('note.fileIds != \'{}\'');
qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)');
}));
} else if (!ps.withQuotes) {
query.andWhere(`
(
note."renoteId" IS NULL
OR (
note.text IS NULL
AND note.cw IS NULL
AND note."replyId" IS NULL
AND note."hasPoll" IS FALSE
AND note."fileIds" = '{}'
)
)
`);
}
if (!ps.withRepliesToOthers && !ps.withRepliesToSelf) {
query.andWhere('reply.id IS NULL');
} else if (!ps.withRepliesToOthers) {
query.andWhere('(reply.id IS NULL OR reply."userId" = note."userId")');
} else if (!ps.withRepliesToSelf) {
query.andWhere('(reply.id IS NULL OR reply."userId" != note."userId")');
}
if (!ps.withNonPublic) {
query.andWhere('note.visibility = \'public\'');
}
if (!ps.withBots) {
query.andWhere('"user"."isBot" = false');
}
return await query.limit(ps.limit).getMany();

View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js';
import { MiNote } from '@/models/Note.js';
const base: MiNote = {
@ -86,4 +86,24 @@ describe('misc:is-renote', () => {
expect(isRenote(note)).toBe(true);
expect(isQuote(note as any)).toBe(true);
});
describe('isPureRenote', () => {
it('should return true when note is pure renote', () => {
const note = new MiNote({ renoteId: 'abc123', fileIds: [] });
const result = isPureRenote(note);
expect(result).toBeTruthy();
});
it('should return false when note is quote', () => {
const note = new MiNote({ renoteId: 'abc123', text: 'text', fileIds: [] });
const result = isPureRenote(note);
expect(result).toBeFalsy();
});
it('should return false when note is not renote', () => {
const note = new MiNote({ renoteId: null, fileIds: [] });
const result = isPureRenote(note);
expect(result).toBeFalsy();
});
});
});

View file

@ -0,0 +1,66 @@
import { SkLatestNote } from '@/models/LatestNote.js';
import { MiNote } from '@/models/Note.js';
describe(SkLatestNote, () => {
describe('keyFor', () => {
it('should include userId', () => {
const note = new MiNote({ userId: 'abc123', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.userId).toBe(note.userId);
});
it('should include isPublic when is public', () => {
const note = new MiNote({ visibility: 'public', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isPublic).toBeTruthy();
});
it('should include isPublic when is home-only', () => {
const note = new MiNote({ visibility: 'home', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isPublic).toBeFalsy();
});
it('should include isPublic when is followers-only', () => {
const note = new MiNote({ visibility: 'followers', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isPublic).toBeFalsy();
});
it('should include isPublic when is specified', () => {
const note = new MiNote({ visibility: 'specified', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isPublic).toBeFalsy();
});
it('should include isReply when is reply', () => {
const note = new MiNote({ replyId: 'abc123', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isReply).toBeTruthy();
});
it('should include isReply when is not reply', () => {
const note = new MiNote({ replyId: null, fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isReply).toBeFalsy();
});
it('should include isQuote when is quote', () => {
const note = new MiNote({ renoteId: 'abc123', text: 'text', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isQuote).toBeTruthy();
});
it('should include isQuote when is reblog', () => {
const note = new MiNote({ renoteId: 'abc123', fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isQuote).toBeFalsy();
});
it('should include isQuote when is neither quote nor reblog', () => {
const note = new MiNote({ renoteId: null, fileIds: [] });
const key = SkLatestNote.keyFor(note);
expect(key.isQuote).toBeFalsy();
});
});
});