From c277934328e92f9136023a4bb7ff4ee5b960dbc7 Mon Sep 17 00:00:00 2001 From: Kagami Sascha Rosylight Date: Wed, 12 Jul 2023 02:13:27 +0200 Subject: [PATCH] feat(backend): allow disabling cache for sensitive files --- locales/index.d.ts | 2 + locales/ja-JP.yml | 2 + .../migration/1689102832143-nsfw-cache.js | 11 ++ .../core/activitypub/models/ApImageService.ts | 13 +- packages/backend/src/models/entities/Meta.ts | 6 +- .../src/server/api/endpoints/admin/meta.ts | 6 +- .../server/api/endpoints/admin/update-meta.ts | 5 + .../backend/src/server/api/endpoints/meta.ts | 7 +- packages/backend/test/unit/activitypub.ts | 129 +++++++++++++++++- .../frontend/src/pages/admin/settings.vue | 15 +- packages/misskey-js/etc/misskey-js.api.md | 7 +- packages/misskey-js/src/api.types.ts | 1 + packages/misskey-js/src/entities.ts | 1 + 13 files changed, 189 insertions(+), 16 deletions(-) create mode 100644 packages/backend/migration/1689102832143-nsfw-cache.js diff --git a/locales/index.d.ts b/locales/index.d.ts index 42dddebbff..0e03f0a36e 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -158,6 +158,8 @@ export interface Locale { "settingGuide": string; "cacheRemoteFiles": string; "cacheRemoteFilesDescription": string; + "cacheRemoteSensitiveFiles": string; + "cacheRemoteSensitiveFilesDescription": string; "flagAsBot": string; "flagAsBotDescription": string; "flagAsCat": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 16014da511..b36aa00f7d 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -155,6 +155,8 @@ addEmoji: "絵文字を追加" settingGuide: "おすすめ設定" cacheRemoteFiles: "リモートのファイルをキャッシュする" cacheRemoteFilesDescription: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。サーバーのストレージを節約できますが、サムネイルが生成されないので通信量が増加します。" +cacheRemoteSensitiveFiles: "リモートのNSFWファイルをキャッシュする" +cacheRemoteSensitiveFilesDescription: "この設定を無効にすると、リモートのNSFWファイルだけはキャッシュせず直リンクするようになります。" flagAsBot: "Botとして設定" flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。" flagAsCat: "にゃああああああああああああああ！！！！！！！！！！！！" diff --git a/packages/backend/migration/1689102832143-nsfw-cache.js b/packages/backend/migration/1689102832143-nsfw-cache.js new file mode 100644 index 0000000000..cdce0dae09 --- /dev/null +++ b/packages/backend/migration/1689102832143-nsfw-cache.js @@ -0,0 +1,11 @@ +export class NsfwCache1689102832143 { + name = 'NsfwCache1689102832143' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "cacheRemoteSensitiveFiles" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "cacheRemoteSensitiveFiles"`); + } +} diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 0da312241f..1f2984894c 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -1,7 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository } from '@/models/index.js'; -import type { Config } from '@/config.js'; import type { RemoteUser } from '@/models/entities/User.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import { MetaService } from '@/core/MetaService.js'; @@ -20,9 +19,6 @@ export class ApImageService { private logger: Logger; constructor( - @Inject(DI.config) - private config: Config, - @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, @@ -47,7 +43,7 @@ export class ApImageService { const image = await this.apResolverService.createResolver().resolve(value); if (image.url == null) { - throw new Error('invalid image: url not privided'); + throw new Error('invalid image: url not provided'); } if (typeof image.url !== 'string') { @@ -62,12 +58,17 @@ export class ApImageService { const instance = await this.metaService.fetch(); + // Cache if remote file cache is on AND either + // 1. remote sensitive file is also on + // 2. or the image is not sensitive + const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive); + const file = await this.driveService.uploadFromUrl({ url: image.url, user: actor, uri: image.url, sensitive: image.sensitive, - isLink: !instance.cacheRemoteFiles, + isLink: !shouldBeCached, comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH), }); if (!file.isLink || file.url === image.url) return file; diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index a251c0b31c..7bb1b67712 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -1,7 +1,6 @@ import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm'; import { id } from '../id.js'; import { User } from './User.js'; -import type { Clip } from './Clip.js'; @Entity() export class Meta { @@ -126,6 +125,11 @@ export class Meta { }) public cacheRemoteFiles: boolean; + @Column('boolean', { + default: true, + }) + public cacheRemoteSensitiveFiles: boolean; + @Column({ ...id(), nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 28aec7a090..084bdb598b 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -1,5 +1,4 @@ import { Inject, Injectable } from '@nestjs/common'; -import { MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { MetaService } from '@/core/MetaService.js'; import type { Config } from '@/config.js'; @@ -20,6 +19,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + cacheRemoteSensitiveFiles: { + type: 'boolean', + optional: false, nullable: false, + }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -332,6 +335,7 @@ export default class extends Endpoint { enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, cacheRemoteFiles: instance.cacheRemoteFiles, + cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, pinnedUsers: instance.pinnedUsers, hiddenTags: instance.hiddenTags, blockedHosts: instance.blockedHosts, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 0b20b058fd..144360a921 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -43,6 +43,7 @@ export const paramDef = { defaultLightTheme: { type: 'string', nullable: true }, defaultDarkTheme: { type: 'string', nullable: true }, cacheRemoteFiles: { type: 'boolean' }, + cacheRemoteSensitiveFiles: { type: 'boolean' }, emailRequiredForSignup: { type: 'boolean' }, enableHcaptcha: { type: 'boolean' }, hcaptchaSiteKey: { type: 'string', nullable: true }, @@ -193,6 +194,10 @@ export default class extends Endpoint { set.cacheRemoteFiles = ps.cacheRemoteFiles; } + if (ps.cacheRemoteSensitiveFiles !== undefined) { + set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles; + } + if (ps.emailRequiredForSignup !== undefined) { set.emailRequiredForSignup = ps.emailRequiredForSignup; } diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 915a1e54f8..3d0146e315 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -83,6 +83,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + cacheRemoteSensitiveFiles: { + type: 'boolean', + optional: false, nullable: false, + }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -272,7 +276,7 @@ export default class extends Endpoint { .orWhere('ads.dayOfWeek = 0'); })) .getMany(); - + const response: any = { maintainerName: instance.maintainerName, maintainerEmail: instance.maintainerEmail, @@ -329,6 +333,7 @@ export default class extends Endpoint { ...(ps.detail ? { cacheRemoteFiles: instance.cacheRemoteFiles, + cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, requireSetup: (await this.usersRepository.countBy({ host: IsNull(), })) === 0, diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 02b900da9b..78b916c112 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -4,6 +4,7 @@ import * as assert from 'assert'; import { Test } from '@nestjs/testing'; import { jest } from '@jest/globals'; +import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; @@ -11,9 +12,12 @@ import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; -import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js'; -import { Note } from '@/models/index.js'; +import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js'; +import { Meta, Note } from '@/models/index.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; +import { DownloadService } from '@/core/DownloadService.js'; +import { MetaService } from '@/core/MetaService.js'; +import type { RemoteUser } from '@/models/entities/User.js'; import { MockResolver } from '../misc/mock-resolver.js'; const host = 'https://host1.test'; @@ -63,16 +67,47 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe }; } +async function createRandomRemoteUser( + resolver: MockResolver, + personService: ApPersonService, +): Promise { + const actor = createRandomActor(); + resolver.register(actor.id, actor); + + return await personService.createPerson(actor.id, resolver); +} + describe('ActivityPub', () => { + let imageService: ApImageService; let noteService: ApNoteService; let personService: ApPersonService; let rendererService: ApRendererService; let resolver: MockResolver; + const metaInitial = { + cacheRemoteFiles: true, + cacheRemoteSensitiveFiles: true, + blockedHosts: [] as string[], + sensitiveWords: [] as string[], + } as Meta; + let meta = metaInitial; + beforeAll(async () => { const app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - }).compile(); + }) + .overrideProvider(DownloadService).useValue({ + async downloadUrl(): Promise<{ filename: string }> { + return { + filename: 'dummy.tmp', + }; + }, + }) + .overrideProvider(MetaService).useValue({ + async fetch(): Promise { + return meta; + }, + }).compile(); await app.init(); app.enableShutdownHooks(); @@ -80,6 +115,7 @@ describe('ActivityPub', () => { noteService = app.get(ApNoteService); personService = app.get(ApPersonService); rendererService = app.get(ApRendererService); + imageService = app.get(ApImageService); resolver = new MockResolver(await app.resolve(LoggerService)); // Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error @@ -219,4 +255,91 @@ describe('ActivityPub', () => { assert.strictEqual(note.uri, actor2Note.id); }); }); + + describe('Images', () => { + test('Create images', async () => { + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(!driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(!sensitiveDriveFile.isLink); + }); + + test('cacheRemoteFiles=false disables caching', async () => { + meta = { ...metaInitial, cacheRemoteFiles: false }; + + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(sensitiveDriveFile.isLink); + }); + + test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => { + meta = { ...metaInitial, cacheRemoteSensitiveFiles: false }; + + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(!driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(sensitiveDriveFile.isLink); + }); + }); }); diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 4c2fe46f28..bd57c06181 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -37,6 +37,13 @@ + + @@ -104,7 +111,6 @@ import { fetchInstance } from '@/instance'; import { i18n } from '@/i18n'; import { definePageMetadata } from '@/scripts/page-metadata'; import MkButton from '@/components/MkButton.vue'; -import MkColorInput from '@/components/MkColorInput.vue'; let name: string | null = $ref(null); let description: string | null = $ref(null); @@ -112,13 +118,14 @@ let maintainerName: string | null = $ref(null); let maintainerEmail: string | null = $ref(null); let pinnedUsers: string = $ref(''); let cacheRemoteFiles: boolean = $ref(false); +let cacheRemoteSensitiveFiles: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); let deeplAuthKey: string = $ref(''); let deeplIsPro: boolean = $ref(false); -async function init() { +async function init(): Promise { const meta = await os.api('admin/meta'); name = meta.name; description = meta.description; @@ -126,6 +133,7 @@ async function init() { maintainerEmail = meta.maintainerEmail; pinnedUsers = meta.pinnedUsers.join('\n'); cacheRemoteFiles = meta.cacheRemoteFiles; + cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles; enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; @@ -133,7 +141,7 @@ async function init() { deeplIsPro = meta.deeplIsPro; } -function save() { +function save(): void { os.apiWithDialog('admin/update-meta', { name, description, @@ -141,6 +149,7 @@ function save() { maintainerEmail, pinnedUsers: pinnedUsers.split('\n'), cacheRemoteFiles, + cacheRemoteSensitiveFiles, enableServiceWorker, swPublicKey, swPrivateKey, diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index c9b3fd6056..15d924ba78 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -271,6 +271,7 @@ type DetailedInstanceMetadata = LiteInstanceMetadata & { pinnedPages: string[]; pinnedClipId: string | null; cacheRemoteFiles: boolean; + cacheRemoteSensitiveFiles: boolean; requireSetup: boolean; proxyAccountName: string | null; features: Record; @@ -327,6 +328,10 @@ export type Endpoints = { req: TODO; res: TODO; }; + 'admin/meta': { + req: TODO; + res: TODO; + }; 'admin/reset-password': { req: TODO; res: TODO; @@ -2756,7 +2761,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u // // src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts // src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts -// src/api.types.ts:620:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts +// src/api.types.ts:621:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts // src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/packages/misskey-js/src/api.types.ts b/packages/misskey-js/src/api.types.ts index 93f327e67e..305171d87c 100644 --- a/packages/misskey-js/src/api.types.ts +++ b/packages/misskey-js/src/api.types.ts @@ -20,6 +20,7 @@ export type Endpoints = { 'admin/get-table-stats': { req: TODO; res: TODO; }; 'admin/invite': { req: TODO; res: TODO; }; 'admin/logs': { req: TODO; res: TODO; }; + 'admin/meta': { req: TODO; res: TODO; }; 'admin/reset-password': { req: TODO; res: TODO; }; 'admin/resolve-abuse-user-report': { req: TODO; res: TODO; }; 'admin/resync-chart': { req: TODO; res: TODO; }; diff --git a/packages/misskey-js/src/entities.ts b/packages/misskey-js/src/entities.ts index e1305452eb..52f05d4e94 100644 --- a/packages/misskey-js/src/entities.ts +++ b/packages/misskey-js/src/entities.ts @@ -338,6 +338,7 @@ export type DetailedInstanceMetadata = LiteInstanceMetadata & { pinnedPages: string[]; pinnedClipId: string | null; cacheRemoteFiles: boolean; + cacheRemoteSensitiveFiles: boolean; requireSetup: boolean; proxyAccountName: string | null; features: Record;