Merge branch 'develop' into future-2024-04-25-post

packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts

@@ -33,6 +33,12 @@ export class CleanRemoteFilesProcessorService {
 
 		let deletedCount = 0;
 		let cursor: MiDriveFile['id'] | null = null;
+		let errorCount = 0;
+
+		const total = await this.driveFilesRepository.countBy({
+			userHost: Not(IsNull()),
+			isLink: false,
+		});
 
 		while (true) {
 			const files = await this.driveFilesRepository.find({
@@ -41,7 +47,7 @@ export class CleanRemoteFilesProcessorService {
 					isLink: false,
 					...(cursor ? { id: MoreThan(cursor) } : {}),
 				},
-				take: 8,
+				take: 256,
 				order: {
 					id: 1,
 				},
@@ -54,18 +60,22 @@ export class CleanRemoteFilesProcessorService {
 
 			cursor = files.at(-1)?.id ?? null;
 
-			await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true)));
+			// Handle deletion in a batch
+			const results = await Promise.allSettled(files.map(file => this.driveService.deleteFileSync(file, true)));
 
-			deletedCount += 8;
-
-			const total = await this.driveFilesRepository.countBy({
-				userHost: Not(IsNull()),
-				isLink: false,
+			results.forEach((result, index) => {
+				if (result.status === 'fulfilled') {
+					deletedCount++;
+				} else {
+					this.logger.error(`Failed to delete file ID ${files[index].id}: ${result.reason}`);
+					errorCount++;
+				}
 			});
 
-			job.updateProgress(100 / total * deletedCount);
+			await job.updateProgress(100 / total * deletedCount);
+
 		}
 
-		this.logger.succ('All cached remote files has been deleted.');
+		this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`);
 	}
 }

packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts

@@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
 		});
 
 		for (const emoji of customEmojis) {
-			if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) {
+			if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
 				this.logger.error(`invalid emoji name: ${emoji.name}`);
 				continue;
 			}

packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts

@@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
 					continue;
 				}
 				const emojiInfo = record.emoji;
-				if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) {
-					this.logger.error(`invalid emojiname: ${emojiInfo.name}`);
+				const nameNfc = emojiInfo.name.normalize('NFC');
+				if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
+					this.logger.error(`invalid emojiname: ${nameNfc}`);
 					continue;
 				}
 				const emojiPath = outputPath + '/' + record.fileName;
 				await this.emojisRepository.delete({
-					name: emojiInfo.name,
+					name: nameNfc,
 				});
 				const driveFile = await this.driveService.addFile({
 					user: null,
@@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
 					force: true,
 				});
 				await this.customEmojiService.add({
-					name: emojiInfo.name,
-					category: emojiInfo.category,
+					name: nameNfc,
+					category: emojiInfo.category?.normalize('NFC'),
 					host: null,
-					aliases: emojiInfo.aliases,
+					aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
 					driveFile,
 					license: emojiInfo.license,
 					isSensitive: emojiInfo.isSensitive,

packages/backend/src/queue/processors/InboxProcessorService.ts

@@ -15,6 +15,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
 import ApRequestChart from '@/core/chart/charts/ap-request.js';
 import FederationChart from '@/core/chart/charts/federation.js';
 import { getApId } from '@/core/activitypub/type.js';
+import type { IActivity } from '@/core/activitypub/type.js';
 import type { MiRemoteUser } from '@/models/User.js';
 import type { MiUserPublickey } from '@/models/UserPublickey.js';
 import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
@@ -52,7 +53,7 @@ export class InboxProcessorService {
 	@bindThis
 	public async process(job: Bull.Job<InboxJobData>): Promise<string> {
 		const signature = job.data.signature;	// HTTP-signature
-		const activity = job.data.activity;
+		let activity = job.data.activity;
 
 		//#region Log
 		const info = Object.assign({}, activity);
@@ -150,6 +151,17 @@ export class InboxProcessorService {
 					throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
 				}
 
+				// アクティビティを正規化
+				delete activity.signature;
+				try {
+					activity = await ldSignature.compact(activity) as IActivity;
+				} catch (e) {
+					throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`);
+				}
+				// TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする
+				// https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29
+				activity.signature = ldSignature;
+
 				// もう一度actorチェック
 				if (authUser.user.uri !== activity.actor) {
 					throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);