diff --git a/src/queue/index.ts b/src/queue/index.ts
index 70233d8318..a7e9b9814f 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) {
});
}
+export function createCleanRemoteFilesJob() {
+ return objectStorageQueue.add('cleanRemoteFiles', {}, {
+ removeOnComplete: true,
+ removeOnFail: true
+ });
+}
+
export default function() {
if (!program.onlyServer) {
deliverQueue.process(128, processDeliver);
diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts
index 5ecee9139a..a2fd9050a9 100644
--- a/src/queue/processors/db/delete-drive-files.ts
+++ b/src/queue/processors/db/delete-drive-files.ts
@@ -1,7 +1,7 @@
import * as Bull from 'bull';
import { queueLogger } from '../../logger';
-import { deleteFile } from '../../../services/drive/delete-file';
+import { deleteFileSync } from '../../../services/drive/delete-file';
import { Users, DriveFiles } from '../../../models';
import { MoreThan } from 'typeorm';
@@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
cursor = files[files.length - 1].id;
for (const file of files) {
- await deleteFile(file);
+ await deleteFileSync(file);
deletedCount++;
}
diff --git a/src/queue/processors/object-storage/clean-remote-files.ts b/src/queue/processors/object-storage/clean-remote-files.ts
new file mode 100644
index 0000000000..7b34892e1f
--- /dev/null
+++ b/src/queue/processors/object-storage/clean-remote-files.ts
@@ -0,0 +1,50 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { deleteFileSync } from '../../../services/drive/delete-file';
+import { DriveFiles } from '../../../models';
+import { MoreThan, Not, IsNull } from 'typeorm';
+
+const logger = queueLogger.createSubLogger('clean-remote-files');
+
+export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise<void> {
+ logger.info(`Deleting cached remote files...`);
+
+ let deletedCount = 0;
+ let cursor: any = null;
+
+ while (true) {
+ const files = await DriveFiles.find({
+ where: {
+ userHost: Not(IsNull()),
+ isLink: false,
+ ...(cursor ? { id: MoreThan(cursor) } : {})
+ },
+ take: 8,
+ order: {
+ id: 1
+ }
+ });
+
+ if (files.length === 0) {
+ job.progress(100);
+ break;
+ }
+
+ cursor = files[files.length - 1].id;
+
+ await Promise.all(files.map(file => deleteFileSync(file, true)));
+
+ deletedCount += 8;
+
+ const total = await DriveFiles.count({
+ userHost: Not(IsNull()),
+ isLink: false,
+ });
+
+ job.progress(deletedCount / total);
+ }
+
+ logger.succ(`All cahced remote files has been deleted.`);
+ done();
+}
diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts
index 8e6b5f959e..f899df7d2e 100644
--- a/src/queue/processors/object-storage/delete-file.ts
+++ b/src/queue/processors/object-storage/delete-file.ts
@@ -1,22 +1,10 @@
import * as Bull from 'bull';
-import * as Minio from 'minio';
-import { fetchMeta } from '../../../misc/fetch-meta';
+import { deleteObjectStorageFile } from '../../../services/drive/delete-file';
export default async (job: Bull.Job) => {
- const meta = await fetchMeta();
-
- const minio = new Minio.Client({
- endPoint: meta.objectStorageEndpoint!,
- region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
- port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
- useSSL: meta.objectStorageUseSSL,
- accessKey: meta.objectStorageAccessKey!,
- secretKey: meta.objectStorageSecretKey!,
- });
-
const key: string = job.data.key;
- await minio.removeObject(meta.objectStorageBucket!, key);
+ await deleteObjectStorageFile(key);
return 'Success';
};
diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts
index 60f732ca64..33ef665b38 100644
--- a/src/queue/processors/object-storage/index.ts
+++ b/src/queue/processors/object-storage/index.ts
@@ -1,8 +1,10 @@
import * as Bull from 'bull';
import deleteFile from './delete-file';
+import cleanRemoteFiles from './clean-remote-files';
const jobs = {
deleteFile,
+ cleanRemoteFiles,
} as any;
export default function(q: Bull.Queue) {
diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts
index 69cfe0db94..e837ae1bb6 100644
--- a/src/server/api/endpoints/admin/drive/clean-remote-files.ts
+++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts
@@ -1,7 +1,5 @@
-import { Not, IsNull } from 'typeorm';
import define from '../../../define';
-import { deleteFile } from '../../../../../services/drive/delete-file';
-import { DriveFiles } from '../../../../../models';
+import { createCleanRemoteFilesJob } from '../../../../../queue';
export const meta = {
tags: ['admin'],
@@ -11,12 +9,5 @@ export const meta = {
};
export default define(meta, async (ps, me) => {
- const files = await DriveFiles.find({
- userHost: Not(IsNull()),
- isLink: false,
- });
-
- for (const file of files) {
- deleteFile(file, true);
- }
+ createCleanRemoteFilesJob();
});
diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts
index 258c0853fb..5b44a0817a 100644
--- a/src/services/drive/delete-file.ts
+++ b/src/services/drive/delete-file.ts
@@ -1,8 +1,10 @@
+import * as Minio from 'minio';
import { DriveFile } from '../../models/entities/drive-file';
import { InternalStorage } from './internal-storage';
import { DriveFiles, Instances, Notes } from '../../models';
import { driveChart, perUserDriveChart, instanceChart } from '../chart';
import { createDeleteObjectStorageFileJob } from '../../queue';
+import { fetchMeta } from '../../misc/fetch-meta';
export async function deleteFile(file: DriveFile, isExpired = false) {
if (file.storedInternal) {
@@ -27,6 +29,40 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
}
}
+ postProcess(file, isExpired);
+}
+
+export async function deleteFileSync(file: DriveFile, isExpired = false) {
+ if (file.storedInternal) {
+ InternalStorage.del(file.accessKey!);
+
+ if (file.thumbnailUrl) {
+ InternalStorage.del(file.thumbnailAccessKey!);
+ }
+
+ if (file.webpublicUrl) {
+ InternalStorage.del(file.webpublicAccessKey!);
+ }
+ } else if (!file.isLink) {
+ const promises = [];
+
+ promises.push(deleteObjectStorageFile(file.accessKey!));
+
+ if (file.thumbnailUrl) {
+ promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!));
+ }
+
+ if (file.webpublicUrl) {
+ promises.push(deleteObjectStorageFile(file.webpublicAccessKey!));
+ }
+
+ await Promise.all(promises);
+ }
+
+ postProcess(file, isExpired);
+}
+
+function postProcess(file: DriveFile, isExpired = false) {
// リモートファイル期限切れ削除後は直リンクにする
if (isExpired && file.userHost !== null && file.uri != null) {
DriveFiles.update(file.id, {
@@ -53,3 +89,18 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
Instances.decrement({ host: file.userHost }, 'driveFiles', 1);
}
}
+
+export async function deleteObjectStorageFile(key: string) {
+ const meta = await fetchMeta();
+
+ const minio = new Minio.Client({
+ endPoint: meta.objectStorageEndpoint!,
+ region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
+ port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
+ useSSL: meta.objectStorageUseSSL,
+ accessKey: meta.objectStorageAccessKey!,
+ secretKey: meta.objectStorageSecretKey!,
+ });
+
+ await minio.removeObject(meta.objectStorageBucket!, key);
+}