enhance(server): videoThumbnailGenerator config (#9845)

* enhance(server): videoThumbnailGenerator config

* ✌️

* fix

* 相対url

* サムネイルのproxyRemoteFilesは直接プロキシを指定する

* メディアプロキシ

.config/example.yml

@@ -131,11 +131,20 @@ proxyBypassHosts:
 
 # Media Proxy
 # Reference Implementation: https://github.com/misskey-dev/media-proxy
+# * Deliver a common cache between instances
+# * Perform image compression (on a different server resource than the main process)
 #mediaProxy: https://example.com/proxy
 
 # Proxy remote files (default: false)
+# Proxy remote files by this instance or mediaProxy to prevent remote files from running in remote domains.
 #proxyRemoteFiles: true
 
+# Movie Thumbnail Generation URL
+# There is no reference implementation.
+# For example, Misskey will point to the following URL:
+#   https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4
+#videoThumbnailGenerator: https://example.com
+
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 

packages/backend/src/config.ts

@@ -67,6 +67,7 @@ export type Source = {
 
 	mediaProxy?: string;
 	proxyRemoteFiles?: boolean;
+	videoThumbnailGenerator?: string;
 
 	signToActivityPubGet?: boolean;
 };
@@ -89,6 +90,7 @@ export type Mixin = {
 	clientManifestExists: boolean;
 	mediaProxy: string;
 	externalMediaProxyEnabled: boolean;
+	videoThumbnailGenerator: string | null;
 };
 
 export type Config = Source & Mixin;
@@ -144,6 +146,10 @@ export function loadConfig() {
 	mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy;
 	mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy;
 
+	mixin.videoThumbnailGenerator = config.videoThumbnailGenerator ?
+		config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
+		: null;
+
 	if (!config.redis.prefix) config.redis.prefix = mixin.host;
 
 	return Object.assign(config, mixin);

packages/backend/src/core/DriveService.ts

@@ -250,6 +250,14 @@ export class DriveService {
 	@bindThis
 	public async generateAlts(path: string, type: string, generateWeb: boolean) {
 		if (type.startsWith('video/')) {
+			if (this.config.videoThumbnailGenerator != null) {
+				// videoThumbnailGeneratorが指定されていたら動画サムネイル生成はスキップ
+				return {
+					webpublic: null,
+					thumbnail: null,
+				}
+			}
+
 			try {
 				const thumbnail = await this.videoProcessingService.generateVideoThumbnail(path);
 				return {

packages/backend/src/core/VideoProcessingService.ts

@@ -6,6 +6,7 @@ import { ImageProcessingService } from '@/core/ImageProcessingService.js';
 import type { IImage } from '@/core/ImageProcessingService.js';
 import { createTempDir } from '@/misc/create-temp.js';
 import { bindThis } from '@/decorators.js';
+import { appendQuery, query } from '@/misc/prelude/url.js';
 
 @Injectable()
 export class VideoProcessingService {
@@ -41,5 +42,18 @@ export class VideoProcessingService {
 			cleanup();
 		}
 	}
+
+	@bindThis
+	public getExternalVideoThumbnailUrl(url: string): string | null {
+		if (this.config.videoThumbnailGenerator == null) return null;
+
+		return appendQuery(
+			`${this.config.videoThumbnailGenerator}/thumbnail.webp`,
+			query({
+				thumbnail: '1',
+				url,
+			})
+		)
+	}
 }
 

packages/backend/src/core/entities/DriveFileEntityService.ts

@@ -13,6 +13,7 @@ import { deepClone } from '@/misc/clone.js';
 import { UtilityService } from '../UtilityService.js';
 import { UserEntityService } from './UserEntityService.js';
 import { DriveFolderEntityService } from './DriveFolderEntityService.js';
+import { VideoProcessingService } from '../VideoProcessingService.js';
 
 type PackOptions = {
 	detail?: boolean,
@@ -43,6 +44,7 @@ export class DriveFileEntityService {
 
 		private utilityService: UtilityService,
 		private driveFolderEntityService: DriveFolderEntityService,
+		private videoProcessingService: VideoProcessingService,
 	) {
 	}
 	
@@ -72,40 +74,63 @@ export class DriveFileEntityService {
 	}
 
 	@bindThis
-	public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail
+	private getProxiedUrl(url: string, mode?: 'static' | 'avatar'): string | null {
-		const proxiedUrl = (url: string) => appendQuery(
+		return appendQuery(
 			`${this.config.mediaProxy}/${mode ?? 'image'}.webp`,
 			query({
 				url,
 				...(mode ? { [mode]: '1' } : {}),
 			})
-		);
+		)
+	}
 
+	@bindThis
+	public getThumbnailUrl(file: DriveFile): string | null {
+		if (file.type.startsWith('video')) {
+			if (file.thumbnailUrl) return file.thumbnailUrl;
+
+			if (this.config.videoThumbnailGenerator == null) {
+				return this.videoProcessingService.getExternalVideoThumbnailUrl(file.webpublicUrl ?? file.url ?? file.uri);
+			}
+		} else if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
+			// 動画ではなくリモートかつメディアプロキシ
+			return this.getProxiedUrl(file.uri, 'static');
+		}
+
+		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
+			// リモートかつ期限切れはローカルプロキシを試みる
+			// 従来は/files/${thumbnailAccessKey}にアクセスしていたが、
+			// /filesはメディアプロキシにリダイレクトするようにしたため直接メディアプロキシを指定する
+			return this.getProxiedUrl(file.uri, 'static');
+		}
+
+		const url = file.webpublicUrl ?? file.url;
+
+		return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? this.getProxiedUrl(url, 'static') : null);
+	}
+
+	@bindThis
+	public getPublicUrl(file: DriveFile, mode?: 'avatar'): string | null { // static = thumbnail
 		// リモートかつメディアプロキシ
 		if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
-			if (!(mode === 'static' && file.type.startsWith('video'))) {
+			return this.getProxiedUrl(file.uri, mode);
-				return proxiedUrl(file.uri);
-			}
 		}
 
 		// リモートかつ期限切れはローカルプロキシを試みる
 		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
-			const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey;
+			const key = file.webpublicAccessKey;
 
 			if (key && !key.match('/')) {	// 古いものはここにオブジェクトストレージキーが入ってるので除外
 				const url = `${this.config.url}/files/${key}`;
-				if (mode === 'avatar') return proxiedUrl(file.uri);
+				if (mode === 'avatar') return this.getProxiedUrl(file.uri, 'avatar');
 				return url;
 			}
 		}
 
 		const url = file.webpublicUrl ?? file.url;
 
-		if (mode === 'static') {
-			return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? proxiedUrl(url) : null);
-		}
 		if (mode === 'avatar') {
-			return proxiedUrl(url);
+			return this.getProxiedUrl(url, 'avatar');
 		}
 		return url;
 	}
@@ -183,7 +208,7 @@ export class DriveFileEntityService {
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
 			url: opts.self ? file.url : this.getPublicUrl(file),
-			thumbnailUrl: this.getPublicUrl(file, 'static'),
+			thumbnailUrl: this.getThumbnailUrl(file),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {
@@ -218,7 +243,7 @@ export class DriveFileEntityService {
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
 			url: opts.self ? file.url : this.getPublicUrl(file),
-			thumbnailUrl: this.getPublicUrl(file, 'static'),
+			thumbnailUrl: this.getThumbnailUrl(file),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {

packages/backend/src/server/FileServerService.ts

@@ -150,6 +150,12 @@ export class FileServerService {
 						file.cleanup();
 						return await reply.redirect(301, url.toString());
 					} else if (file.mime.startsWith('video/')) {
+						const externalThumbnail = this.videoProcessingService.getExternalVideoThumbnailUrl(file.url);
+						if (externalThumbnail) {
+							file.cleanup();
+							return await reply.redirect(301, externalThumbnail);
+						}
+
 						image = await this.videoProcessingService.generateVideoThumbnail(file.path);
 					}
 				}