From 453ba25af2707ff9883c617bd2a3116cab4c67e5 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 1 Jan 2023 17:18:57 +0000 Subject: [PATCH] wip? --- packages/backend/src/core/DownloadService.ts | 21 ++++++-- packages/backend/src/core/FileInfoService.ts | 54 +++++++++++++++---- .../src/core/ImageProcessingService.ts | 37 ++++++++++--- .../backend/src/server/FileServerService.ts | 32 ++++++++--- .../src/server/MediaProxyServerService.ts | 51 ++++++++++++------ 5 files changed, 148 insertions(+), 47 deletions(-) diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 62123246a7..587fbefe64 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -32,8 +32,8 @@ export class DownloadService { } @bindThis - public async downloadUrl(url: string, path: string): Promise { - this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); + public gotUrl(url: string): Got.Request { + this.logger.info(`Downloading ${chalk.cyan(url)} ...`); const timeout = 30 * 1000; const operationTimeout = 60 * 1000; @@ -82,9 +82,16 @@ export class DownloadService { req.destroy(); } }); - + + return req; + } + + @bindThis + public async pipeRequestToFile(req: Got.Request, path: string): Promise { + const copied = req.pipe(new stream.PassThrough()); try { - await pipeline(req, fs.createWriteStream(path)); + this.logger.info(`Saving File to ${chalk.cyanBright(path)} from downloading ...`); + await pipeline(copied, fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); @@ -92,7 +99,11 @@ export class DownloadService { throw e; } } - + } + + @bindThis + public async downloadUrl(url: string, path: string): Promise { + await this.pipeRequestToFile(this.gotUrl(url), path); this.logger.succ(`Download finished: ${chalk.cyan(url)}`); } diff --git a/packages/backend/src/core/FileInfoService.ts b/packages/backend/src/core/FileInfoService.ts index dad94da421..ce3ee67baf 100644 --- a/packages/backend/src/core/FileInfoService.ts +++ b/packages/backend/src/core/FileInfoService.ts @@ -5,7 +5,7 @@ import * as stream from 'node:stream'; import * as util from 'node:util'; import { Inject, Injectable } from '@nestjs/common'; import { FSWatcher } from 'chokidar'; -import { fileTypeFromFile } from 'file-type'; +import { fileTypeFromFile, fileTypeFromStream } from 'file-type'; import FFmpeg from 'fluent-ffmpeg'; import isSvg from 'is-svg'; import probeImageSize from 'probe-image-size'; @@ -15,6 +15,7 @@ import { encode } from 'blurhash'; import { createTempDir } from '@/misc/create-temp.js'; import { AiService } from '@/core/AiService.js'; import { bindThis } from '@/decorators.js'; +import type { Request } from 'got'; const pipeline = util.promisify(stream.pipeline); @@ -39,7 +40,7 @@ const TYPE_OCTET_STREAM = { ext: null, }; -const TYPE_SVG = { +export const TYPE_SVG = { mime: 'image/svg+xml', ext: 'svg', }; @@ -306,9 +307,9 @@ export class FileInfoService { */ @bindThis public async detectType(path: string): Promise<{ - mime: string; - ext: string | null; -}> { + mime: string; + ext: string | null; + }> { // Check 0 byte const fileSize = await this.getFileSize(path); if (fileSize === 0) { @@ -332,21 +333,54 @@ export class FileInfoService { // 種類が不明でもSVGかもしれない if (await this.checkSvg(path)) { return TYPE_SVG; - } + } // それでも種類が不明なら application/octet-stream にする return TYPE_OCTET_STREAM; } + /** + * Detect MIME Type and extension by stream for performance (this cannot detect SVG) + */ + @bindThis + public async detectRequestType(request: Request): Promise<{ + mime: string; + ext: string | null; + }> { + // Check 0 byte + if ((request.response?.complete || request.closed) && !request.response?.rawBody?.length) { + return TYPE_OCTET_STREAM; + } + + const copied = request.pipe(new stream.PassThrough()); + + const type = await fileTypeFromStream(copied); + + if (type) { + return { + mime: type.mime, + ext: type.ext, + }; + } + + // 種類が不明なら application/octet-stream にする + return TYPE_OCTET_STREAM; + } + /** * Check the file is SVG or not */ @bindThis - public async checkSvg(path: string) { + public async checkSvg(target: string | Buffer) { try { - const size = await this.getFileSize(path); - if (size > 1 * 1024 * 1024) return false; - return isSvg(fs.readFileSync(path)); + if (typeof target === 'string') { + const size = await this.getFileSize(target); + if (size > 1 * 1024 * 1024) return false; + return isSvg(await fs.promises.readFile(target)); + } else { + if (target.length > 1 * 1024 * 1024) return false; + return isSvg(target); + } } catch { return false; } diff --git a/packages/backend/src/core/ImageProcessingService.ts b/packages/backend/src/core/ImageProcessingService.ts index 312189eea4..3bfb171ebc 100644 --- a/packages/backend/src/core/ImageProcessingService.ts +++ b/packages/backend/src/core/ImageProcessingService.ts @@ -9,6 +9,14 @@ export type IImage = { type: string; }; +export type IImageStream = { + data: NodeJS.ReadableStream; + ext: string | null; + type: string; +}; + +export type IImageStreamable = IImage | IImageStream; + export const webpDefault: sharp.WebpOptions = { quality: 85, alphaQuality: 95, @@ -62,6 +70,26 @@ export class ImageProcessingService { * Convert to WebP * with resize, remove metadata, resolve orientation, stop animation */ + @bindThis + public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): sharp.Sharp { + return sharp + .resize(width, height, { + fit: 'inside', + withoutEnlargement: true, + }) + .rotate() + .webp(options); + } + + @bindThis + public convertSharpToWebpStreamObj(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream { + return { + data: this.convertSharpToWebpStream(sharp, width, height, options), + ext: 'webp', + type: 'image/webp', + } + } + @bindThis public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { return this.convertSharpToWebp(await sharp(path), width, height, options); @@ -69,14 +97,7 @@ export class ImageProcessingService { @bindThis public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { - const data = await sharp - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .rotate() - .webp(options) - .toBuffer(); + const data = await this.convertSharpToWebpStream(sharp, width, height, options).toBuffer(); return { data, diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 134b3df327..47cc85e257 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -12,14 +12,17 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; import { StatusError } from '@/misc/status-error.js'; import type Logger from '@/logger.js'; import { DownloadService } from '@/core/DownloadService.js'; -import { ImageProcessingService } from '@/core/ImageProcessingService.js'; +import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js'; import { VideoProcessingService } from '@/core/VideoProcessingService.js'; import { InternalStorageService } from '@/core/InternalStorageService.js'; import { contentDisposition } from '@/misc/content-disposition.js'; -import { FileInfoService } from '@/core/FileInfoService.js'; +import { FileInfoService, TYPE_SVG } from '@/core/FileInfoService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; +import { PassThrough } from 'node:stream'; +import sharp from 'sharp'; +import { Request } from 'got'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -106,29 +109,43 @@ export class FileServerService { if (!file.storedInternal) { if (file.isLink && file.uri) { // 期限切れリモートファイル const [path, cleanup] = await createTemp(); + const got = this.downloadService.gotUrl(file.uri);; try { - await this.downloadService.downloadUrl(file.uri, path); + const fileSaving = this.downloadService.pipeRequestToFile(got, path); + const streamCopy = got.pipe(new PassThrough()); - const { mime, ext } = await this.fileInfoService.detectType(path); + let { mime, ext } = await this.fileInfoService.detectRequestType(got); + if (mime === 'application/octet-stream' || mime === 'application/xml') { + await fileSaving; + if (await this.fileInfoService.checkSvg(path)) { + mime = TYPE_SVG.mime; + ext = TYPE_SVG.ext; + } + } const convertFile = async () => { if (isThumbnail) { if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(mime)) { - return await this.imageProcessingService.convertToWebp(path, 498, 280); + return this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 498, 280); } else if (mime.startsWith('video/')) { + await fileSaving; return await this.videoProcessingService.generateVideoThumbnail(path); } } if (isWebpublic) { if (['image/svg+xml'].includes(mime)) { - return await this.imageProcessingService.convertToPng(path, 2048, 2048); + return { + data: this.imageProcessingService.convertSharpToWebpStream(streamCopy.pipe(sharp()), 2048, 2048, { ...webpDefault, lossless: true }), + ext: 'webp', + type: 'image/webp', + }; } } return { - data: fs.readFileSync(path), + data: streamCopy, ext, type: mime, }; @@ -141,6 +158,7 @@ export class FileServerService { } catch (err) { this.logger.error(`${err}`); + if (!got.closed) got.destroy(); if (err instanceof StatusError && err.isClientError) { reply.code(err.statusCode); reply.header('Cache-Control', 'max-age=86400'); diff --git a/packages/backend/src/server/MediaProxyServerService.ts b/packages/backend/src/server/MediaProxyServerService.ts index 4491a17545..0e09ab1820 100644 --- a/packages/backend/src/server/MediaProxyServerService.ts +++ b/packages/backend/src/server/MediaProxyServerService.ts @@ -9,15 +9,17 @@ import type { Config } from '@/config.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; import { createTemp } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; -import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js'; +import { IImageStreamable, ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js'; import type { IImage } from '@/core/ImageProcessingService.js'; import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; import { StatusError } from '@/misc/status-error.js'; import type Logger from '@/logger.js'; -import { FileInfoService } from '@/core/FileInfoService.js'; +import { FileInfoService, TYPE_SVG } from '@/core/FileInfoService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type { FastifyInstance, FastifyPluginOptions, FastifyReply, FastifyRequest } from 'fastify'; +import { PassThrough, pipeline } from 'node:stream'; +import { Request } from 'got'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -73,22 +75,33 @@ export class MediaProxyServerService { // Create temp file const [path, cleanup] = await createTemp(); + const got = this.downloadService.gotUrl(url); try { - await this.downloadService.downloadUrl(url, path); - - const { mime, ext } = await this.fileInfoService.detectType(path); + const fileSaving = this.downloadService.pipeRequestToFile(got, path); + const streamCopy = got.pipe(new PassThrough()); + + let { mime, ext } = await this.fileInfoService.detectRequestType(got); + if (mime === 'application/octet-stream' || mime === 'application/xml') { + await fileSaving; + if (await this.fileInfoService.checkSvg(path)) { + mime = TYPE_SVG.mime; + ext = TYPE_SVG.ext; + } + } const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image'); - let image: IImage; + let image: IImageStreamable; if ('emoji' in request.query && isConvertibleImage) { - const data = await sharp(path, { animated: !('static' in request.query) }) - .resize({ - height: 128, - withoutEnlargement: true, - }) - .webp(webpDefault) - .toBuffer(); + const data = pipeline( + streamCopy, + sharp({ animated: !('static' in request.query) }) + .resize({ + height: 128, + withoutEnlargement: true, + }) + .webp(webpDefault), + ); image = { data, @@ -96,15 +109,17 @@ export class MediaProxyServerService { type: 'image/webp', }; } else if ('static' in request.query && isConvertibleImage) { - image = await this.imageProcessingService.convertToWebp(path, 498, 280); + image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 498, 280); } else if ('preview' in request.query && isConvertibleImage) { - image = await this.imageProcessingService.convertToWebp(path, 200, 200); + image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 200, 200); } else if ('badge' in request.query) { if (!isConvertibleImage) { // 画像でないなら404でお茶を濁す throw new StatusError('Unexpected mime', 404); } + await fileSaving; + const mask = sharp(path) .resize(96, 96, { fit: 'inside', @@ -135,12 +150,12 @@ export class MediaProxyServerService { type: 'image/png', }; } else if (mime === 'image/svg+xml') { - image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, webpDefault); + image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 2048, 2048); } else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) { throw new StatusError('Rejected type', 403, 'Rejected type'); } else { image = { - data: fs.readFileSync(path), + data: streamCopy, ext, type: mime, }; @@ -152,6 +167,8 @@ export class MediaProxyServerService { } catch (err) { this.logger.error(`${err}`); + if (!got.closed) got.destroy(); + if ('fallback' in request.query) { return reply.sendFile('/dummy.png', assets); }