From 12afc294f7b6cf4abc6a900fe4d7bda11116960f Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 6 Jan 2023 16:29:45 +0000 Subject: [PATCH] wip???? --- packages/backend/src/core/DownloadService.ts | 16 ++++++-- packages/backend/src/core/FileInfoService.ts | 17 ++++---- .../backend/src/server/FileServerService.ts | 16 ++++---- .../src/server/MediaProxyServerService.ts | 40 +++++++++---------- 4 files changed, 48 insertions(+), 41 deletions(-) diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 0eac15c161..a440ac3c85 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -14,10 +14,13 @@ import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { buildConnector } from 'undici'; +import type { Response } from 'undici'; const pipeline = util.promisify(stream.pipeline); import { bindThis } from '@/decorators.js'; +type NonNullBodyResponse = Response & { body: ReadableStream }; + @Injectable() export class DownloadService { private logger: Logger; @@ -52,7 +55,7 @@ export class DownloadService { } @bindThis - public fetchUrl(url: string): any { + public async fetchUrl(url: string): Promise { this.logger.info(`Downloading ${chalk.cyan(url)} ...`); const timeout = 30 * 1000; @@ -75,11 +78,16 @@ export class DownloadService { this.logger.succ(`Download finished: ${chalk.cyan(url)}`); - return response; + return response as NonNullBodyResponse; } @bindThis - public async pipeRequestToFile(response: any, path: string): Promise { + public async pipeRequestToFile(_response: Response, path: string): Promise { + const response = _response.clone(); + if (response.body === null) { + throw new StatusError('No body', 400, 'No body'); + } + try { this.logger.info(`Saving File to ${chalk.cyanBright(path)} from downloading ...`); await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path)); @@ -94,7 +102,7 @@ export class DownloadService { @bindThis public async downloadUrl(url: string, path: string): Promise { - await this.pipeRequestToFile(this.fetchUrl(url), path); + await this.pipeRequestToFile(await this.fetchUrl(url), path); this.logger.succ(`Download finished: ${chalk.cyan(url)}`); } diff --git a/packages/backend/src/core/FileInfoService.ts b/packages/backend/src/core/FileInfoService.ts index d18191a56d..f6756ad465 100644 --- a/packages/backend/src/core/FileInfoService.ts +++ b/packages/backend/src/core/FileInfoService.ts @@ -15,7 +15,8 @@ import { encode } from 'blurhash'; import { createTempDir } from '@/misc/create-temp.js'; import { AiService } from '@/core/AiService.js'; import { bindThis } from '@/decorators.js'; -import type { Request } from 'got'; +import { Response } from 'undici'; +import { StatusError } from '@/misc/status-error.js'; const pipeline = util.promisify(stream.pipeline); @@ -343,18 +344,18 @@ export class FileInfoService { * Detect MIME Type and extension by stream for performance (this cannot detect SVG) */ @bindThis - public async detectRequestType(request: Request): Promise<{ + public async detectRequestType(_response: Response): Promise<{ mime: string; ext: string | null; }> { + const response = _response.clone(); + // Check 0 byte - if ((request.response?.complete || request.closed) && !request.response?.rawBody?.length) { - return TYPE_OCTET_STREAM; + if (!response.body) { + throw new StatusError('No Body', 400, 'No Body'); } - const copied = request.pipe(new stream.PassThrough()); - - const type = await fileTypeFromStream(copied); + const type = await fileTypeFromStream(stream.Readable.fromWeb(response.body)); if (type) { return { @@ -375,7 +376,7 @@ export class FileInfoService { try { const size = await this.getFileSize(path); if (size > 1 * 1024 * 1024) return false; - return isSvg(await fs.promises.readFile(target)); + return isSvg(await fs.promises.readFile(path)); } catch { return false; } diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 47cc85e257..1b7b887a88 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -20,7 +20,7 @@ import { FileInfoService, TYPE_SVG } from '@/core/FileInfoService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; -import { PassThrough } from 'node:stream'; +import { PassThrough, Readable } from 'node:stream'; import sharp from 'sharp'; import { Request } from 'got'; @@ -109,13 +109,12 @@ export class FileServerService { if (!file.storedInternal) { if (file.isLink && file.uri) { // 期限切れリモートファイル const [path, cleanup] = await createTemp(); - const got = this.downloadService.gotUrl(file.uri);; + const response = await this.downloadService.fetchUrl(file.uri);; try { - const fileSaving = this.downloadService.pipeRequestToFile(got, path); - const streamCopy = got.pipe(new PassThrough()); + const fileSaving = this.downloadService.pipeRequestToFile(response, path); - let { mime, ext } = await this.fileInfoService.detectRequestType(got); + let { mime, ext } = await this.fileInfoService.detectRequestType(response); if (mime === 'application/octet-stream' || mime === 'application/xml') { await fileSaving; if (await this.fileInfoService.checkSvg(path)) { @@ -127,7 +126,7 @@ export class FileServerService { const convertFile = async () => { if (isThumbnail) { if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(mime)) { - return this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 498, 280); + return this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 498, 280); } else if (mime.startsWith('video/')) { await fileSaving; return await this.videoProcessingService.generateVideoThumbnail(path); @@ -137,7 +136,7 @@ export class FileServerService { if (isWebpublic) { if (['image/svg+xml'].includes(mime)) { return { - data: this.imageProcessingService.convertSharpToWebpStream(streamCopy.pipe(sharp()), 2048, 2048, { ...webpDefault, lossless: true }), + data: this.imageProcessingService.convertSharpToWebpStream(Readable.fromWeb(response.body).pipe(sharp()), 2048, 2048, { ...webpDefault, lossless: true }), ext: 'webp', type: 'image/webp', }; @@ -145,7 +144,7 @@ export class FileServerService { } return { - data: streamCopy, + data: Readable.fromWeb(response.body), ext, type: mime, }; @@ -158,7 +157,6 @@ export class FileServerService { } catch (err) { this.logger.error(`${err}`); - if (!got.closed) got.destroy(); if (err instanceof StatusError && err.isClientError) { reply.code(err.statusCode); reply.header('Cache-Control', 'max-age=86400'); diff --git a/packages/backend/src/server/MediaProxyServerService.ts b/packages/backend/src/server/MediaProxyServerService.ts index 0e09ab1820..3926e863b6 100644 --- a/packages/backend/src/server/MediaProxyServerService.ts +++ b/packages/backend/src/server/MediaProxyServerService.ts @@ -18,7 +18,7 @@ import { FileInfoService, TYPE_SVG } from '@/core/FileInfoService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type { FastifyInstance, FastifyPluginOptions, FastifyReply, FastifyRequest } from 'fastify'; -import { PassThrough, pipeline } from 'node:stream'; +import { PassThrough, Readable, pipeline } from 'node:stream'; import { Request } from 'got'; const _filename = fileURLToPath(import.meta.url); @@ -67,21 +67,20 @@ export class MediaProxyServerService { @bindThis private async handler(request: FastifyRequest<{ Params: { url: string; }; Querystring: { url?: string; }; }>, reply: FastifyReply) { const url = 'url' in request.query ? request.query.url : 'https://' + request.params.url; - + if (typeof url !== 'string') { reply.code(400); return; } - + // Create temp file const [path, cleanup] = await createTemp(); - const got = this.downloadService.gotUrl(url); - - try { - const fileSaving = this.downloadService.pipeRequestToFile(got, path); - const streamCopy = got.pipe(new PassThrough()); + const response = await this.downloadService.fetchUrl(url); - let { mime, ext } = await this.fileInfoService.detectRequestType(got); + try { + const fileSaving = this.downloadService.pipeRequestToFile(response, path); + + let { mime, ext } = await this.fileInfoService.detectRequestType(response); if (mime === 'application/octet-stream' || mime === 'application/xml') { await fileSaving; if (await this.fileInfoService.checkSvg(path)) { @@ -90,11 +89,11 @@ export class MediaProxyServerService { } } const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image'); - - let image: IImageStreamable; + + let image: IImageStreamable | null = null; if ('emoji' in request.query && isConvertibleImage) { const data = pipeline( - streamCopy, + Readable.fromWeb(response.body), sharp({ animated: !('static' in request.query) }) .resize({ height: 128, @@ -109,9 +108,9 @@ export class MediaProxyServerService { type: 'image/webp', }; } else if ('static' in request.query && isConvertibleImage) { - image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 498, 280); + image = this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 498, 280); } else if ('preview' in request.query && isConvertibleImage) { - image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 200, 200); + image = this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 200, 200); } else if ('badge' in request.query) { if (!isConvertibleImage) { // 画像でないなら404でお茶を濁す @@ -150,25 +149,25 @@ export class MediaProxyServerService { type: 'image/png', }; } else if (mime === 'image/svg+xml') { - image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 2048, 2048); + image = this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 2048, 2048); } else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) { throw new StatusError('Rejected type', 403, 'Rejected type'); - } else { + } + + if (!image) { image = { - data: streamCopy, + data: Readable.fromWeb(response.body), ext, type: mime, }; } - + reply.header('Content-Type', image.type); reply.header('Cache-Control', 'max-age=31536000, immutable'); return image.data; } catch (err) { this.logger.error(`${err}`); - if (!got.closed) got.destroy(); - if ('fallback' in request.query) { return reply.sendFile('/dummy.png', assets); } @@ -181,5 +180,6 @@ export class MediaProxyServerService { } finally { cleanup(); } + return; // Not all code paths return a value. 対策 } }