Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

wip????

Browse source
This commit is contained in:
tamaina 2023-01-06 16:29:45 +00:00
parent ca0025b499
commit 12afc294f7
4 changed files with 48 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
packages/backend/src/core/DownloadService.ts
View file

@ -14,10 +14,13 @@ import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { buildConnector } from 'undici';
import type { Response } from 'undici';
const pipeline = util.promisify(stream.pipeline);
import { bindThis } from '@/decorators.js';
type NonNullBodyResponse = Response & { body: ReadableStream };
@Injectable()
export class DownloadService {
private logger: Logger;
@ -52,7 +55,7 @@ export class DownloadService {
}
@bindThis
public fetchUrl(url: string): any {
public async fetchUrl(url: string): Promise<NonNullBodyResponse> {
this.logger.info(`Downloading ${chalk.cyan(url)} ...`);
const timeout = 30 * 1000;
@ -75,11 +78,16 @@ export class DownloadService {
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
return response;
return response as NonNullBodyResponse;
}
@bindThis
public async pipeRequestToFile(response: any, path: string): Promise<void> {
public async pipeRequestToFile(_response: Response, path: string): Promise<void> {
const response = _response.clone();
if (response.body === null) {
throw new StatusError('No body', 400, 'No body');
}
try {
this.logger.info(`Saving File to ${chalk.cyanBright(path)} from downloading ...`);
await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
@ -94,7 +102,7 @@ export class DownloadService {
@bindThis
public async downloadUrl(url: string, path: string): Promise<void> {
await this.pipeRequestToFile(this.fetchUrl(url), path);
await this.pipeRequestToFile(await this.fetchUrl(url), path);
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
}

17
packages/backend/src/core/FileInfoService.ts
View file

@ -15,7 +15,8 @@ import { encode } from 'blurhash';
import { createTempDir } from '@/misc/create-temp.js';
import { AiService } from '@/core/AiService.js';
import { bindThis } from '@/decorators.js';
import type { Request } from 'got';
import { Response } from 'undici';
import { StatusError } from '@/misc/status-error.js';
const pipeline = util.promisify(stream.pipeline);
@ -343,18 +344,18 @@ export class FileInfoService {
* Detect MIME Type and extension by stream for performance (this cannot detect SVG)
*/
@bindThis
public async detectRequestType(request: Request): Promise<{
public async detectRequestType(_response: Response): Promise<{
mime: string;
ext: string | null;
}> {
const response = _response.clone();
// Check 0 byte
if ((request.response?.complete || request.closed) && !request.response?.rawBody?.length) {
return TYPE_OCTET_STREAM;
if (!response.body) {
throw new StatusError('No Body', 400, 'No Body');
}
const copied = request.pipe(new stream.PassThrough());
const type = await fileTypeFromStream(copied);
const type = await fileTypeFromStream(stream.Readable.fromWeb(response.body));
if (type) {
return {
@ -375,7 +376,7 @@ export class FileInfoService {
try {
const size = await this.getFileSize(path);
if (size > 1 * 1024 * 1024) return false;
return isSvg(await fs.promises.readFile(target));
return isSvg(await fs.promises.readFile(path));
} catch {
return false;
}

16
packages/backend/src/server/FileServerService.ts
View file

@ -20,7 +20,7 @@ import { FileInfoService, TYPE_SVG } from '@/core/FileInfoService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify';
import { PassThrough } from 'node:stream';
import { PassThrough, Readable } from 'node:stream';
import sharp from 'sharp';
import { Request } from 'got';
@ -109,13 +109,12 @@ export class FileServerService {
if (!file.storedInternal) {
if (file.isLink && file.uri) { // 期限切れリモートファイル
const [path, cleanup] = await createTemp();
const got = this.downloadService.gotUrl(file.uri);;
const response = await this.downloadService.fetchUrl(file.uri);;
try {
const fileSaving = this.downloadService.pipeRequestToFile(got, path);
const streamCopy = got.pipe(new PassThrough());
const fileSaving = this.downloadService.pipeRequestToFile(response, path);
let { mime, ext } = await this.fileInfoService.detectRequestType(got);
let { mime, ext } = await this.fileInfoService.detectRequestType(response);
if (mime === 'application/octet-stream' || mime === 'application/xml') {
await fileSaving;
if (await this.fileInfoService.checkSvg(path)) {
@ -127,7 +126,7 @@ export class FileServerService {
const convertFile = async () => {
if (isThumbnail) {
if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(mime)) {
return this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 498, 280);
return this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 498, 280);
} else if (mime.startsWith('video/')) {
await fileSaving;
return await this.videoProcessingService.generateVideoThumbnail(path);
@ -137,7 +136,7 @@ export class FileServerService {
if (isWebpublic) {
if (['image/svg+xml'].includes(mime)) {
return {
data: this.imageProcessingService.convertSharpToWebpStream(streamCopy.pipe(sharp()), 2048, 2048, { ...webpDefault, lossless: true }),
data: this.imageProcessingService.convertSharpToWebpStream(Readable.fromWeb(response.body).pipe(sharp()), 2048, 2048, { ...webpDefault, lossless: true }),
ext: 'webp',
type: 'image/webp',
};
@ -145,7 +144,7 @@ export class FileServerService {
}
return {
data: streamCopy,
data: Readable.fromWeb(response.body),
ext,
type: mime,
};
@ -158,7 +157,6 @@ export class FileServerService {
} catch (err) {
this.logger.error(`${err}`);
if (!got.closed) got.destroy();
if (err instanceof StatusError && err.isClientError) {
reply.code(err.statusCode);
reply.header('Cache-Control', 'max-age=86400');

40
packages/backend/src/server/MediaProxyServerService.ts
View file

@ -18,7 +18,7 @@ import { FileInfoService, TYPE_SVG } from '@/core/FileInfoService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import type { FastifyInstance, FastifyPluginOptions, FastifyReply, FastifyRequest } from 'fastify';
import { PassThrough, pipeline } from 'node:stream';
import { PassThrough, Readable, pipeline } from 'node:stream';
import { Request } from 'got';
const _filename = fileURLToPath(import.meta.url);
@ -67,21 +67,20 @@ export class MediaProxyServerService {
@bindThis
private async handler(request: FastifyRequest<{ Params: { url: string; }; Querystring: { url?: string; }; }>, reply: FastifyReply) {
const url = 'url' in request.query ? request.query.url : 'https://' + request.params.url;
if (typeof url !== 'string') {
reply.code(400);
return;
}
// Create temp file
const [path, cleanup] = await createTemp();
const got = this.downloadService.gotUrl(url);
try {
const fileSaving = this.downloadService.pipeRequestToFile(got, path);
const streamCopy = got.pipe(new PassThrough());
const response = await this.downloadService.fetchUrl(url);
let { mime, ext } = await this.fileInfoService.detectRequestType(got);
try {
const fileSaving = this.downloadService.pipeRequestToFile(response, path);
let { mime, ext } = await this.fileInfoService.detectRequestType(response);
if (mime === 'application/octet-stream' || mime === 'application/xml') {
await fileSaving;
if (await this.fileInfoService.checkSvg(path)) {
@ -90,11 +89,11 @@ export class MediaProxyServerService {
}
}
const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image');
let image: IImageStreamable;
let image: IImageStreamable | null = null;
if ('emoji' in request.query && isConvertibleImage) {
const data = pipeline(
streamCopy,
Readable.fromWeb(response.body),
sharp({ animated: !('static' in request.query) })
.resize({
height: 128,
@ -109,9 +108,9 @@ export class MediaProxyServerService {
type: 'image/webp',
};
} else if ('static' in request.query && isConvertibleImage) {
image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 498, 280);
image = this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 498, 280);
} else if ('preview' in request.query && isConvertibleImage) {
image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 200, 200);
image = this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 200, 200);
} else if ('badge' in request.query) {
if (!isConvertibleImage) {
// 画像でないなら404でお茶を濁す
@ -150,25 +149,25 @@ export class MediaProxyServerService {
type: 'image/png',
};
} else if (mime === 'image/svg+xml') {
image = this.imageProcessingService.convertSharpToWebpStreamObj(streamCopy.pipe(sharp()), 2048, 2048);
image = this.imageProcessingService.convertSharpToWebpStreamObj(Readable.fromWeb(response.body).pipe(sharp()), 2048, 2048);
} else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) {
throw new StatusError('Rejected type', 403, 'Rejected type');
} else {
}
if (!image) {
image = {
data: streamCopy,
data: Readable.fromWeb(response.body),
ext,
type: mime,
};
}
reply.header('Content-Type', image.type);
reply.header('Cache-Control', 'max-age=31536000, immutable');
return image.data;
} catch (err) {
this.logger.error(`${err}`);
if (!got.closed) got.destroy();
if ('fallback' in request.query) {
return reply.sendFile('/dummy.png', assets);
}
@ -181,5 +180,6 @@ export class MediaProxyServerService {
} finally {
cleanup();
}
return; // Not all code paths return a value. 対策
}
}