Mizah/Sharkey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Log user ips (#8872)

Browse source 
* wip

* store ip and headers

* Update admin-file.vue

* require admin for view ip/headers

* IP (recent) 消した

* admin必須

* opt in

* clean ips periodically

* respect logging setting in drive/files/create
This commit is contained in:
syuilo 2022-07-02 15:12:11 +09:00 • committed by GitHub
parent ded0f6f0df
commit eccc90c843
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 371 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

51
packages/backend/src/services/drive/add-file.ts
View file

@ -2,26 +2,26 @@ import * as fs from 'node:fs';
import { v4 as uuid } from 'uuid';
import S3 from 'aws-sdk/clients/s3.js';
import sharp from 'sharp';
import { IsNull } from 'typeorm';
import { publishMainStream, publishDriveStream } from '@/services/stream.js';
import { deleteFile } from './delete-file.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { GenerateVideoThumbnail } from './generate-video-thumbnail.js';
import { driveLogger } from './logger.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
import { contentDisposition } from '@/misc/content-disposition.js';
import { getFileInfo } from '@/misc/get-file-info.js';
import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '@/models/index.js';
import { InternalStorage } from './internal-storage.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { IRemoteUser, User } from '@/models/entities/user.js';
import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/index.js';
import { genId } from '@/misc/gen-id.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import S3 from 'aws-sdk/clients/s3.js';
import { getS3 } from './s3.js';
import sharp from 'sharp';
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { IsNull } from 'typeorm';
import { getS3 } from './s3.js';
import { InternalStorage } from './internal-storage.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
import { driveLogger } from './logger.js';
import { GenerateVideoThumbnail } from './generate-video-thumbnail.js';
import { deleteFile } from './delete-file.js';
const logger = driveLogger.createSubLogger('register', 'yellow');
@ -171,7 +171,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
}
if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) {
logger.debug(`web image and thumbnail not created (not an required file)`);
logger.debug('web image and thumbnail not created (not an required file)');
return {
webpublic: null,
thumbnail: null,
@ -212,7 +212,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
let webpublic: IImage | null = null;
if (generateWeb && !satisfyWebpublic) {
logger.info(`creating web image`);
logger.info('creating web image');
try {
if (['image/jpeg', 'image/webp'].includes(type)) {
@ -222,14 +222,14 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
} else if (['image/svg+xml'].includes(type)) {
webpublic = await convertSharpToPng(img, 2048, 2048);
} else {
logger.debug(`web image not created (not an required image)`);
logger.debug('web image not created (not an required image)');
}
} catch (err) {
logger.warn(`web image not created (an error occured)`, err as Error);
logger.warn('web image not created (an error occured)', err as Error);
}
} else {
if (satisfyWebpublic) logger.info(`web image not created (original satisfies webpublic)`);
else logger.info(`web image not created (from remote)`);
if (satisfyWebpublic) logger.info('web image not created (original satisfies webpublic)');
else logger.info('web image not created (from remote)');
}
// #endregion webpublic
@ -240,10 +240,10 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) {
thumbnail = await convertSharpToWebp(img, 498, 280);
} else {
logger.debug(`thumbnail not created (not an required file)`);
logger.debug('thumbnail not created (not an required file)');
}
} catch (err) {
logger.warn(`thumbnail not created (an error occured)`, err as Error);
logger.warn('thumbnail not created (an error occured)', err as Error);
}
// #endregion thumbnail
@ -276,7 +276,7 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string,
const s3 = getS3(meta);
const upload = s3.upload(params, {
partSize: s3.endpoint?.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
});
const result = await upload.promise();
@ -326,6 +326,9 @@ type AddFileArgs = {
uri?: string | null;
/** Mark file as sensitive */
sensitive?: boolean | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
};
/**
@ -342,7 +345,9 @@ export async function addFile({
isLink = false,
url = null,
uri = null,
sensitive = null
sensitive = null,
requestIp = null,
requestHeaders = null,
}: AddFileArgs): Promise<DriveFile> {
const info = await getFileInfo(path);
logger.info(`${JSON.stringify(info)}`);
@ -427,11 +432,13 @@ export async function addFile({
file.properties = properties;
file.blurhash = info.blurhash || null;
file.isLink = isLink;
file.requestIp = requestIp;
file.requestHeaders = requestHeaders;
file.isSensitive = user
? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
: false;
if (url !== null) {

10
packages/backend/src/services/drive/upload-from-url.ts
View file

@ -1,12 +1,12 @@
import { URL } from 'node:url';
import { addFile } from './add-file.js';
import { User } from '@/models/entities/user.js';
import { driveLogger } from './logger.js';
import { createTemp } from '@/misc/create-temp.js';
import { downloadUrl } from '@/misc/download-url.js';
import { DriveFolder } from '@/models/entities/drive-folder.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { DriveFiles } from '@/models/index.js';
import { driveLogger } from './logger.js';
import { addFile } from './add-file.js';
const logger = driveLogger.createSubLogger('downloader');
@ -19,6 +19,8 @@ type Args = {
force?: boolean;
isLink?: boolean;
comment?: string | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
};
export async function uploadFromUrl({
@ -30,6 +32,8 @@ export async function uploadFromUrl({
force = false,
isLink = false,
comment = null,
requestIp = null,
requestHeaders = null,
}: Args): Promise<DriveFile> {
let name = new URL(url).pathname.split('/').pop() || null;
if (name == null || !DriveFiles.validateFileName(name)) {
@ -49,7 +53,7 @@ export async function uploadFromUrl({
// write content at URL to temp file
await downloadUrl(url, path);
const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive });
const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders });
logger.succ(`Got: ${driveFile.id}`);
return driveFile!;
} catch (e) {