Mizah/Sharkey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

allow setting separate timeout / max size for imports - fixes #479

Browse source
This commit is contained in:
dakkar 2024-06-03 16:29:19 +00:00 • committed by Amelia Yukii
parent 55fc2879f3
commit 082e1d1afb
4 changed files with 38 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

21
packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View file

@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
import type { Config } from '@/config.js';
@Injectable()
export class ImportNotesProcessorService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -73,6 +77,11 @@ export class ImportNotesProcessorService {
}
}
@bindThis
private downloadUrl(url: string, path:string): Promise<{filename: string}> {
return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
}
@bindThis
private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
type NotesMap = {
@ -176,7 +185,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -206,7 +215,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -239,7 +248,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -297,7 +306,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(path, '', 'utf-8');
await this.downloadService.downloadUrl(file.url, path);
await this.downloadUrl(file.url, path);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -349,7 +358,7 @@ export class ImportNotesProcessorService {
if (!exists) {
try {
await this.downloadService.downloadUrl(file.url, filePath);
await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string));
}
@ -488,7 +497,7 @@ export class ImportNotesProcessorService {
if (!exists) {
try {
await this.downloadService.downloadUrl(file.url, filePath);
await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string));
}