Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix(backend): キューのエラーログを簡略化するように (#14748)

Browse source 
* reduce federation log spam

* Don't record stack trace for unrecoverable errors.
* Avoid logging duplicate stace traces.

(cherry picked from commit ed0570110bf8cb8e8959591dccfa3c35999106ce)

* improve error summaries

(cherry picked from commit 20dd66f735d9778df0371001e303549dce619260)

* fix lint errors

(cherry picked from commit 83869e1c470b12b3bf4b23d885514d926620662a)

* condense job info

(cherry picked from commit 786702e076ad1af14538849512ad31c0ced7afe6)

* fix maxAttempts calculation

(cherry picked from commit b4d10aa8f821e594ec9c907eb2a5bdb3c73c67d5)

* condense error info

(cherry picked from commit f62cd8941ced74a4865aa5eae4f4a1c7aa1d30f1)

* normalize ID logging

(cherry picked from commit d8e1e4890d28347239162e26235eb68b1ff96654)

* further condense error details

(cherry picked from commit d867c2089b3b24680df0713a2aa0914789e45670)

* collapse AbortErrors

(cherry picked from commit 5171ba7113ebc7242527768afb9ab4cec534e3b3)

* don't log job name unless it has one

(cherry picked from commit a5316c06ed770b60f7b4c7ff5aa8c71cc0558db7)

* Update Changelog

* Record origin

---------

Co-authored-by: Hazel K <acomputerdog@gmail.com>
This commit is contained in:
かっこかり 2024-10-11 21:29:03 +09:00 committed by GitHub
parent 777804605e
commit 2f09d69773
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 52 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -12,6 +12,10 @@
- Feat: モデレータ権限を持つユーザが全員7日間活動しなかった場合は自動的に招待制へと移行するように ( #13437 ) - Feat: モデレータ権限を持つユーザが全員7日間活動しなかった場合は自動的に招待制へと移行するように ( #13437 )
- Fix: `admin/emoji/update`エンドポイントのidのみ指定した時不正なエラーが発生するバグを修正 - Fix: `admin/emoji/update`エンドポイントのidのみ指定した時不正なエラーが発生するバグを修正
### Server
- Fix: キューのエラーログを簡略化するように
(Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/649)
## 2024.10.0 ## 2024.10.0
### Note ### Note

76
packages/backend/src/queue/QueueProcessorService.ts
View file

@ -67,7 +67,7 @@ function getJobInfo(job: Bull.Job | undefined, increment = false): string {
// onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする
const currentAttempts = job.attemptsMade + (increment ? 1 : 0); const currentAttempts = job.attemptsMade + (increment ? 1 : 0);
const maxAttempts = job.opts ? job.opts.attempts : 0; const maxAttempts = job.opts.attempts ?? 0;
return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
} }
@ -126,20 +126,30 @@ export class QueueProcessorService implements OnApplicationShutdown {
) { ) {
this.logger = this.queueLoggerService.logger; this.logger = this.queueLoggerService.logger;
function renderError(e: Error): any { function renderError(e?: Error) {
if (e) { // 何故かeがundefinedで来ることがある // 何故かeがundefinedで来ることがある
if (!e) return '?';
if (e instanceof Bull.UnrecoverableError || e.name === 'AbortError') {
return `${e.name}: ${e.message}`;
}
return { return {
stack: e.stack, stack: e.stack,
message: e.message, message: e.message,
name: e.name, name: e.name,
}; };
} else {
return {
stack: '?',
message: '?',
name: '?',
};
} }
function renderJob(job?: Bull.Job) {
if (!job) return '?';
return {
name: job.name || undefined,
info: getJobInfo(job),
failedReason: job.failedReason || undefined,
data: job.data,
};
} }
//#region system //#region system
@ -175,15 +185,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`)) .on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err: Error) => { .on('failed', (job, err: Error) => {
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.message}`, { Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -232,15 +242,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`)) .on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.message}`, { Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -272,15 +282,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Deliver: ${err.message}`, { Sentry.captureMessage(`Queue: Deliver: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -312,15 +322,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)}`)) .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }); logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job: renderJob(job), e: renderError(err) });
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Inbox: ${err.message}`, { Sentry.captureMessage(`Queue: Inbox: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -352,15 +362,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.message}`, { Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -392,15 +402,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.message}`, { Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -439,15 +449,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`)) .on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.message}`, { Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion
@ -480,15 +490,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`)) .on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => { .on('failed', (job, err) => {
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
if (config.sentryForBackend) { if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.message}`, { Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error', level: 'error',
extra: { job, err }, extra: { job, err },
}); });
} }
}) })
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) })) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
} }
//#endregion //#endregion