Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'misskey-dev/develop' into io

Browse source
This commit is contained in:
まっちゃとーにゅ 2024-03-22 07:57:12 +09:00
commit 92280818ae
No known key found for this signature in database
GPG key ID: 6AFBBF529601C1DB
29 changed files with 1512 additions and 1121 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/backend/src/core/entities/MetaEntityService.ts
View file

@ -111,6 +111,7 @@ export class MetaEntityService {
policies: { ...DEFAULT_POLICIES, ...instance.policies },
mediaProxy: this.config.mediaProxy,
enableUrlPreview: instance.urlPreviewEnabled,
};
}

38
packages/backend/src/models/Meta.ts
View file

@ -282,12 +282,6 @@ export class MiMeta {
})
public enableSensitiveMediaDetectionForVideos: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public summalyProxy: string | null;
@Column('boolean', {
default: false,
})
@ -608,4 +602,36 @@ export class MiMeta {
length: 1024, array: true, default: '{}',
})
public featuredGameChannels: string[];
@Column('boolean', {
default: true,
})
public urlPreviewEnabled: boolean;
@Column('integer', {
default: 10000,
})
public urlPreviewTimeout: number;
@Column('bigint', {
default: 1024 * 1024 * 10,
})
public urlPreviewMaximumContentLength: number;
@Column('boolean', {
default: true,
})
public urlPreviewRequireContentLength: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public urlPreviewSummaryProxyUrl: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public urlPreviewUserAgent: string | null;
}

4
packages/backend/src/models/json-schema/meta.ts
View file

@ -212,6 +212,10 @@ export const packedMetaLiteSchema = {
type: 'string',
optional: false, nullable: false,
},
enableUrlPreview: {
type: 'boolean',
optional: false, nullable: false,
},
backgroundImageUrl: {
type: 'string',
optional: false, nullable: true,

34
packages/backend/src/server/api/endpoints/admin/meta.ts
View file

@ -467,6 +467,8 @@ export const meta = {
summalyProxy: {
type: 'string',
optional: false, nullable: true,
deprecated: true,
description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
},
themeColor: {
type: 'string',
@ -484,6 +486,30 @@ export const meta = {
type: 'string',
optional: false, nullable: false,
},
urlPreviewEnabled: {
type: 'boolean',
optional: false, nullable: false,
},
urlPreviewTimeout: {
type: 'number',
optional: false, nullable: false,
},
urlPreviewMaximumContentLength: {
type: 'number',
optional: false, nullable: false,
},
urlPreviewRequireContentLength: {
type: 'boolean',
optional: false, nullable: false,
},
urlPreviewUserAgent: {
type: 'string',
optional: false, nullable: true,
},
urlPreviewSummaryProxyUrl: {
type: 'string',
optional: false, nullable: true,
},
},
},
} as const;
@ -567,7 +593,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
proxyAccountId: instance.proxyAccountId,
summalyProxy: instance.summalyProxy,
email: instance.email,
smtpSecure: instance.smtpSecure,
smtpHost: instance.smtpHost,
@ -614,6 +639,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
notesPerOneAd: instance.notesPerOneAd,
urlPreviewDenyList: instance.urlPreviewDenyList,
featuredGameChannels: instance.featuredGameChannels,
summalyProxy: instance.urlPreviewSummaryProxyUrl,
urlPreviewEnabled: instance.urlPreviewEnabled,
urlPreviewTimeout: instance.urlPreviewTimeout,
urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength,
urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength,
urlPreviewUserAgent: instance.urlPreviewUserAgent,
urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl,
};
});
}

41
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View file

@ -90,7 +90,6 @@ export const paramDef = {
type: 'string',
},
},
summalyProxy: { type: 'string', nullable: true },
deeplAuthKey: { type: 'string', nullable: true },
deeplIsPro: { type: 'boolean' },
enableEmail: { type: 'boolean' },
@ -170,6 +169,16 @@ export const paramDef = {
type: 'string',
},
},
summalyProxy: {
type: 'string', nullable: true,
description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
},
urlPreviewEnabled: { type: 'boolean' },
urlPreviewTimeout: { type: 'integer' },
urlPreviewMaximumContentLength: { type: 'integer' },
urlPreviewRequireContentLength: { type: 'boolean' },
urlPreviewUserAgent: { type: 'string', nullable: true },
urlPreviewSummaryProxyUrl: { type: 'string', nullable: true },
},
required: [],
} as const;
@ -397,10 +406,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.langs = ps.langs.filter(Boolean);
}
if (ps.summalyProxy !== undefined) {
set.summalyProxy = ps.summalyProxy;
}
if (ps.enableEmail !== undefined) {
set.enableEmail = ps.enableEmail;
}
@ -625,6 +630,32 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.bannedEmailDomains = ps.bannedEmailDomains;
}
if (ps.urlPreviewEnabled !== undefined) {
set.urlPreviewEnabled = ps.urlPreviewEnabled;
}
if (ps.urlPreviewTimeout !== undefined) {
set.urlPreviewTimeout = ps.urlPreviewTimeout;
}
if (ps.urlPreviewMaximumContentLength !== undefined) {
set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength;
}
if (ps.urlPreviewRequireContentLength !== undefined) {
set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength;
}
if (ps.urlPreviewUserAgent !== undefined) {
const value = (ps.urlPreviewUserAgent ?? '').trim();
set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent;
}
if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) {
const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim();
set.urlPreviewSummaryProxyUrl = value === '' ? null : value;
}
const before = await this.metaService.fetch(true);
await this.metaService.update(set);

69
packages/backend/src/server/web/UrlPreviewService.ts
View file

@ -4,8 +4,9 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { summaly } from '@misskey-dev/summaly';
import RE2 from 're2';
import { summaly } from '@misskey-dev/summaly';
import { SummalyResult } from '@misskey-dev/summaly/built/summary.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js';
@ -15,6 +16,7 @@ import { query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import { ApiError } from '@/server/api/error.js';
import { MiMeta } from '@/models/Meta.js';
import type { FastifyRequest, FastifyReply } from 'fastify';
@Injectable()
@ -63,24 +65,25 @@ export class UrlPreviewService {
const meta = await this.metaService.fetch();
this.logger.info(meta.summalyProxy
if (!meta.urlPreviewEnabled) {
reply.code(403);
return {
error: new ApiError({
message: 'URL preview is disabled',
code: 'URL_PREVIEW_DISABLED',
id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8',
}),
};
}
this.logger.info(meta.urlPreviewSummaryProxyUrl
? `(Proxy) Getting preview of ${url}@${lang} ...`
: `Getting preview of ${url}@${lang} ...`);
try {
const summary = meta.summalyProxy ?
await this.httpRequestService.getJson<ReturnType<typeof summaly>>(`${meta.summalyProxy}?${query({
url: url,
lang: lang ?? 'ja-JP',
})}`)
:
await summaly(url, {
followRedirects: false,
lang: lang ?? 'ja-JP',
agent: this.config.proxy ? {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
} : undefined,
});
const summary = meta.urlPreviewSummaryProxyUrl
? await this.fetchSummaryFromProxy(url, meta, lang)
: await this.fetchSummary(url, meta, lang);
this.logger.succ(`Got preview of ${url}: ${summary.title}`);
@ -118,6 +121,7 @@ export class UrlPreviewService {
return summary;
} catch (err) {
this.logger.warn(`Failed to get preview of ${url}: ${err}`);
reply.code(422);
reply.header('Cache-Control', 'max-age=86400, immutable');
return {
@ -129,4 +133,37 @@ export class UrlPreviewService {
};
}
}
private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const agent = this.config.proxy
? {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
}
: undefined;
return summaly(url, {
followRedirects: false,
lang: lang ?? 'ja-JP',
agent: agent,
userAgent: meta.urlPreviewUserAgent ?? undefined,
operationTimeout: meta.urlPreviewTimeout,
contentLengthLimit: meta.urlPreviewMaximumContentLength,
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
}
private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const proxy = meta.urlPreviewSummaryProxyUrl!;
const queryStr = query({
url: url,
lang: lang ?? 'ja-JP',
userAgent: meta.urlPreviewUserAgent ?? undefined,
operationTimeout: meta.urlPreviewTimeout,
contentLengthLimit: meta.urlPreviewMaximumContentLength,
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
}
}