Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'misskey-dev/develop' into io

Browse source
This commit is contained in:
まっちゃとーにゅ 2024-03-22 07:57:12 +09:00
commit 92280818ae
No known key found for this signature in database
GPG key ID: 6AFBBF529601C1DB
29 changed files with 1512 additions and 1121 deletions
Download patch file Download diff file Expand all files Collapse all files

42
packages/backend/migration/1710512074000-url-preview-meta.js Normal file
View file

@ -0,0 +1,42 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class UrlPreviewMeta1710512074000 {
name = 'UrlPreviewMeta1710512074000'
async up(queryRunner) {
await queryRunner.query(`
alter table meta
rename column "summalyProxy" to "urlPreviewSummaryProxyUrl";
alter table meta
add "urlPreviewEnabled" boolean default true not null;
alter table meta
add "urlPreviewTimeout" integer default 10000 not null;
alter table meta
add "urlPreviewMaximumContentLength" bigint default 10485760 not null;
alter table meta
add "urlPreviewRequireContentLength" boolean default false not null;
alter table meta
add "urlPreviewUserAgent" varchar(1024) default null;
`);
}
async down(queryRunner) {
await queryRunner.query(`
alter table meta
rename column "urlPreviewSummaryProxyUrl" to "summalyProxy";
alter table meta
drop column "urlPreviewEnabled";
alter table meta
drop column "urlPreviewTimeout";
alter table meta
drop column "urlPreviewMaximumContentLength";
alter table meta
drop column "urlPreviewRequireContentLength";
alter table meta
drop column "urlPreviewUserAgent";
`);
}
}

14
packages/backend/package.json
View file

@ -78,7 +78,7 @@
"@fastify/express": "2.3.0",
"@fastify/formbody": "7.4.0",
"@fastify/http-proxy": "9.5.0",
"@fastify/multipart": "8.1.0",
"@fastify/multipart": "8.2.0",
"@fastify/static": "7.0.1",
"@fastify/view": "9.0.0",
"@misskey-dev/sharp-read-bmp": "1.2.0",
@ -100,7 +100,7 @@
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"body-parser": "1.20.2",
"bullmq": "5.4.3",
"bullmq": "5.4.4",
"cacheable-lookup": "7.0.0",
"cbor": "9.0.2",
"chalk": "5.3.0",
@ -144,7 +144,7 @@
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"node-forge": "1.3.1",
"nodemailer": "6.9.12",
"nodemailer": "6.9.13",
"nsfwjs": "2.4.2",
"oauth": "0.10.0",
"oauth2orize": "1.12.0",
@ -154,7 +154,7 @@
"parse5": "7.1.2",
"pg": "8.11.3",
"pino": "8.19.0",
"pino-pretty": "10.3.1",
"pino-pretty": "11.0.0",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
@ -171,7 +171,7 @@
"rss-parser": "3.13.0",
"rxjs": "7.8.1",
"samlify": "2.8.11",
"sanitize-html": "2.12.1",
"sanitize-html": "2.13.0",
"secure-json-parse": "2.7.0",
"sharp": "0.33.2",
"slacc": "0.0.10",
@ -183,7 +183,7 @@
"tsc-alias": "1.8.8",
"tsconfig-paths": "4.2.0",
"typeorm": "0.3.20",
"typescript": "5.4.2",
"typescript": "5.4.3",
"ulid": "2.3.0",
"vary": "1.1.2",
"web-push": "3.6.7",
@ -219,7 +219,7 @@
"@types/oauth": "0.9.4",
"@types/oauth2orize": "1.11.4",
"@types/oauth2orize-pkce": "0.1.2",
"@types/pg": "8.11.3",
"@types/pg": "8.11.4",
"@types/pug": "2.0.10",
"@types/punycode": "2.1.4",
"@types/qrcode": "1.5.5",

1
packages/backend/src/core/entities/MetaEntityService.ts
View file

@ -111,6 +111,7 @@ export class MetaEntityService {
policies: { ...DEFAULT_POLICIES, ...instance.policies },
mediaProxy: this.config.mediaProxy,
enableUrlPreview: instance.urlPreviewEnabled,
};
}

38
packages/backend/src/models/Meta.ts
View file

@ -282,12 +282,6 @@ export class MiMeta {
})
public enableSensitiveMediaDetectionForVideos: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public summalyProxy: string | null;
@Column('boolean', {
default: false,
})
@ -608,4 +602,36 @@ export class MiMeta {
length: 1024, array: true, default: '{}',
})
public featuredGameChannels: string[];
@Column('boolean', {
default: true,
})
public urlPreviewEnabled: boolean;
@Column('integer', {
default: 10000,
})
public urlPreviewTimeout: number;
@Column('bigint', {
default: 1024 * 1024 * 10,
})
public urlPreviewMaximumContentLength: number;
@Column('boolean', {
default: true,
})
public urlPreviewRequireContentLength: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public urlPreviewSummaryProxyUrl: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public urlPreviewUserAgent: string | null;
}

4
packages/backend/src/models/json-schema/meta.ts
View file

@ -212,6 +212,10 @@ export const packedMetaLiteSchema = {
type: 'string',
optional: false, nullable: false,
},
enableUrlPreview: {
type: 'boolean',
optional: false, nullable: false,
},
backgroundImageUrl: {
type: 'string',
optional: false, nullable: true,

34
packages/backend/src/server/api/endpoints/admin/meta.ts
View file

@ -467,6 +467,8 @@ export const meta = {
summalyProxy: {
type: 'string',
optional: false, nullable: true,
deprecated: true,
description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
},
themeColor: {
type: 'string',
@ -484,6 +486,30 @@ export const meta = {
type: 'string',
optional: false, nullable: false,
},
urlPreviewEnabled: {
type: 'boolean',
optional: false, nullable: false,
},
urlPreviewTimeout: {
type: 'number',
optional: false, nullable: false,
},
urlPreviewMaximumContentLength: {
type: 'number',
optional: false, nullable: false,
},
urlPreviewRequireContentLength: {
type: 'boolean',
optional: false, nullable: false,
},
urlPreviewUserAgent: {
type: 'string',
optional: false, nullable: true,
},
urlPreviewSummaryProxyUrl: {
type: 'string',
optional: false, nullable: true,
},
},
},
} as const;
@ -567,7 +593,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
proxyAccountId: instance.proxyAccountId,
summalyProxy: instance.summalyProxy,
email: instance.email,
smtpSecure: instance.smtpSecure,
smtpHost: instance.smtpHost,
@ -614,6 +639,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
notesPerOneAd: instance.notesPerOneAd,
urlPreviewDenyList: instance.urlPreviewDenyList,
featuredGameChannels: instance.featuredGameChannels,
summalyProxy: instance.urlPreviewSummaryProxyUrl,
urlPreviewEnabled: instance.urlPreviewEnabled,
urlPreviewTimeout: instance.urlPreviewTimeout,
urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength,
urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength,
urlPreviewUserAgent: instance.urlPreviewUserAgent,
urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl,
};
});
}

41
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View file

@ -90,7 +90,6 @@ export const paramDef = {
type: 'string',
},
},
summalyProxy: { type: 'string', nullable: true },
deeplAuthKey: { type: 'string', nullable: true },
deeplIsPro: { type: 'boolean' },
enableEmail: { type: 'boolean' },
@ -170,6 +169,16 @@ export const paramDef = {
type: 'string',
},
},
summalyProxy: {
type: 'string', nullable: true,
description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
},
urlPreviewEnabled: { type: 'boolean' },
urlPreviewTimeout: { type: 'integer' },
urlPreviewMaximumContentLength: { type: 'integer' },
urlPreviewRequireContentLength: { type: 'boolean' },
urlPreviewUserAgent: { type: 'string', nullable: true },
urlPreviewSummaryProxyUrl: { type: 'string', nullable: true },
},
required: [],
} as const;
@ -397,10 +406,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.langs = ps.langs.filter(Boolean);
}
if (ps.summalyProxy !== undefined) {
set.summalyProxy = ps.summalyProxy;
}
if (ps.enableEmail !== undefined) {
set.enableEmail = ps.enableEmail;
}
@ -625,6 +630,32 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.bannedEmailDomains = ps.bannedEmailDomains;
}
if (ps.urlPreviewEnabled !== undefined) {
set.urlPreviewEnabled = ps.urlPreviewEnabled;
}
if (ps.urlPreviewTimeout !== undefined) {
set.urlPreviewTimeout = ps.urlPreviewTimeout;
}
if (ps.urlPreviewMaximumContentLength !== undefined) {
set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength;
}
if (ps.urlPreviewRequireContentLength !== undefined) {
set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength;
}
if (ps.urlPreviewUserAgent !== undefined) {
const value = (ps.urlPreviewUserAgent ?? '').trim();
set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent;
}
if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) {
const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim();
set.urlPreviewSummaryProxyUrl = value === '' ? null : value;
}
const before = await this.metaService.fetch(true);
await this.metaService.update(set);

69
packages/backend/src/server/web/UrlPreviewService.ts
View file

@ -4,8 +4,9 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { summaly } from '@misskey-dev/summaly';
import RE2 from 're2';
import { summaly } from '@misskey-dev/summaly';
import { SummalyResult } from '@misskey-dev/summaly/built/summary.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js';
@ -15,6 +16,7 @@ import { query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import { ApiError } from '@/server/api/error.js';
import { MiMeta } from '@/models/Meta.js';
import type { FastifyRequest, FastifyReply } from 'fastify';
@Injectable()
@ -63,24 +65,25 @@ export class UrlPreviewService {
const meta = await this.metaService.fetch();
this.logger.info(meta.summalyProxy
if (!meta.urlPreviewEnabled) {
reply.code(403);
return {
error: new ApiError({
message: 'URL preview is disabled',
code: 'URL_PREVIEW_DISABLED',
id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8',
}),
};
}
this.logger.info(meta.urlPreviewSummaryProxyUrl
? `(Proxy) Getting preview of ${url}@${lang} ...`
: `Getting preview of ${url}@${lang} ...`);
try {
const summary = meta.summalyProxy ?
await this.httpRequestService.getJson<ReturnType<typeof summaly>>(`${meta.summalyProxy}?${query({
url: url,
lang: lang ?? 'ja-JP',
})}`)
:
await summaly(url, {
followRedirects: false,
lang: lang ?? 'ja-JP',
agent: this.config.proxy ? {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
} : undefined,
});
const summary = meta.urlPreviewSummaryProxyUrl
? await this.fetchSummaryFromProxy(url, meta, lang)
: await this.fetchSummary(url, meta, lang);
this.logger.succ(`Got preview of ${url}: ${summary.title}`);
@ -118,6 +121,7 @@ export class UrlPreviewService {
return summary;
} catch (err) {
this.logger.warn(`Failed to get preview of ${url}: ${err}`);
reply.code(422);
reply.header('Cache-Control', 'max-age=86400, immutable');
return {
@ -129,4 +133,37 @@ export class UrlPreviewService {
};
}
}
private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const agent = this.config.proxy
? {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
}
: undefined;
return summaly(url, {
followRedirects: false,
lang: lang ?? 'ja-JP',
agent: agent,
userAgent: meta.urlPreviewUserAgent ?? undefined,
operationTimeout: meta.urlPreviewTimeout,
contentLengthLimit: meta.urlPreviewMaximumContentLength,
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
}
private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const proxy = meta.urlPreviewSummaryProxyUrl!;
const queryStr = query({
url: url,
lang: lang ?? 'ja-JP',
userAgent: meta.urlPreviewUserAgent ?? undefined,
operationTimeout: meta.urlPreviewTimeout,
contentLengthLimit: meta.urlPreviewMaximumContentLength,
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
}
}