Mizah/mizzkey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'mi-dev/master' into report

Browse source
This commit is contained in:
mattyatea 2023-12-28 04:16:03 +09:00
commit cb1586658e
No known key found for this signature in database
GPG key ID: 068E54E2C33BEF9A
567 changed files with 10281 additions and 3996 deletions
Download patch file Download diff file Expand all files Collapse all files

25
packages/backend/src/core/EmailService.ts
View file

@ -7,8 +7,8 @@ import { URLSearchParams } from 'node:url';
import * as nodemailer from 'nodemailer';
import { Inject, Injectable } from '@nestjs/common';
import { validate as validateEmail } from 'deep-email-validator';
import { SubOutputFormat } from 'deep-email-validator/dist/output/output.js';
import { MetaService } from '@/core/MetaService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
@ -30,6 +30,7 @@ export class EmailService {
private metaService: MetaService,
private loggerService: LoggerService,
private utilityService: UtilityService,
private httpRequestService: HttpRequestService,
) {
this.logger = this.loggerService.getLogger('email');
@ -155,7 +156,7 @@ export class EmailService {
@bindThis
public async validateEmailForAccount(emailAddress: string): Promise<{
available: boolean;
reason: null | 'used' | 'format' | 'disposable' | 'mx' | 'smtp';
reason: null | 'used' | 'format' | 'disposable' | 'mx' | 'smtp' | 'banned';
}> {
const meta = await this.metaService.fetch();
@ -164,32 +165,38 @@ export class EmailService {
email: emailAddress,
});
const verifymailApi = meta.enableVerifymailApi && meta.verifymailAuthKey != null;
let validated;
let validated: {
valid: boolean,
reason?: string | null,
};
if (meta.enableActiveEmailValidation && meta.verifymailAuthKey) {
if (verifymailApi) {
if (meta.enableActiveEmailValidation) {
if (meta.enableVerifymailApi && meta.verifymailAuthKey != null) {
validated = await this.verifyMail(emailAddress, meta.verifymailAuthKey);
} else {
validated = meta.enableActiveEmailValidation ? await validateEmail({
validated = await validateEmail({
email: emailAddress,
validateRegex: true,
validateMx: true,
validateTypo: false, // TLDを見ているみたいだけどclubとか弾かれるので
validateDisposable: true, // 捨てアドかどうかチェック
validateSMTP: false, // 日本だと25ポートが殆どのプロバイダーで塞がれていてタイムアウトになるので
}) : { valid: true, reason: null };
});
}
} else {
validated = { valid: true, reason: null };
}
const available = exist === 0 && validated.valid;
const emailDomain: string = emailAddress.split('@')[1];
const isBanned = this.utilityService.isBlockedHost(meta.bannedEmailDomains, emailDomain);
const available = exist === 0 && validated.valid && !isBanned;
return {
available,
reason: available ? null :
exist !== 0 ? 'used' :
isBanned ? 'banned' :
validated.reason === 'regex' ? 'format' :
validated.reason === 'disposable' ? 'disposable' :
validated.reason === 'mx' ? 'mx' :

16
packages/backend/src/core/FeaturedService.ts
View file

@ -77,6 +77,17 @@ export class FeaturedService {
return Array.from(ranking.keys());
}
@bindThis
private async removeFromRanking(name: string, windowRange: number, element: string): Promise<void> {
const currentWindow = this.getCurrentWindow(windowRange);
const previousWindow = currentWindow - 1;
const redisPipeline = this.redisClient.pipeline();
redisPipeline.zrem(`${name}:${currentWindow}`, element);
redisPipeline.zrem(`${name}:${previousWindow}`, element);
await redisPipeline.exec();
}
@bindThis
public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
@ -126,4 +137,9 @@ export class FeaturedService {
public getHashtagsRanking(threshold: number): Promise<string[]> {
return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, threshold);
}
@bindThis
public removeHashtagsFromRanking(hashtag: string): Promise<void> {
return this.removeFromRanking('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag);
}
}

3
packages/backend/src/core/HashtagService.ts
View file

@ -15,6 +15,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { MetaService } from '@/core/MetaService.js';
import { UtilityService } from '@/core/UtilityService.js';
@Injectable()
export class HashtagService {
@ -29,6 +30,7 @@ export class HashtagService {
private featuredService: FeaturedService,
private idService: IdService,
private metaService: MetaService,
private utilityService: UtilityService,
) {
}
@ -161,6 +163,7 @@ export class HashtagService {
const instance = await this.metaService.fetch();
const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t));
if (hiddenTags.includes(hashtag)) return;
if (this.utilityService.isSensitiveWordIncluded(hashtag, instance.sensitiveWords)) return;
// YYYYMMDDHHmm (10分間隔)
const now = new Date();

25
packages/backend/src/core/MetaService.ts
View file

@ -11,6 +11,7 @@ import { MiMeta } from '@/models/Meta.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -25,6 +26,7 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
//this.onMessage = this.onMessage.bind(this);
@ -95,6 +97,8 @@ export class MetaService implements OnApplicationShutdown {
@bindThis
public async update(data: Partial<MiMeta>): Promise<MiMeta> {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
@ -102,10 +106,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
const meta = metas[0];
before = metas[0];
if (meta) {
await transactionalEntityManager.update(MiMeta, meta.id, data);
if (before) {
await transactionalEntityManager.update(MiMeta, before.id, data);
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
@ -119,6 +123,21 @@ export class MetaService implements OnApplicationShutdown {
}
});
if (data.hiddenTags) {
process.nextTick(() => {
const hiddenTags = new Set<string>(data.hiddenTags);
if (before) {
for (const previousHiddenTag of before.hiddenTags) {
hiddenTags.delete(previousHiddenTag);
}
}
for (const hiddenTag of hiddenTags) {
this.featuredService.removeHashtagsFromRanking(hiddenTag);
}
});
}
this.globalEventService.publishInternalEvent('metaUpdated', updated);
return updated;

34
packages/backend/src/core/NoteCreateService.ts
View file

@ -253,7 +253,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.visibility === 'public' && data.channel == null) {
const sensitiveWords = meta.sensitiveWords;
if (this.isSensitive(data, sensitiveWords)) {
if (this.utilityService.isSensitiveWordIncluded(data.cw ?? data.text ?? '', sensitiveWords)) {
data.visibility = 'home';
} else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) {
data.visibility = 'home';
@ -293,7 +293,7 @@ export class NoteCreateService implements OnApplicationShutdown {
}
// Check blocking
if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) {
if (data.renote && !this.isQuote(data)) {
if (data.renote.userHost === null) {
if (data.renote.userId !== user.id) {
const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
@ -622,7 +622,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// If it is renote
if (data.renote) {
const type = data.text ? 'quote' : 'renote';
const type = this.isQuote(data) ? 'quote' : 'renote';
// Notify
if (data.renote.userHost === null) {
@ -705,28 +705,9 @@ export class NoteCreateService implements OnApplicationShutdown {
}
@bindThis
private isSensitive(note: Option, sensitiveWord: string[]): boolean {
if (sensitiveWord.length > 0) {
const text = note.cw ?? note.text ?? '';
if (text === '') return false;
const matched = sensitiveWord.some(filter => {
// represents RegExp
const regexp = filter.match(/^\/(.+)\/(.*)$/);
// This should never happen due to input sanitisation.
if (!regexp) {
const words = filter.split(' ');
return words.every(keyword => text.includes(keyword));
}
try {
return new RE2(regexp[1], regexp[2]).test(text);
} catch (err) {
// This should never happen due to input sanitisation.
return false;
}
});
if (matched) return true;
}
return false;
private isQuote(note: Option): note is Option & { renote: MiNote } {
// sync with misc/is-quote.ts
return !!note.renote && (!!note.text || !!note.cw || (!!note.files && !!note.files.length) || !!note.poll);
}
@bindThis
@ -794,7 +775,7 @@ export class NoteCreateService implements OnApplicationShutdown {
private async renderNoteOrRenoteActivity(data: Option, note: MiNote) {
if (data.localOnly) return null;
const content = data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)
const content = data.renote && !this.isQuote(data)
? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note)
: this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, false), note);
@ -906,6 +887,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// ダイレクトのとき、そのリストが対象外のユーザーの場合
if (
note.visibility === 'specified' &&
note.userId !== userListMembership.userListUserId &&
!note.visibleUserIds.some(v => v === userListMembership.userListUserId)
) continue;

2
packages/backend/src/core/ReactionService.ts
View file

@ -138,7 +138,7 @@ export class ReactionService {
reaction = reacterHost ? `:${name}@${reacterHost}:` : `:${name}:`;
// センシティブ
if ((note.reactionAcceptance === 'nonSensitiveOnly') && emoji.isSensitive) {
if ((note.reactionAcceptance === 'nonSensitiveOnly' || note.reactionAcceptance === 'nonSensitiveOnlyForLocalLikeOnlyForRemote') && emoji.isSensitive) {
reaction = FALLBACK;
}
} else {

32
packages/backend/src/core/RoleService.ts
View file

@ -6,7 +6,14 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { In } from 'typeorm';
import type { MiRole, MiRoleAssignment, RoleAssignmentsRepository, RolesRepository, UsersRepository } from '@/models/_.js';
import { ModuleRef } from '@nestjs/core';
import type {
MiRole,
MiRoleAssignment,
RoleAssignmentsRepository,
RolesRepository,
UsersRepository,
} from '@/models/_.js';
import { MemoryKVCache, MemorySingleCache } from '@/misc/cache.js';
import type { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
@ -16,12 +23,13 @@ import { CacheService } from '@/core/CacheService.js';
import type { RoleCondFormulaValue } from '@/models/Role.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import type { Packed } from '@/misc/json-schema.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
import { NotificationService } from '@/core/NotificationService.js';
import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
export type RolePolicies = {
gtlAvailable: boolean;
@ -47,6 +55,7 @@ export type RolePolicies = {
userListLimit: number;
userEachUserListsLimit: number;
rateLimitFactor: number;
avatarDecorationLimit: number;
};
export const DEFAULT_POLICIES: RolePolicies = {
@ -73,17 +82,21 @@ export const DEFAULT_POLICIES: RolePolicies = {
userListLimit: 10,
userEachUserListsLimit: 50,
rateLimitFactor: 1,
avatarDecorationLimit: 1,
};
@Injectable()
export class RoleService implements OnApplicationShutdown {
export class RoleService implements OnApplicationShutdown, OnModuleInit {
private rolesCache: MemorySingleCache<MiRole[]>;
private roleAssignmentByUserIdCache: MemoryKVCache<MiRoleAssignment[]>;
private notificationService: NotificationService;
public static AlreadyAssignedError = class extends Error {};
public static NotAssignedError = class extends Error {};
constructor(
private moduleRef: ModuleRef,
@Inject(DI.redis)
private redisClient: Redis.Redis,
@ -118,6 +131,10 @@ export class RoleService implements OnApplicationShutdown {
this.redisForSub.on('message', this.onMessage);
}
async onModuleInit() {
this.notificationService = this.moduleRef.get(NotificationService.name);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
@ -326,6 +343,7 @@ export class RoleService implements OnApplicationShutdown {
userListLimit: calc('userListLimit', vs => Math.max(...vs)),
userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)),
rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)),
avatarDecorationLimit: calc('avatarDecorationLimit', vs => Math.max(...vs)),
};
}
@ -424,6 +442,12 @@ export class RoleService implements OnApplicationShutdown {
this.globalEventService.publishInternalEvent('userRoleAssigned', created);
if (role.isPublic) {
this.notificationService.createNotification(userId, 'roleAssigned', {
roleId: roleId,
});
}
if (moderator) {
const user = await this.usersRepository.findOneByOrFail({ id: userId });
this.moderationLogService.log(moderator, 'assignRole', {

17
packages/backend/src/core/UserListService.ts
View file

@ -3,30 +3,34 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import { Inject, Injectable, OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
import * as Redis from 'ioredis';
import { ModuleRef } from '@nestjs/core';
import type { UserListMembershipsRepository } from '@/models/_.js';
import type { MiUser } from '@/models/User.js';
import type { MiUserList } from '@/models/UserList.js';
import type { MiUserListMembership } from '@/models/UserListMembership.js';
import { IdService } from '@/core/IdService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ProxyAccountService } from '@/core/ProxyAccountService.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { QueueService } from '@/core/QueueService.js';
import { RedisKVCache } from '@/misc/cache.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { RoleService } from '@/core/RoleService.js';
@Injectable()
export class UserListService implements OnApplicationShutdown {
export class UserListService implements OnApplicationShutdown, OnModuleInit {
public static TooManyUsersError = class extends Error {};
public membersCache: RedisKVCache<Set<string>>;
private roleService: RoleService;
constructor(
private moduleRef: ModuleRef,
@Inject(DI.redis)
private redisClient: Redis.Redis,
@ -38,7 +42,6 @@ export class UserListService implements OnApplicationShutdown {
private userEntityService: UserEntityService,
private idService: IdService,
private roleService: RoleService,
private globalEventService: GlobalEventService,
private proxyAccountService: ProxyAccountService,
private queueService: QueueService,
@ -54,6 +57,10 @@ export class UserListService implements OnApplicationShutdown {
this.redisForSub.on('message', this.onMessage);
}
async onModuleInit() {
this.roleService = this.moduleRef.get(RoleService.name);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);

28
packages/backend/src/core/UtilityService.ts
View file

@ -6,6 +6,7 @@
import { URL } from 'node:url';
import { toASCII } from 'punycode';
import { Inject, Injectable } from '@nestjs/common';
import RE2 from 're2';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
@ -41,6 +42,33 @@ export class UtilityService {
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isSensitiveWordIncluded(text: string, sensitiveWords: string[]): boolean {
if (sensitiveWords.length === 0) return false;
if (text === '') return false;
const regexpregexp = /^\/(.+)\/(.*)$/;
const matched = sensitiveWords.some(filter => {
// represents RegExp
const regexp = filter.match(regexpregexp);
// This should never happen due to input sanitisation.
if (!regexp) {
const words = filter.split(' ');
return words.every(keyword => text.includes(keyword));
}
try {
// TODO: RE2インスタンスをキャッシュ
return new RE2(regexp[1], regexp[2]).test(text);
} catch (err) {
// This should never happen due to input sanitisation.
return false;
}
});
return matched;
}
@bindThis
public extractDbHost(uri: string): string {
const url = new URL(uri);

16
packages/backend/src/core/entities/NotificationEntityService.ts
View file

@ -15,8 +15,8 @@ import type { Packed } from '@/misc/json-schema.js';
import { bindThis } from '@/decorators.js';
import { isNotNull } from '@/misc/is-not-null.js';
import { FilterUnionByProperty, notificationTypes } from '@/types.js';
import { RoleEntityService } from './RoleEntityService.js';
import type { OnModuleInit } from '@nestjs/common';
import type { CustomEmojiService } from '../CustomEmojiService.js';
import type { UserEntityService } from './UserEntityService.js';
import type { NoteEntityService } from './NoteEntityService.js';
@ -27,7 +27,7 @@ const NOTE_REQUIRED_GROUPED_NOTIFICATION_TYPES = new Set(['note', 'mention', 're
export class NotificationEntityService implements OnModuleInit {
private userEntityService: UserEntityService;
private noteEntityService: NoteEntityService;
private customEmojiService: CustomEmojiService;
private roleEntityService: RoleEntityService;
constructor(
private moduleRef: ModuleRef,
@ -43,14 +43,13 @@ export class NotificationEntityService implements OnModuleInit {
//private userEntityService: UserEntityService,
//private noteEntityService: NoteEntityService,
//private customEmojiService: CustomEmojiService,
) {
}
onModuleInit() {
this.userEntityService = this.moduleRef.get('UserEntityService');
this.noteEntityService = this.moduleRef.get('NoteEntityService');
this.customEmojiService = this.moduleRef.get('CustomEmojiService');
this.roleEntityService = this.moduleRef.get('RoleEntityService');
}
@bindThis
@ -81,6 +80,7 @@ export class NotificationEntityService implements OnModuleInit {
detail: false,
})
) : undefined;
const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined;
return await awaitAll({
id: notification.id,
@ -92,6 +92,9 @@ export class NotificationEntityService implements OnModuleInit {
...(notification.type === 'reaction' ? {
reaction: notification.reaction,
} : {}),
...(notification.type === 'roleAssigned' ? {
role: role,
} : {}),
...(notification.type === 'achievementEarned' ? {
achievement: notification.achievement,
} : {}),
@ -216,6 +219,8 @@ export class NotificationEntityService implements OnModuleInit {
});
}
const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined;
return await awaitAll({
id: notification.id,
createdAt: new Date(notification.createdAt).toISOString(),
@ -226,6 +231,9 @@ export class NotificationEntityService implements OnModuleInit {
...(notification.type === 'reaction' ? {
reaction: notification.reaction,
} : {}),
...(notification.type === 'roleAssigned' ? {
role: role,
} : {}),
...(notification.type === 'achievementEarned' ? {
achievement: notification.achievement,
} : {}),

13
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -332,13 +332,13 @@ export class UserEntityService implements OnModuleInit {
const profile = opts.detail ? (opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id })) : null;
const followingCount = profile == null ? null :
(profile.ffVisibility === 'public') || isMe ? user.followingCount :
(profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount :
(profile.followingVisibility === 'public') || isMe ? user.followingCount :
(profile.followingVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount :
null;
const followersCount = profile == null ? null :
(profile.ffVisibility === 'public') || isMe ? user.followersCount :
(profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount :
(profile.followersVisibility === 'public') || isMe ? user.followersCount :
(profile.followersVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount :
null;
const isModerator = isMe && opts.detail ? this.roleService.isModerator(user) : null;
@ -362,6 +362,8 @@ export class UserEntityService implements OnModuleInit {
id: ud.id,
angle: ud.angle || undefined,
flipH: ud.flipH || undefined,
offsetX: ud.offsetX || undefined,
offsetY: ud.offsetY || undefined,
url: decorations.find(d => d.id === ud.id)!.url,
}))) : [],
isBot: user.isBot,
@ -415,7 +417,8 @@ export class UserEntityService implements OnModuleInit {
pinnedPageId: profile!.pinnedPageId,
pinnedPage: profile!.pinnedPageId ? this.pageEntityService.pack(profile!.pinnedPageId, me) : null,
publicReactions: profile!.publicReactions,
ffVisibility: profile!.ffVisibility,
followersVisibility: profile!.followersVisibility,
followingVisibility: profile!.followingVisibility,
twoFactorEnabled: profile!.twoFactorEnabled,
usePasswordLessLogin: profile!.usePasswordLessLogin,
securityKeys: profile!.twoFactorEnabled