Merge branch 'develop' into enh-i-update-0

This commit is contained in:
かっこかり 2024-07-30 12:59:28 +09:00 committed by GitHub
commit cd26043c82
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
88 changed files with 3932 additions and 3728 deletions

View file

@ -31,7 +31,7 @@
"test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e",
"test-and-coverage": "pnpm jest-and-coverage",
"test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e",
"generate-api-json": "pnpm build && node ./scripts/generate_api_json.js"
"generate-api-json": "node ./scripts/generate_api_json.js"
},
"optionalDependencies": {
"@swc/core-android-arm64": "1.3.11",
@ -65,11 +65,11 @@
"utf-8-validate": "6.0.3"
},
"dependencies": {
"@aws-sdk/client-s3": "3.600.0",
"@aws-sdk/lib-storage": "3.600.0",
"@bull-board/api": "5.20.5",
"@bull-board/fastify": "5.20.5",
"@bull-board/ui": "5.20.5",
"@aws-sdk/client-s3": "3.620.0",
"@aws-sdk/lib-storage": "3.620.0",
"@bull-board/api": "5.21.1",
"@bull-board/fastify": "5.21.1",
"@bull-board/ui": "5.21.1",
"@discordapp/twemoji": "15.0.3",
"@fastify/accepts": "4.3.0",
"@fastify/cookie": "9.3.1",
@ -86,22 +86,22 @@
"@nestjs/core": "10.3.10",
"@nestjs/testing": "10.3.10",
"@peertube/http-signature": "1.7.0",
"@sentry/node": "8.13.0",
"@sentry/profiling-node": "8.13.0",
"@simplewebauthn/server": "10.0.0",
"@sentry/node": "8.20.0",
"@sentry/profiling-node": "8.20.0",
"@simplewebauthn/server": "10.0.1",
"@sinonjs/fake-timers": "11.2.2",
"@smithy/node-http-handler": "2.5.0",
"@swc/cli": "0.3.12",
"@swc/core": "1.6.6",
"@twemoji/parser": "15.1.1",
"accepts": "1.3.8",
"ajv": "8.16.0",
"ajv": "8.17.1",
"archiver": "7.0.1",
"async-mutex": "0.5.0",
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"body-parser": "1.20.2",
"bullmq": "5.8.3",
"bullmq": "5.10.4",
"cacheable-lookup": "7.0.0",
"cbor": "9.0.2",
"chalk": "5.3.0",
@ -115,10 +115,10 @@
"fastify": "4.28.1",
"fastify-raw-body": "4.3.0",
"feed": "4.2.2",
"file-type": "19.0.0",
"file-type": "19.3.0",
"fluent-ffmpeg": "2.1.3",
"form-data": "4.0.0",
"got": "14.4.1",
"got": "14.4.2",
"happy-dom": "10.0.3",
"hpagent": "1.2.0",
"htmlescape": "1.1.1",
@ -128,7 +128,7 @@
"ipaddr.js": "2.2.0",
"is-svg": "5.0.1",
"js-yaml": "4.1.0",
"jsdom": "24.1.0",
"jsdom": "24.1.1",
"json5": "2.2.3",
"jsonld": "8.3.2",
"jsrsasign": "11.1.0",
@ -177,11 +177,11 @@
"tsc-alias": "1.8.10",
"tsconfig-paths": "4.2.0",
"typeorm": "0.3.20",
"typescript": "5.5.3",
"typescript": "5.5.4",
"ulid": "2.3.0",
"vary": "1.1.2",
"web-push": "3.6.7",
"ws": "8.17.1",
"ws": "8.18.0",
"xev": "3.0.2"
},
"devDependencies": {
@ -201,11 +201,11 @@
"@types/jest": "29.5.12",
"@types/js-yaml": "4.0.9",
"@types/jsdom": "21.1.7",
"@types/jsonld": "1.5.14",
"@types/jsonld": "1.5.15",
"@types/jsrsasign": "10.5.14",
"@types/mime-types": "2.1.4",
"@types/ms": "0.7.34",
"@types/node": "20.14.9",
"@types/node": "20.14.12",
"@types/nodemailer": "6.4.15",
"@types/oauth": "0.9.5",
"@types/oauth2orize": "1.11.5",
@ -225,18 +225,18 @@
"@types/tmp": "0.2.6",
"@types/vary": "1.1.3",
"@types/web-push": "3.6.3",
"@types/ws": "8.5.10",
"@typescript-eslint/eslint-plugin": "7.15.0",
"@typescript-eslint/parser": "7.15.0",
"@types/ws": "8.5.11",
"@typescript-eslint/eslint-plugin": "7.17.0",
"@typescript-eslint/parser": "7.17.0",
"aws-sdk-client-mock": "4.0.1",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.29.1",
"execa": "9.2.0",
"execa": "9.3.0",
"fkill": "9.0.0",
"jest": "29.7.0",
"jest-mock": "29.7.0",
"nodemon": "3.1.4",
"pid-port": "1.0.0",
"simple-oauth2": "5.0.1"
"simple-oauth2": "5.1.0"
}
}

View file

@ -3,11 +3,34 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { loadConfig } from '../built/config.js'
import { genOpenapiSpec } from '../built/server/api/openapi/gen-spec.js'
import { writeFileSync } from "node:fs";
import { execa } from 'execa';
import { writeFileSync, existsSync } from "node:fs";
const config = loadConfig();
const spec = genOpenapiSpec(config, true);
async function main() {
if (!process.argv.includes('--no-build')) {
await execa('pnpm', ['run', 'build'], {
stdout: process.stdout,
stderr: process.stderr,
});
}
writeFileSync('./built/api.json', JSON.stringify(spec), 'utf-8');
if (!existsSync('./built')) {
throw new Error('`built` directory does not exist.');
}
/** @type {import('../src/config.js')} */
const { loadConfig } = await import('../built/config.js');
/** @type {import('../src/server/api/openapi/gen-spec.js')} */
const { genOpenapiSpec } = await import('../built/server/api/openapi/gen-spec.js');
const config = loadConfig();
const spec = genOpenapiSpec(config, true);
writeFileSync('./built/api.json', JSON.stringify(spec), 'utf-8');
}
main().catch(e => {
console.error(e);
process.exit(1);
});

View file

@ -44,7 +44,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
/**
* Redisイベントを用いて{@link abuseReports}.
* {@link RoleService.getModeratorIds}.
* {@link getModeratorIds}.
*
* @see RoleService.getModeratorIds
* @see GlobalEventService.publishAdminStream

View file

@ -933,10 +933,13 @@ export class NoteCreateService implements OnApplicationShutdown {
}
}
if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL
this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
// 自分自身のHTL
if (note.userHost == null) {
if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) {
this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r);
if (note.fileIds.length > 0) {
this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r);
}
}
}

View file

@ -505,14 +505,15 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
this.globalEventService.publishInternalEvent('userRoleAssigned', created);
if (role.isPublic) {
const user = await this.usersRepository.findOneByOrFail({ id: userId });
if (role.isPublic && user.host === null) {
this.notificationService.createNotification(userId, 'roleAssigned', {
roleId: roleId,
});
}
if (moderator) {
const user = await this.usersRepository.findOneByOrFail({ id: userId });
this.moderationLogService.log(moderator, 'assignRole', {
roleId: roleId,
roleName: role.name,

View file

@ -21,6 +21,7 @@ import { bindThis } from '@/decorators.js';
import UsersChart from '@/core/chart/charts/users.js';
import { UtilityService } from '@/core/UtilityService.js';
import { MetaService } from '@/core/MetaService.js';
import { UserService } from '@/core/UserService.js';
@Injectable()
export class SignupService {
@ -35,6 +36,7 @@ export class SignupService {
private usedUsernamesRepository: UsedUsernamesRepository,
private utilityService: UtilityService,
private userService: UserService,
private userEntityService: UserEntityService,
private idService: IdService,
private metaService: MetaService,
@ -148,7 +150,8 @@ export class SignupService {
}));
});
this.usersChart.update(account, true);
this.usersChart.update(account, true).then();
this.userService.notifySystemWebhook(account, 'userCreated').then();
return { account, secret };
}

View file

@ -279,8 +279,10 @@ export class UserFollowingService implements OnModuleInit {
});
// 通知を作成
this.notificationService.createNotification(follower.id, 'followRequestAccepted', {
}, followee.id);
if (follower.host === null) {
this.notificationService.createNotification(follower.id, 'followRequestAccepted', {
}, followee.id);
}
}
if (alreadyFollowed) return;

View file

@ -8,15 +8,18 @@ import type { FollowingsRepository, UsersRepository } from '@/models/_.js';
import type { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { SystemWebhookService } from '@/core/SystemWebhookService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
@Injectable()
export class UserService {
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
private systemWebhookService: SystemWebhookService,
private userEntityService: UserEntityService,
) {
}
@ -50,4 +53,23 @@ export class UserService {
});
}
}
/**
* SystemWebhookを用いてユーザに関する操作内容を管理者各位に通知する.
* JobQueueへのエンキューのみを行うため.
*
* @see SystemWebhookService.enqueueSystemWebhook
*/
@bindThis
public async notifySystemWebhook(user: MiUser, type: 'userCreated') {
const packedUser = await this.userEntityService.pack(user, null, { schema: 'UserLite' });
const recipientWebhookIds = await this.systemWebhookService.fetchSystemWebhooks({ isActive: true, on: [type] });
for (const webhookId of recipientWebhookIds) {
await this.systemWebhookService.enqueueSystemWebhook(
webhookId,
type,
packedUser,
);
}
}
}

View file

@ -12,6 +12,8 @@ export const systemWebhookEventTypes = [
'abuseReport',
// 通報を処理したとき
'abuseReportResolved',
// ユーザが作成された時
'userCreated',
] as const;
export type SystemWebhookEventType = typeof systemWebhookEventTypes[number];

View file

@ -204,6 +204,7 @@ export const packedNoteSchema = {
reactionAcceptance: {
type: 'string',
optional: false, nullable: true,
enum: ['likeOnly', 'likeOnlyForRemote', 'nonSensitiveOnly', 'nonSensitiveOnlyForLocalLikeOnlyForRemote', null],
},
reactionEmojis: {
type: 'object',

View file

@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { PollVotesRepository, NotesRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
import { CacheService } from '@/core/CacheService.js';
import { NotificationService } from '@/core/NotificationService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
@ -24,6 +25,7 @@ export class EndedPollNotificationProcessorService {
@Inject(DI.pollVotesRepository)
private pollVotesRepository: PollVotesRepository,
private cacheService: CacheService,
private notificationService: NotificationService,
private queueLoggerService: QueueLoggerService,
) {
@ -47,9 +49,12 @@ export class EndedPollNotificationProcessorService {
const userIds = [...new Set([note.userId, ...votes.map(v => v.userId)])];
for (const userId of userIds) {
this.notificationService.createNotification(userId, 'pollEnded', {
noteId: note.id,
});
const profile = await this.cacheService.userProfileCache.fetch(userId);
if (profile.userHost === null) {
this.notificationService.createNotification(userId, 'pollEnded', {
noteId: note.id,
});
}
}
}
}

View file

@ -5,65 +5,24 @@
import { entities } from 'misskey-js';
import { beforeEach, describe, test } from '@jest/globals';
import Fastify from 'fastify';
import { api, randomString, role, signup, startJobQueue, UserToken } from '../../utils.js';
import {
api,
captureWebhook,
randomString,
role,
signup,
startJobQueue,
UserToken,
WEBHOOK_HOST,
} from '../../utils.js';
import type { INestApplicationContext } from '@nestjs/common';
const WEBHOOK_HOST = 'http://localhost:15080';
const WEBHOOK_PORT = 15080;
process.env.NODE_ENV = 'test';
describe('[シナリオ] ユーザ通報', () => {
let queue: INestApplicationContext;
let admin: entities.SignupResponse;
let alice: entities.SignupResponse;
let bob: entities.SignupResponse;
type SystemWebhookPayload = {
server: string;
hookId: string;
eventId: string;
createdAt: string;
type: string;
body: any;
}
// -------------------------------------------------------------------------------------------
async function captureWebhook<T = SystemWebhookPayload>(postAction: () => Promise<void>): Promise<T> {
const fastify = Fastify();
let timeoutHandle: NodeJS.Timeout | null = null;
const result = await new Promise<string>(async (resolve, reject) => {
fastify.all('/', async (req, res) => {
timeoutHandle && clearTimeout(timeoutHandle);
const body = JSON.stringify(req.body);
res.status(200).send('ok');
await fastify.close();
resolve(body);
});
await fastify.listen({ port: WEBHOOK_PORT });
timeoutHandle = setTimeout(async () => {
await fastify.close();
reject(new Error('timeout'));
}, 3000);
try {
await postAction();
} catch (e) {
await fastify.close();
reject(e);
}
});
await fastify.close();
return JSON.parse(result) as T;
}
async function createSystemWebhook(args?: Partial<entities.AdminSystemWebhookCreateRequest>, credential?: UserToken): Promise<entities.AdminSystemWebhookCreateResponse> {
const res = await api(
'admin/system-webhook/create',

View file

@ -0,0 +1,130 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { setTimeout } from 'node:timers/promises';
import { entities } from 'misskey-js';
import { beforeEach, describe, test } from '@jest/globals';
import {
api,
captureWebhook,
randomString,
role,
signup,
startJobQueue,
UserToken,
WEBHOOK_HOST,
} from '../../utils.js';
import type { INestApplicationContext } from '@nestjs/common';
describe('[シナリオ] ユーザ作成', () => {
let queue: INestApplicationContext;
let admin: entities.SignupResponse;
async function createSystemWebhook(args?: Partial<entities.AdminSystemWebhookCreateRequest>, credential?: UserToken): Promise<entities.AdminSystemWebhookCreateResponse> {
const res = await api(
'admin/system-webhook/create',
{
isActive: true,
name: randomString(),
on: ['userCreated'],
url: WEBHOOK_HOST,
secret: randomString(),
...args,
},
credential ?? admin,
);
return res.body;
}
// -------------------------------------------------------------------------------------------
beforeAll(async () => {
queue = await startJobQueue();
admin = await signup({ username: 'admin' });
await role(admin, { isAdministrator: true });
}, 1000 * 60 * 2);
afterAll(async () => {
await queue.close();
});
// -------------------------------------------------------------------------------------------
describe('SystemWebhook', () => {
beforeEach(async () => {
const webhooks = await api('admin/system-webhook/list', {}, admin);
for (const webhook of webhooks.body) {
await api('admin/system-webhook/delete', { id: webhook.id }, admin);
}
});
test('ユーザが作成された -> userCreatedが送出される', async () => {
const webhook = await createSystemWebhook({
on: ['userCreated'],
isActive: true,
});
let alice: any = null;
const webhookBody = await captureWebhook(async () => {
alice = await signup({ username: 'alice' });
});
// webhookの送出後にいろいろやってるのでちょっと待つ必要がある
await setTimeout(2000);
console.log(alice);
console.log(JSON.stringify(webhookBody, null, 2));
expect(webhookBody.hookId).toBe(webhook.id);
expect(webhookBody.type).toBe('userCreated');
const body = webhookBody.body as entities.UserLite;
expect(alice.id).toBe(body.id);
expect(alice.name).toBe(body.name);
expect(alice.username).toBe(body.username);
expect(alice.host).toBe(body.host);
expect(alice.avatarUrl).toBe(body.avatarUrl);
expect(alice.avatarBlurhash).toBe(body.avatarBlurhash);
expect(alice.avatarDecorations).toEqual(body.avatarDecorations);
expect(alice.isBot).toBe(body.isBot);
expect(alice.isCat).toBe(body.isCat);
expect(alice.instance).toEqual(body.instance);
expect(alice.emojis).toEqual(body.emojis);
expect(alice.onlineStatus).toBe(body.onlineStatus);
expect(alice.badgeRoles).toEqual(body.badgeRoles);
});
test('ユーザ作成 -> userCreatedが未許可の場合は送出されない', async () => {
await createSystemWebhook({
on: [],
isActive: true,
});
let alice: any = null;
const webhookBody = await captureWebhook(async () => {
alice = await signup({ username: 'alice' });
}).catch(e => e.message);
expect(webhookBody).toBe('timeout');
expect(alice.id).not.toBeNull();
});
test('ユーザ作成 -> Webhookが無効の場合は送出されない', async () => {
await createSystemWebhook({
on: ['userCreated'],
isActive: false,
});
let alice: any = null;
const webhookBody = await captureWebhook(async () => {
alice = await signup({ username: 'alice' });
}).catch(e => e.message);
expect(webhookBody).toBe('timeout');
expect(alice.id).not.toBeNull();
});
});
});

View file

@ -9,8 +9,8 @@
import * as assert from 'assert';
import { setTimeout } from 'node:timers/promises';
import { Redis } from 'ioredis';
import { loadConfig } from '@/config.js';
import { api, post, randomString, sendEnvUpdateRequest, signup, uploadUrl } from '../utils.js';
import { loadConfig } from '@/config.js';
function genHost() {
return randomString() + '.example.com';
@ -492,6 +492,44 @@ describe('Timelines', () => {
assert.strictEqual(res.body.some(note => note.id === bobNote.id), false);
});
test.concurrent('FTT: ローカルユーザーの HTL にはプッシュされる', async () => {
const [alice, bob, carol] = await Promise.all([signup(), signup(), signup()]);
await api('following/create', {
userId: alice.id,
}, bob);
const aliceNote = await post(alice, { text: 'I\'m Alice.' });
const bobNote = await post(bob, { text: 'I\'m Bob.' });
const carolNote = await post(carol, { text: 'I\'m Carol.' });
await waitForPushToTl();
// NOTE: notes/timeline だと DB へのフォールバックが効くので Redis を直接見て確かめる
assert.strictEqual(await redisForTimelines.exists(`list:homeTimeline:${bob.id}`), 1);
const bobHTL = await redisForTimelines.lrange(`list:homeTimeline:${bob.id}`, 0, -1);
assert.strictEqual(bobHTL.includes(aliceNote.id), true);
assert.strictEqual(bobHTL.includes(bobNote.id), true);
assert.strictEqual(bobHTL.includes(carolNote.id), false);
});
test.concurrent('FTT: リモートユーザーの HTL にはプッシュされない', async () => {
const [alice, bob] = await Promise.all([signup(), signup({ host: genHost() })]);
await api('following/create', {
userId: alice.id,
}, bob);
await post(alice, { text: 'I\'m Alice.' });
await post(bob, { text: 'I\'m Bob.' });
await waitForPushToTl();
// NOTE: notes/timeline だと DB へのフォールバックが効くので Redis を直接見て確かめる
assert.strictEqual(await redisForTimelines.exists(`list:homeTimeline:${bob.id}`), 0);
});
});
describe('Local TL', () => {

View file

@ -12,13 +12,14 @@ import WebSocket, { ClientOptions } from 'ws';
import fetch, { File, RequestInit, type Headers } from 'node-fetch';
import { DataSource } from 'typeorm';
import { JSDOM } from 'jsdom';
import { DEFAULT_POLICIES } from '@/core/RoleService.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
import { type Response } from 'node-fetch';
import Fastify from 'fastify';
import { entities } from '../src/postgres.js';
import { loadConfig } from '../src/config.js';
import type * as misskey from 'misskey-js';
import { type Response } from 'node-fetch';
import { ApiError } from "@/server/api/error.js";
import { DEFAULT_POLICIES } from '@/core/RoleService.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
import { ApiError } from '@/server/api/error.js';
export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js';
@ -27,11 +28,23 @@ export interface UserToken {
bearer?: boolean;
}
export type SystemWebhookPayload = {
server: string;
hookId: string;
eventId: string;
createdAt: string;
type: string;
body: any;
}
const config = loadConfig();
export const port = config.port;
export const origin = config.url;
export const host = new URL(config.url).host;
export const WEBHOOK_HOST = 'http://localhost:15080';
export const WEBHOOK_PORT = 15080;
export const cookie = (me: UserToken): string => {
return `token=${me.token};`;
};
@ -645,3 +658,37 @@ export async function sendEnvResetRequest() {
export function castAsError(obj: Record<string, unknown>): { error: ApiError } {
return obj as { error: ApiError };
}
export async function captureWebhook<T = SystemWebhookPayload>(postAction: () => Promise<void>, port = WEBHOOK_PORT): Promise<T> {
const fastify = Fastify();
let timeoutHandle: NodeJS.Timeout | null = null;
const result = await new Promise<string>(async (resolve, reject) => {
fastify.all('/', async (req, res) => {
timeoutHandle && clearTimeout(timeoutHandle);
const body = JSON.stringify(req.body);
res.status(200).send('ok');
await fastify.close();
resolve(body);
});
await fastify.listen({ port });
timeoutHandle = setTimeout(async () => {
await fastify.close();
reject(new Error('timeout'));
}, 3000);
try {
await postAction();
} catch (e) {
await fastify.close();
reject(e);
}
});
await fastify.close();
return JSON.parse(result) as T;
}