From 12d2a9a5eb70c8065499f53c76860df745fbb4ba Mon Sep 17 00:00:00 2001 From: mattyatea Date: Sat, 13 Jul 2024 04:52:40 +0900 Subject: [PATCH 1/3] feat(backend): Add ability to use ElasticSearch as search backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 皐月ăȘご (Nafu Satsuki) --- packages/backend/package.json | 1 + packages/backend/src/GlobalModule.ts | 29 ++++- packages/backend/src/config.ts | 19 +++ packages/backend/src/core/SearchService.ts | 143 ++++++++++++++++++++- packages/backend/src/di-symbols.ts | 1 + pnpm-lock.yaml | 36 ++++++ 6 files changed, 220 insertions(+), 9 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 19547c5033..b989eb9eb7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -69,6 +69,7 @@ "@bull-board/fastify": "6.0.0", "@bull-board/ui": "6.0.0", "@discordapp/twemoji": "15.1.0", + "@elastic/elasticsearch": "^8.14.0", "@fastify/accepts": "5.0.0", "@fastify/cookie": "10.0.0", "@fastify/cors": "10.0.0", diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 6ae8ccfbb3..0b67bd7911 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -7,6 +7,7 @@ import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; import { MeiliSearch } from 'meilisearch'; +import { Client as ElasticSearch } from '@elastic/elasticsearch'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; @@ -45,6 +46,30 @@ const $meilisearch: Provider = { inject: [DI.config], }; +const $elasticsearch: Provider = { + provide: DI.elasticsearch, + useFactory: (config: Config) => { + if (config.elasticsearch) { + return new ElasticSearch({ + nodes: { + url: new URL(`${config.elasticsearch.ssl ? 'https' : 'http'}://${config.elasticsearch.host}:${config.elasticsearch.port}`), + ssl: { + rejectUnauthorized: config.elasticsearch.rejectUnauthorized, + }, + }, + auth: (config.elasticsearch.user && config.elasticsearch.pass) ? { + username: config.elasticsearch.user, + password: config.elasticsearch.pass, + } : undefined, + pingTimeout: 30000, + }); + } else { + return null; + } + }, + inject: [DI.config], +}; + const $redis: Provider = { provide: DI.redis, useFactory: (config: Config) => { @@ -148,8 +173,8 @@ const $meta: Provider = { @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions], - exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule], + providers: [$config, $db, $meta, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions], + exports: [$config, $db, $meta, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index c9411326a9..cfb5f6e1a7 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -64,6 +64,15 @@ type Source = { sentryForFrontend?: { options: Partial }; publishTarballInsteadOfProvideRepositoryUrl?: boolean; + elasticsearch?: { + host: string; + port: string; + user: string; + pass: string; + ssl?: boolean; + rejectUnauthorized?: boolean; + index: string; + }; proxy?: string; proxySmtp?: string; @@ -143,6 +152,15 @@ export type Config = { index: string; scope?: 'local' | 'global' | string[]; } | undefined; + elasticsearch: { + host: string; + port: string; + user: string; + pass: string; + ssl?: boolean; + rejectUnauthorized?: boolean; + index: string; + } | undefined; proxy: string | undefined; proxySmtp: string | undefined; proxyBypassHosts: string[] | undefined; @@ -287,6 +305,7 @@ export function loadConfig(): Config { dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, meilisearch: config.meilisearch, + elasticsearch: config.elasticsearch, redis, redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 6dc3e85fc8..a0221aa4f5 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -17,6 +17,7 @@ import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; import type { Index, MeiliSearch } from 'meilisearch'; +import type { Client as ElasticSearch } from '@elastic/elasticsearch'; type K = string; type V = string | number | boolean; @@ -65,7 +66,7 @@ function compileQuery(q: Q): string { export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; private meilisearchNoteIndex: Index | null = null; - + private elasticsearchNoteIndex: string | null = null; constructor( @Inject(DI.config) private config: Config, @@ -73,6 +74,9 @@ export class SearchService { @Inject(DI.meilisearch) private meilisearch: MeiliSearch | null, + @Inject(DI.elasticsearch) + private elasticsearch: ElasticSearch | null, + @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -106,9 +110,56 @@ export class SearchService { }, }); } - - if (this.config.meilisearch?.scope) { - this.meilisearchIndexScope = this.config.meilisearch.scope; + }); + } else if (this.elasticsearch) { + this.elasticsearchNoteIndex = `${config.elasticsearch!.index}---notes`; + this.elasticsearch.indices.exists({ + index: this.elasticsearchNoteIndex, + }).then((indexExists) => { + if (!indexExists) { + this.elasticsearch?.indices.create( + { + index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}`, + mappings: { + properties: { + text: { type: 'text' }, + cw: { type: 'text' }, + createdAt: { type: 'long' }, + userId: { type: 'keyword' }, + userHost: { type: 'keyword' }, + channelId: { type: 'keyword' }, + tags: { type: 'keyword' }, + }, + }, + settings: { + index: { + analysis: { + tokenizer: { + kuromoji: { + type: 'kuromoji_tokenizer', + mode: 'search', + }, + }, + analyzer: { + kuromoji_analyzer: { + type: 'custom', + tokenizer: 'kuromoji', + }, + }, + }, + }, + }, + }, + ).catch((error) => { + console.error(error); + }); + } + }).catch((error) => { + console.error(error); + }); + } + if (config.meilisearch?.scope) { + this.meilisearchIndexScope = config.meilisearch.scope; } } @@ -146,6 +197,23 @@ export class SearchService { }], { primaryKey: 'id', }); + } else if (this.elasticsearch) { + const body = { + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + }; + await this.elasticsearch.index({ + index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}` as string, + id: note.id, + body: body, + }).catch((error) => { + console.error(error); + }); } } @@ -190,7 +258,7 @@ export class SearchService { if (opts.filetype) { if (opts.filetype === 'image') { filter.qs.push({ op: 'or', qs: [ - { op: '=', k: 'attachedFileTypes', v: 'image/webp' }, + { op: '=', k: 'attachedFileTypes', v: 'image/webp' }, { op: '=', k: 'attachedFileTypes', v: 'image/png' }, { op: '=', k: 'attachedFileTypes', v: 'image/jpeg' }, { op: '=', k: 'attachedFileTypes', v: 'image/avif' }, @@ -199,14 +267,14 @@ export class SearchService { ] }); } else if (opts.filetype === 'video') { filter.qs.push({ op: 'or', qs: [ - { op: '=', k: 'attachedFileTypes', v: 'video/mp4' }, + { op: '=', k: 'attachedFileTypes', v: 'video/mp4' }, { op: '=', k: 'attachedFileTypes', v: 'video/webm' }, { op: '=', k: 'attachedFileTypes', v: 'video/mpeg' }, { op: '=', k: 'attachedFileTypes', v: 'video/x-m4v' }, ] }); } else if (opts.filetype === 'audio') { filter.qs.push({ op: 'or', qs: [ - { op: '=', k: 'attachedFileTypes', v: 'audio/mpeg' }, + { op: '=', k: 'attachedFileTypes', v: 'audio/mpeg' }, { op: '=', k: 'attachedFileTypes', v: 'audio/flac' }, { op: '=', k: 'attachedFileTypes', v: 'audio/wav' }, { op: '=', k: 'attachedFileTypes', v: 'audio/aac' }, @@ -247,6 +315,67 @@ export class SearchService { if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; return true; }); + return notes.sort((a, b) => a.id > b.id ? -1 : 1); + } else if (this.elasticsearch) { + const esFilter: any = { + bool: { + must: [], + }, + }; + + if (pagination.untilId) esFilter.bool.must.push({ range: { createdAt: { lt: this.idService.parse(pagination.untilId).date.getTime() } } }); + if (pagination.sinceId) esFilter.bool.must.push({ range: { createdAt: { gt: this.idService.parse(pagination.sinceId).date.getTime() } } }); + if (opts.userId) esFilter.bool.must.push({ term: { userId: opts.userId } }); + if (opts.channelId) esFilter.bool.must.push({ term: { channelId: opts.channelId } }); + if (opts.host) { + if (opts.host === '.') { + esFilter.bool.must.push({ bool: { must_not: [{ exists: { field: 'userHost' } }] } }); + } else { + esFilter.bool.must.push({ term: { userHost: opts.host } }); + } + } + + if (q !== '') { + esFilter.bool.must.push({ + bool: { + should: [ + { wildcard: { 'text': { value: q } } }, + { simple_query_string: { fields: ['text'], 'query': q, default_operator: 'and' } }, + { wildcard: { 'cw': { value: q } } }, + { simple_query_string: { fields: ['cw'], 'query': q, default_operator: 'and' } }, + ], + minimum_should_match: 1, + }, + }); + } + + const res = await (this.elasticsearch.search)({ + index: this.elasticsearchNoteIndex + '*' as string, + body: { + query: esFilter, + sort: [{ createdAt: { order: 'desc' } }], + }, + _source: ['id', 'createdAt'], + size: pagination.limit, + }); + + const noteIds = res.hits.hits.map((hit: any) => hit._id); + if (noteIds.length === 0) return []; + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) : [new Set(), new Set()]; + const notes = (await this.notesRepository.findBy({ + id: In(noteIds), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } else { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 5ea500ac77..a0b945f078 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -8,6 +8,7 @@ export const DI = { db: Symbol('db'), meta: Symbol('meta'), meilisearch: Symbol('meilisearch'), + elasticsearch: Symbol('elasticsearch'), redis: Symbol('redis'), redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bda23dfd32..e267f6cd48 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -97,6 +97,9 @@ importers: '@discordapp/twemoji': specifier: 15.1.0 version: 15.1.0 + '@elastic/elasticsearch': + specifier: ^8.14.0 + version: 8.14.0 '@fastify/accepts': specifier: 5.0.0 version: 5.0.0 @@ -2056,6 +2059,18 @@ packages: '@emnapi/runtime@1.3.0': resolution: {integrity: sha512-XMBySMuNZs3DM96xcJmLW4EfGnf+uGmFNjzpehMjuX5PLB5j87ar2Zc4e3PVeZ3I5g3tYtAqskB28manlF69Zw==} + '@discoveryjs/json-ext@0.5.7': + resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} + engines: {node: '>=10.0.0'} + + '@elastic/elasticsearch@8.14.0': + resolution: {integrity: sha512-MGrgCI4y+Ozssf5Q2IkVJlqt5bUMnKIICG2qxeOfrJNrVugMCBCAQypyesmSSocAtNm8IX3LxfJ3jQlFHmKe2w==} + engines: {node: '>=18'} + + '@elastic/transport@8.7.0': + resolution: {integrity: sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A==} + engines: {node: '>=18'} + '@esbuild/aix-ppc64@0.19.11': resolution: {integrity: sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==} engines: {node: '>=12'} @@ -12724,6 +12739,27 @@ snapshots: jsonfile: 5.0.0 universalify: 0.1.2 + '@discoveryjs/json-ext@0.5.7': {} + + '@elastic/elasticsearch@8.14.0': + dependencies: + '@elastic/transport': 8.7.0 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + + '@elastic/transport@8.7.0': + dependencies: + '@opentelemetry/api': 1.9.0 + debug: 4.3.4(supports-color@8.1.1) + hpagent: 1.2.0 + ms: 2.1.3 + secure-json-parse: 2.7.0 + tslib: 2.6.2 + undici: 6.19.2 + transitivePeerDependencies: + - supports-color + '@emnapi/runtime@1.3.0': dependencies: tslib: 2.6.3 From 020f7cbde499252668a83d13a363b11eaf53c8d8 Mon Sep 17 00:00:00 2001 From: mattyatea Date: Sat, 13 Jul 2024 07:44:58 +0900 Subject: [PATCH 2/3] =?UTF-8?q?SearchService.ts=E3=81=AE=E6=94=B9=E8=A1=8C?= =?UTF-8?q?=E3=81=AA=E3=81=A9=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/SearchService.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index a0221aa4f5..bd5861ee6a 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -8,6 +8,7 @@ import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { MiNote } from '@/models/Note.js'; import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; @@ -16,6 +17,7 @@ import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import type Logger from '@/logger.js'; import type { Index, MeiliSearch } from 'meilisearch'; import type { Client as ElasticSearch } from '@elastic/elasticsearch'; @@ -67,6 +69,8 @@ export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; private meilisearchNoteIndex: Index | null = null; private elasticsearchNoteIndex: string | null = null; + private logger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -84,8 +88,13 @@ export class SearchService { private queryService: QueryService, private idService: IdService, ) { + this.logger = this.loggerService.getLogger('note:search'); + if (meilisearch) { - this.meilisearchNoteIndex = meilisearch.index(`${this.config.meilisearch?.index}---notes`); + this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`); + if (config.meilisearch?.scope) { + this.meilisearchIndexScope = config.meilisearch.scope; + } this.meilisearchNoteIndex.updateSettings({ searchableAttributes: [ 'text', @@ -151,16 +160,13 @@ export class SearchService { }, }, ).catch((error) => { - console.error(error); + this.logger.error(error); }); } }).catch((error) => { - console.error(error); + this.logger.error('Error while checking if index exists', error); }); } - if (config.meilisearch?.scope) { - this.meilisearchIndexScope = config.meilisearch.scope; - } } @bindThis From ef019c070fc50fac94a0cfc6fffd9a244bf6836f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=88=E6=A7=8D=E7=94=B1=E7=B4=80?= Date: Sat, 19 Oct 2024 21:43:51 -0400 Subject: [PATCH 3/3] fix pnpm lock restore lock file another attempt Revert "fix pnpm lock" This reverts commit 57279b4ea20acc9e7aa6d511b7f2a1efe7274c54. Revert "restore lock file" This reverts commit 84c77a238adb53f098fb3d0f7ef3c7d9ba11a0f1. Revert "another attempt" This reverts commit 7c2ee05b6974983402fc2a0b606b4a69ee775d75. fix lock file Fix dependencies --- pnpm-lock.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e267f6cd48..fc8c90586f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12751,12 +12751,12 @@ snapshots: '@elastic/transport@8.7.0': dependencies: '@opentelemetry/api': 1.9.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.5(supports-color@8.1.1) hpagent: 1.2.0 ms: 2.1.3 secure-json-parse: 2.7.0 tslib: 2.6.2 - undici: 6.19.2 + undici: 6.20.1 transitivePeerDependencies: - supports-color