Use PostgreSQL instead of MongoDB (#4572)
* wip * Update note.ts * Update timeline.ts * Update core.ts * wip * Update generate-visibility-query.ts * wip * wip * wip * wip * wip * Update global-timeline.ts * wip * wip * wip * Update vote.ts * wip * wip * Update create.ts * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * Update files.ts * wip * wip * Update CONTRIBUTING.md * wip * wip * wip * wip * wip * wip * wip * wip * Update read-notification.ts * wip * wip * wip * wip * wip * wip * wip * Update cancel.ts * wip * wip * wip * Update show.ts * wip * wip * Update gen-id.ts * Update create.ts * Update id.ts * wip * wip * wip * wip * wip * wip * wip * Docker: Update files about Docker (#4599) * Docker: Use cache if files used by `yarn install` was not updated This patch reduces the number of times to installing node_modules. For example, `yarn install` step will be skipped when only ".config/default.yml" is updated. * Docker: Migrate MongoDB to Postgresql Misskey uses Postgresql as a database instead of Mongodb since version 11. * Docker: Uncomment about data persistence This patch will save a lot of databases. * wip * wip * wip * Update activitypub.ts * wip * wip * wip * Update logs.ts * wip * Update drive-file.ts * Update register.ts * wip * wip * Update mentions.ts * wip * wip * wip * Update recommendation.ts * wip * Update index.ts * wip * Update recommendation.ts * Doc: Update docker.ja.md and docker.en.md (#1) (#4608) Update how to set up misskey. * wip * ✌️ * wip * Update note.ts * Update postgre.ts * wip * wip * wip * wip * Update add-file.ts * wip * wip * wip * Clean up * Update logs.ts * wip * 🍕 * wip * Ad notes * wip * Update api-visibility.ts * Update note.ts * Update add-file.ts * tests * tests * Update postgre.ts * Update utils.ts * wip * wip * Refactor * wip * Refactor * wip * wip * Update show-users.ts * Update update-instance.ts * wip * Update feed.ts * Update outbox.ts * Update outbox.ts * Update user.ts * wip * Update list.ts * Update update-hashtag.ts * wip * Update update-hashtag.ts * Refactor * Update update.ts * wip * wip * ✌️ * clean up * docs * Update push.ts * wip * Update api.ts * wip * ✌️ * Update make-pagination-query.ts * ✌️ * Delete hashtags.ts * Update instances.ts * Update instances.ts * Update create.ts * Update search.ts * Update reversi-game.ts * Update signup.ts * Update user.ts * id * Update example.yml * 🎨 * objectid * fix * reversi * reversi * Fix bug of chart engine * Add test of chart engine * Improve test * Better testing * Improve chart engine * Refactor * Add test of chart engine * Refactor * Add chart test * Fix bug * コミットし忘れ * Refactoring * ✌️ * Add tests * Add test * Extarct note tests * Refactor * 存在しないユーザーにメンションできなくなっていた問題を修正 * Fix bug * Update update-meta.ts * Fix bug * Update mention.vue * Fix bug * Update meta.ts * Update CONTRIBUTING.md * Fix bug * Fix bug * Fix bug * Clean up * Clean up * Update notification.ts * Clean up * Add mute tests * Add test * Refactor * Add test * Fix test * Refactor * Refactor * Add tests * Update utils.ts * Update utils.ts * Fix test * Update package.json * Update update.ts * Update manifest.ts * Fix bug * Fix bug * Add test * 🎨 * Update endpoint permissions * Updaye permisison * Update person.ts #4299 * データベースと同期しないように * Fix bug * Fix bug * Update reversi-game.ts * Use a feature of Node v11.7.0 to extract a public key (#4644) * wip * wip * ✌️ * Refactoring #1540 * test * test * test * test * test * test * test * Fix bug * Fix test * 🍣 * wip * #4471 * Add test for #4335 * Refactor * Fix test * Add tests * 🕓 * Fix bug * Add test * Add test * rename * Fix bug
This commit is contained in:
syuilo
GitHub
parent
13caf37991
commit
f0a29721c9
592 changed files with 13463 additions and 14147 deletions
|
|
@ -1,364 +1,25 @@
|
|||
/**
|
||||
* チャートエンジン
|
||||
*/
|
||||
import FederationChart from './charts/classes/federation';
|
||||
import NotesChart from './charts/classes/notes';
|
||||
import UsersChart from './charts/classes/users';
|
||||
import NetworkChart from './charts/classes/network';
|
||||
import ActiveUsersChart from './charts/classes/active-users';
|
||||
import InstanceChart from './charts/classes/instance';
|
||||
import PerUserNotesChart from './charts/classes/per-user-notes';
|
||||
import DriveChart from './charts/classes/drive';
|
||||
import PerUserReactionsChart from './charts/classes/per-user-reactions';
|
||||
import HashtagChart from './charts/classes/hashtag';
|
||||
import PerUserFollowingChart from './charts/classes/per-user-following';
|
||||
import PerUserDriveChart from './charts/classes/per-user-drive';
|
||||
|
||||
import * as moment from 'moment';
|
||||
import * as nestedProperty from 'nested-property';
|
||||
import autobind from 'autobind-decorator';
|
||||
import * as mongo from 'mongodb';
|
||||
import db from '../../db/mongodb';
|
||||
import { ICollection } from 'monk';
|
||||
import Logger from '../logger';
|
||||
import { Schema } from '../../misc/schema';
|
||||
|
||||
const logger = new Logger('chart');
|
||||
|
||||
const utc = moment.utc;
|
||||
|
||||
export type Obj = { [key: string]: any };
|
||||
|
||||
export type Partial<T> = {
|
||||
[P in keyof T]?: Partial<T[P]>;
|
||||
};
|
||||
|
||||
type ArrayValue<T> = {
|
||||
[P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>;
|
||||
};
|
||||
|
||||
type Span = 'day' | 'hour';
|
||||
|
||||
type Log<T extends Obj> = {
|
||||
_id: mongo.ObjectID;
|
||||
|
||||
/**
|
||||
* 集計のグループ
|
||||
*/
|
||||
group?: any;
|
||||
|
||||
/**
|
||||
* 集計日時
|
||||
*/
|
||||
date: Date;
|
||||
|
||||
/**
|
||||
* 集計期間
|
||||
*/
|
||||
span: Span;
|
||||
|
||||
/**
|
||||
* データ
|
||||
*/
|
||||
data: T;
|
||||
|
||||
/**
|
||||
* ユニークインクリメント用
|
||||
*/
|
||||
unique?: Obj;
|
||||
};
|
||||
|
||||
/**
|
||||
* 様々なチャートの管理を司るクラス
|
||||
*/
|
||||
export default abstract class Chart<T extends Obj> {
|
||||
protected collection: ICollection<Log<T>>;
|
||||
protected abstract async getTemplate(init: boolean, latest?: T, group?: any): Promise<T>;
|
||||
private name: string;
|
||||
|
||||
constructor(name: string, grouped = false) {
|
||||
this.name = name;
|
||||
this.collection = db.get<Log<T>>(`chart.${name}`);
|
||||
|
||||
const keys = {
|
||||
span: -1,
|
||||
date: -1
|
||||
} as { [key: string]: 1 | -1; };
|
||||
if (grouped) keys.group = -1;
|
||||
|
||||
this.collection.createIndex(keys, { unique: true });
|
||||
}
|
||||
|
||||
@autobind
|
||||
private convertQuery(x: Obj, path: string): Obj {
|
||||
const query: Obj = {};
|
||||
|
||||
const dive = (x: Obj, path: string) => {
|
||||
for (const [k, v] of Object.entries(x)) {
|
||||
const p = path ? `${path}.${k}` : k;
|
||||
if (typeof v === 'number') {
|
||||
query[p] = v;
|
||||
} else {
|
||||
dive(v, p);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
dive(x, path);
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
@autobind
|
||||
private getCurrentDate(): [number, number, number, number] {
|
||||
const now = moment().utc();
|
||||
|
||||
const y = now.year();
|
||||
const m = now.month();
|
||||
const d = now.date();
|
||||
const h = now.hour();
|
||||
|
||||
return [y, m, d, h];
|
||||
}
|
||||
|
||||
@autobind
|
||||
private getLatestLog(span: Span, group?: any): Promise<Log<T>> {
|
||||
return this.collection.findOne({
|
||||
group: group,
|
||||
span: span
|
||||
}, {
|
||||
sort: {
|
||||
date: -1
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@autobind
|
||||
private async getCurrentLog(span: Span, group?: any): Promise<Log<T>> {
|
||||
const [y, m, d, h] = this.getCurrentDate();
|
||||
|
||||
const current =
|
||||
span == 'day' ? utc([y, m, d]) :
|
||||
span == 'hour' ? utc([y, m, d, h]) :
|
||||
null;
|
||||
|
||||
// 現在(今日または今のHour)のログ
|
||||
const currentLog = await this.collection.findOne({
|
||||
group: group,
|
||||
span: span,
|
||||
date: current.toDate()
|
||||
});
|
||||
|
||||
// ログがあればそれを返して終了
|
||||
if (currentLog != null) {
|
||||
return currentLog;
|
||||
}
|
||||
|
||||
let log: Log<T>;
|
||||
let data: T;
|
||||
|
||||
// 集計期間が変わってから、初めてのチャート更新なら
|
||||
// 最も最近のログを持ってくる
|
||||
// * 例えば集計期間が「日」である場合で考えると、
|
||||
// * 昨日何もチャートを更新するような出来事がなかった場合は、
|
||||
// * ログがそもそも作られずドキュメントが存在しないということがあり得るため、
|
||||
// * 「昨日の」と決め打ちせずに「もっとも最近の」とします
|
||||
const latest = await this.getLatestLog(span, group);
|
||||
|
||||
if (latest != null) {
|
||||
// 空ログデータを作成
|
||||
data = await this.getTemplate(false, latest.data);
|
||||
} else {
|
||||
// ログが存在しなかったら
|
||||
// (Misskeyインスタンスを建てて初めてのチャート更新時など
|
||||
// または何らかの理由でチャートコレクションを抹消した場合)
|
||||
|
||||
// 初期ログデータを作成
|
||||
data = await this.getTemplate(true, null, group);
|
||||
|
||||
logger.info(`${this.name}: Initial commit created`);
|
||||
}
|
||||
|
||||
try {
|
||||
// 新規ログ挿入
|
||||
log = await this.collection.insert({
|
||||
group: group,
|
||||
span: span,
|
||||
date: current.toDate(),
|
||||
data: data
|
||||
});
|
||||
} catch (e) {
|
||||
// 11000 is duplicate key error
|
||||
// 並列動作している他のチャートエンジンプロセスと処理が重なる場合がある
|
||||
// その場合は再度最も新しいログを持ってくる
|
||||
if (e.code === 11000) {
|
||||
log = await this.getLatestLog(span, group);
|
||||
} else {
|
||||
logger.error(e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
return log;
|
||||
}
|
||||
|
||||
@autobind
|
||||
protected commit(query: Obj, group?: any, uniqueKey?: string, uniqueValue?: string): void {
|
||||
const update = (log: Log<T>) => {
|
||||
// ユニークインクリメントの場合、指定のキーに指定の値が既に存在していたら弾く
|
||||
if (
|
||||
uniqueKey &&
|
||||
log.unique &&
|
||||
log.unique[uniqueKey] &&
|
||||
log.unique[uniqueKey].includes(uniqueValue)
|
||||
) return;
|
||||
|
||||
// ユニークインクリメントの指定のキーに値を追加
|
||||
if (uniqueKey) {
|
||||
query['$push'] = {
|
||||
[`unique.${uniqueKey}`]: uniqueValue
|
||||
};
|
||||
}
|
||||
|
||||
// ログ更新
|
||||
this.collection.update({
|
||||
_id: log._id
|
||||
}, query);
|
||||
};
|
||||
|
||||
this.getCurrentLog('day', group).then(log => update(log));
|
||||
this.getCurrentLog('hour', group).then(log => update(log));
|
||||
}
|
||||
|
||||
@autobind
|
||||
protected inc(inc: Partial<T>, group?: any): void {
|
||||
this.commit({
|
||||
$inc: this.convertQuery(inc, 'data')
|
||||
}, group);
|
||||
}
|
||||
|
||||
@autobind
|
||||
protected incIfUnique(inc: Partial<T>, key: string, value: string, group?: any): void {
|
||||
this.commit({
|
||||
$inc: this.convertQuery(inc, 'data')
|
||||
}, group, key, value);
|
||||
}
|
||||
|
||||
@autobind
|
||||
public async getChart(span: Span, range: number, group?: any): Promise<ArrayValue<T>> {
|
||||
const promisedChart: Promise<T>[] = [];
|
||||
|
||||
const [y, m, d, h] = this.getCurrentDate();
|
||||
|
||||
const gt =
|
||||
span == 'day' ? utc([y, m, d]).subtract(range, 'days') :
|
||||
span == 'hour' ? utc([y, m, d, h]).subtract(range, 'hours') :
|
||||
null;
|
||||
|
||||
// ログ取得
|
||||
let logs = await this.collection.find({
|
||||
group: group,
|
||||
span: span,
|
||||
date: {
|
||||
$gte: gt.toDate()
|
||||
}
|
||||
}, {
|
||||
sort: {
|
||||
date: -1
|
||||
},
|
||||
fields: {
|
||||
_id: 0
|
||||
}
|
||||
});
|
||||
|
||||
// 要求された範囲にログがひとつもなかったら
|
||||
if (logs.length == 0) {
|
||||
// もっとも新しいログを持ってくる
|
||||
// (すくなくともひとつログが無いと隙間埋めできないため)
|
||||
const recentLog = await this.collection.findOne({
|
||||
group: group,
|
||||
span: span
|
||||
}, {
|
||||
sort: {
|
||||
date: -1
|
||||
},
|
||||
fields: {
|
||||
_id: 0
|
||||
}
|
||||
});
|
||||
|
||||
if (recentLog) {
|
||||
logs = [recentLog];
|
||||
}
|
||||
|
||||
// 要求された範囲の最も古い箇所に位置するログが存在しなかったら
|
||||
} else if (!utc(logs[logs.length - 1].date).isSame(gt)) {
|
||||
// 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
|
||||
// (隙間埋めできないため)
|
||||
const outdatedLog = await this.collection.findOne({
|
||||
group: group,
|
||||
span: span,
|
||||
date: {
|
||||
$lt: gt.toDate()
|
||||
}
|
||||
}, {
|
||||
sort: {
|
||||
date: -1
|
||||
},
|
||||
fields: {
|
||||
_id: 0
|
||||
}
|
||||
});
|
||||
|
||||
if (outdatedLog) {
|
||||
logs.push(outdatedLog);
|
||||
}
|
||||
}
|
||||
|
||||
// 整形
|
||||
for (let i = (range - 1); i >= 0; i--) {
|
||||
const current =
|
||||
span == 'day' ? utc([y, m, d]).subtract(i, 'days') :
|
||||
span == 'hour' ? utc([y, m, d, h]).subtract(i, 'hours') :
|
||||
null;
|
||||
|
||||
const log = logs.find(l => utc(l.date).isSame(current));
|
||||
|
||||
if (log) {
|
||||
promisedChart.unshift(Promise.resolve(log.data));
|
||||
} else {
|
||||
// 隙間埋め
|
||||
const latest = logs.find(l => utc(l.date).isBefore(current));
|
||||
promisedChart.unshift(this.getTemplate(false, latest ? latest.data : null));
|
||||
}
|
||||
}
|
||||
|
||||
const chart = await Promise.all(promisedChart);
|
||||
|
||||
const res: ArrayValue<T> = {} as any;
|
||||
|
||||
/**
|
||||
* [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
|
||||
* を
|
||||
* { foo: [1, 2, 3], bar: [5, 6, 7] }
|
||||
* にする
|
||||
*/
|
||||
const dive = (x: Obj, path?: string) => {
|
||||
for (const [k, v] of Object.entries(x)) {
|
||||
const p = path ? `${path}.${k}` : k;
|
||||
if (typeof v == 'object') {
|
||||
dive(v, p);
|
||||
} else {
|
||||
nestedProperty.set(res, p, chart.map(s => nestedProperty.get(s, p)));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
dive(chart[0]);
|
||||
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
export function convertLog(logSchema: Schema): Schema {
|
||||
const v: Schema = JSON.parse(JSON.stringify(logSchema)); // copy
|
||||
if (v.type === 'number') {
|
||||
v.type = 'array';
|
||||
v.items = {
|
||||
type: 'number'
|
||||
};
|
||||
} else if (v.type === 'object') {
|
||||
for (const k of Object.keys(v.properties)) {
|
||||
v.properties[k] = convertLog(v.properties[k]);
|
||||
}
|
||||
}
|
||||
return v;
|
||||
}
|
||||
export const federationChart = new FederationChart();
|
||||
export const notesChart = new NotesChart();
|
||||
export const usersChart = new UsersChart();
|
||||
export const networkChart = new NetworkChart();
|
||||
export const activeUsersChart = new ActiveUsersChart();
|
||||
export const instanceChart = new InstanceChart();
|
||||
export const perUserNotesChart = new PerUserNotesChart();
|
||||
export const driveChart = new DriveChart();
|
||||
export const perUserReactionsChart = new PerUserReactionsChart();
|
||||
export const hashtagChart = new HashtagChart();
|
||||
export const perUserFollowingChart = new PerUserFollowingChart();
|
||||
export const perUserDriveChart = new PerUserDriveChart();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue