diff --git a/locales/index.d.ts b/locales/index.d.ts index fb010d9353..fa877ab7d6 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -730,6 +730,14 @@ export interface Locale extends ILocale { * にゃにゃにゃ？？ */ "flagAsCatDescription": string; + /** + * 自動TTS機能が欲しい。 + */ + "flagAsVI": string; + /** + * 自動TTS機能が必要な場合は有効にしてください。 権限のあるユーザーグループに所属している場合、特定の範囲で自動TTS機能を有効にします。 + */ + "flagAsVIDescription": string; /** * タイムラインにノートへの返信を表示する */ @@ -6891,6 +6899,10 @@ export interface Locale extends ILocale { * 翻訳機能の利用 */ "canUseTranslator": string; + /** + * TTS機能の利用 + */ + "canUseTTS": string; /** * アイコンデコレーションの最大取付個数 */ @@ -6937,6 +6949,10 @@ export interface Locale extends ILocale { * botユーザー */ "isBot": string; + /** + * TTSユーザー + */ + "isVI": string; /** * サスペンド済みユーザー */ @@ -7328,6 +7344,10 @@ export interface Locale extends ILocale { * Misskeyを翻訳 */ "translation": string; + /** + * Misskeyを変換 + */ + "convert": string; /** * Misskeyに寄付 */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index c241a9e560..6ee3b17154 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -178,6 +178,8 @@ flagAsBot: "Botとして設定" flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。" flagAsCat: "にゃああああああああああああああ！！！！！！！！！！！！" flagAsCatDescription: "にゃにゃにゃ？？" +flagAsVI: "自動TTS機能が欲しい。" +flagAsVIDescription: "自動TTS機能が必要な場合は有効にしてください。 権限のあるユーザーグループに所属している場合、特定の範囲で自動TTS機能を有効にします。" flagShowTimelineReplies: "タイムラインにノートへの返信を表示する" flagShowTimelineRepliesDescription: "オンにすると、タイムラインにユーザーのノート以外にもそのユーザーの他のノートへの返信を表示します。" autoAcceptFollowed: "フォロー中ユーザーからのフォロリクを自動承認" @@ -1780,6 +1782,7 @@ _role: canHideAds: "広告の非表示" canSearchNotes: "ノート検索の利用" canUseTranslator: "翻訳機能の利用" + canUseTTS: "TTS機能の利用" avatarDecorationLimit: "アイコンデコレーションの最大取付個数" canImportAntennas: "アンテナのインポートを許可" canImportBlocking: "ブロックのインポートを許可" @@ -1792,6 +1795,7 @@ _role: isRemote: "リモートユーザー" isCat: "猫ユーザー" isBot: "botユーザー" + isVI: "TTSユーザー" isSuspended: "サスペンド済みユーザー" isLocked: "鍵アカウントユーザー" isExplorable: "「アカウントを見つけやすくする」が有効なユーザー" @@ -1910,6 +1914,7 @@ _aboutMisskey: original: "オリジナル" thisIsModifiedVersion: "{name}はオリジナルのMisskeyを改変したバージョンを使用しています。" translation: "Misskeyを翻訳" + convert: "Misskeyを変換" donate: "Misskeyに寄付" morePatrons: "他にも多くの方が支援してくれています。ありがとうございます🥰" patrons: "支援者" diff --git a/packages/backend/migration/1724683952000-tts.js b/packages/backend/migration/1724683952000-tts.js new file mode 100644 index 0000000000..60e8d29cc5 --- /dev/null +++ b/packages/backend/migration/1724683952000-tts.js @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class TTSIntegration1724683952000 { + constructor() { + this.name = 'TTSIntegration1724683952000'; + } + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "hfAuthKey" character varying(128)`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfAuthKey"`); + } +} + diff --git a/packages/backend/migration/1724683962000-tts.js b/packages/backend/migration/1724683962000-tts.js new file mode 100644 index 0000000000..a2e21847ee --- /dev/null +++ b/packages/backend/migration/1724683962000-tts.js @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class TTSIntegration1724683962000 { + constructor() { + this.name = 'TTSIntegration1724683962000'; + } + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD "isVI" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "user"."isVI" IS 'Whether the User needs auto TTS.'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfSpace" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfSpaceName" character varying(128)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfexampleAudioURL" character varying(128)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfexampleText" character varying(128)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfexampleLang" character varying(128)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfslice" character varying(128) DEFAULT 'Slice once every 4 sentences'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hftopK" INTEGER DEFAULT 15`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hftopP" INTEGER DEFAULT 100`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfTemperature" INTEGER DEFAULT 100`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfnrm" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfSpeedRate" INTEGER DEFAULT 125`); + await queryRunner.query(`ALTER TABLE "meta" ADD "hfdas" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`COMMENT ON COLUMN "user"."isVI" IS NULL`); + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "isVI"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfSpace"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfSpaceName"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfexampleAudioURL"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfexampleText"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfexampleLang"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfslice"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hftopK"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hftopP"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfTemperature"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfnrm"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfSpeedRate"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfdas"`); + } +} + diff --git a/packages/backend/package.json b/packages/backend/package.json index 0dd738a1e6..85f4327a57 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -81,6 +81,7 @@ "@fastify/multipart": "9.0.1", "@fastify/static": "8.0.1", "@fastify/view": "10.0.1", + "@gradio/client": "1.6.0-beta.3", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.1.0", "@napi-rs/canvas": "0.1.56", @@ -134,8 +135,8 @@ "json5": "2.2.3", "jsonld": "8.3.2", "jsrsasign": "11.1.0", - "meilisearch": "0.42.0", "juice": "11.0.0", + "meilisearch": "0.42.0", "mfm-js": "0.24.0", "microformats-parser": "2.0.2", "mime-types": "2.1.35", diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 5af6b05942..6f88f277e5 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -44,6 +44,7 @@ export type RolePolicies = { canManageAvatarDecorations: boolean; canSearchNotes: boolean; canUseTranslator: boolean; + canUseTTS: boolean; canHideAds: boolean; driveCapacityMb: number; alwaysMarkNsfw: boolean; @@ -78,6 +79,7 @@ export const DEFAULT_POLICIES: RolePolicies = { canManageAvatarDecorations: false, canSearchNotes: false, canUseTranslator: true, + canUseTTS: true, canHideAds: false, driveCapacityMb: 100, alwaysMarkNsfw: false, @@ -256,6 +258,10 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { case 'isCat': { return user.isCat; } + // Auto TTS + case 'isVI': { + return user.isVI; + } // 「ユーザを見つけやすくする」が有効なアカウント case 'isExplorable': { return user.isExplorable; @@ -383,6 +389,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { canManageAvatarDecorations: calc('canManageAvatarDecorations', vs => vs.some(v => v === true)), canSearchNotes: calc('canSearchNotes', vs => vs.some(v => v === true)), canUseTranslator: calc('canUseTranslator', vs => vs.some(v => v === true)), + canUseTTS: calc('canUseTTS', vs => vs.some(v => v === true)), canHideAds: calc('canHideAds', vs => vs.some(v => v === true)), driveCapacityMb: calc('driveCapacityMb', vs => Math.max(...vs)), alwaysMarkNsfw: calc('alwaysMarkNsfw', vs => vs.some(v => v === true)), diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index 55c8a52705..4591052739 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -79,6 +79,7 @@ function generateDummyUser(override?: Partial): MiUser { isLocked: false, isBot: false, isCat: true, + isVI: false, isRoot: false, isExplorable: true, isHibernated: false, @@ -198,6 +199,7 @@ function toPackedUserLite(user: MiUser, override?: Packed<'UserLite'>): Packed<' })), isBot: user.isBot, isCat: user.isCat, + isVI: user.isVI, emojis: user.emojis, onlineStatus: 'active', badgeRoles: [], diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index 409dca3426..9212aa1e6d 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -123,6 +123,7 @@ export class MetaEntityService { enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, + ttsAvailable: instance.hfAuthKey != null, serverRules: instance.serverRules, diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index c9939adf11..ebeaa52664 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -490,6 +490,7 @@ export class UserEntityService implements OnModuleInit { }))) : [], isBot: user.isBot, isCat: user.isCat, + isVI: user.isVI, instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index ad5e31ad6f..e572352327 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -359,6 +359,78 @@ export class MiMeta { }) public deeplIsPro: boolean; + @Column('varchar', { + length: 1024, + nullable: true, + }) + public hfAuthKey: string | null; + + @Column('boolean', { + default: false, + }) + public hfSpace: boolean; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public hfSpaceName: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public hfexampleAudioURL: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public hfexampleText: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public hfexampleLang: string | null; + + @Column('varchar', { + length: 1024, + default: 'Slice once every 4 sentences', + nullable: true, + }) + public hfslice: string | null; + + @Column('integer', { + default: 15, + }) + public hftopK: number; + + @Column('integer', { + default: 100, + }) + public hftopP: number; + + @Column('integer', { + default: 100, + }) + public hfTemperature: number; + + @Column('boolean', { + default: false, + }) + public hfnrm: boolean; + + @Column('integer', { + default: 125, + }) + public hfSpeedRate: number; + + @Column('boolean', { + default: false, + }) + public hfdas: boolean; + @Column('varchar', { length: 1024, nullable: true, diff --git a/packages/backend/src/models/Role.ts b/packages/backend/src/models/Role.ts index a173971b2c..0e7c9f452b 100644 --- a/packages/backend/src/models/Role.ts +++ b/packages/backend/src/models/Role.ts @@ -83,6 +83,13 @@ type CondFormulaValueIsCat = { type: 'isCat'; }; +/** + * Auto TTS + */ +type CondFormulaValueIsVI = { + type: 'isVI'; +}; + /** * 「ユーザを見つけやすくする」が有効なアカウントの場合のみ成立とする */ @@ -164,6 +171,7 @@ export type RoleCondFormulaValue = { id: string } & ( CondFormulaValueIsLocked | CondFormulaValueIsBot | CondFormulaValueIsCat | + CondFormulaValueIsVI | CondFormulaValueIsExplorable | CondFormulaValueRoleAssignedTo | CondFormulaValueCreatedLessThan | diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 805a1e75ae..785a3ae334 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -184,6 +184,12 @@ export class MiUser { }) public isCat: boolean; + @Column('boolean', { + default: false, + comment: 'Whether the User needs auto TTS.', + }) + public isVI: boolean; + @Column('boolean', { default: false, comment: 'Whether the User is the root.', diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index e3fd63464a..1e0d723fdf 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -207,6 +207,10 @@ export const packedMetaLiteSchema = { type: 'boolean', optional: false, nullable: false, }, + ttsAvailable: { + type: 'boolean', + optional: false, nullable: false, + }, mediaProxy: { type: 'string', optional: false, nullable: false, diff --git a/packages/backend/src/models/json-schema/role.ts b/packages/backend/src/models/json-schema/role.ts index 3537de94c8..dd479e527b 100644 --- a/packages/backend/src/models/json-schema/role.ts +++ b/packages/backend/src/models/json-schema/role.ts @@ -66,7 +66,7 @@ export const packedRoleCondFormulaValueUserSettingBooleanSchema = { type: { type: 'string', nullable: false, optional: false, - enum: ['isSuspended', 'isLocked', 'isBot', 'isCat', 'isExplorable'], + enum: ['isSuspended', 'isLocked', 'isBot', 'isCat', 'isVI', 'isExplorable'], }, }, } as const; @@ -216,6 +216,10 @@ export const packedRolePoliciesSchema = { type: 'boolean', optional: false, nullable: false, }, + canUseTTS: { + type: 'boolean', + optional: false, nullable: false, + }, canHideAds: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/models/json-schema/user.ts b/packages/backend/src/models/json-schema/user.ts index 9cffd680f2..cd6ef5ec5e 100644 --- a/packages/backend/src/models/json-schema/user.ts +++ b/packages/backend/src/models/json-schema/user.ts @@ -115,6 +115,10 @@ export const packedUserLiteSchema = { type: 'boolean', nullable: false, optional: true, }, + isVI: { + type: 'boolean', + nullable: false, optional: true, + }, instance: { type: 'object', nullable: false, optional: true, diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 3557fa40a5..18b7decf7d 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -305,6 +305,7 @@ import * as ep___notes_threadMuting_create from './endpoints/notes/thread-muting import * as ep___notes_threadMuting_delete from './endpoints/notes/thread-muting/delete.js'; import * as ep___notes_timeline from './endpoints/notes/timeline.js'; import * as ep___notes_translate from './endpoints/notes/translate.js'; +import * as ep___notes_tts from './endpoints/notes/tts.js'; import * as ep___notes_unrenote from './endpoints/notes/unrenote.js'; import * as ep___notes_userListTimeline from './endpoints/notes/user-list-timeline.js'; import * as ep___notifications_create from './endpoints/notifications/create.js'; @@ -692,6 +693,7 @@ const $notes_threadMuting_create: Provider = { provide: 'ep:notes/thread-muting/ const $notes_threadMuting_delete: Provider = { provide: 'ep:notes/thread-muting/delete', useClass: ep___notes_threadMuting_delete.default }; const $notes_timeline: Provider = { provide: 'ep:notes/timeline', useClass: ep___notes_timeline.default }; const $notes_translate: Provider = { provide: 'ep:notes/translate', useClass: ep___notes_translate.default }; +const $notes_tts: Provider = { provide: 'ep:notes/tts', useClass: ep___notes_tts.default }; const $notes_unrenote: Provider = { provide: 'ep:notes/unrenote', useClass: ep___notes_unrenote.default }; const $notes_userListTimeline: Provider = { provide: 'ep:notes/user-list-timeline', useClass: ep___notes_userListTimeline.default }; const $notifications_create: Provider = { provide: 'ep:notifications/create', useClass: ep___notifications_create.default }; @@ -1083,6 +1085,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $notes_threadMuting_delete, $notes_timeline, $notes_translate, + $notes_tts, $notes_unrenote, $notes_userListTimeline, $notifications_create, @@ -1468,6 +1471,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $notes_threadMuting_delete, $notes_timeline, $notes_translate, + $notes_tts, $notes_unrenote, $notes_userListTimeline, $notifications_create, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 49b07d6ced..75f03603cc 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -311,6 +311,7 @@ import * as ep___notes_threadMuting_create from './endpoints/notes/thread-muting import * as ep___notes_threadMuting_delete from './endpoints/notes/thread-muting/delete.js'; import * as ep___notes_timeline from './endpoints/notes/timeline.js'; import * as ep___notes_translate from './endpoints/notes/translate.js'; +import * as ep___notes_tts from './endpoints/notes/tts.js'; import * as ep___notes_unrenote from './endpoints/notes/unrenote.js'; import * as ep___notes_userListTimeline from './endpoints/notes/user-list-timeline.js'; import * as ep___notifications_create from './endpoints/notifications/create.js'; @@ -696,6 +697,7 @@ const eps = [ ['notes/thread-muting/delete', ep___notes_threadMuting_delete], ['notes/timeline', ep___notes_timeline], ['notes/translate', ep___notes_translate], + ['notes/tts', ep___notes_tts], ['notes/unrenote', ep___notes_unrenote], ['notes/user-list-timeline', ep___notes_userListTimeline], ['notifications/create', ep___notifications_create], diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 64e3cc33bd..87ad35d8e0 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -122,6 +122,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + ttsAvailable: { + type: 'boolean', + optional: false, nullable: false, + }, silencedHosts: { type: 'array', optional: true, @@ -412,6 +416,58 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + hfAuthKey: { + type: 'string', + optional: false, nullable: true, + }, + hfSpace: { + type: 'boolean', + optional: false, nullable: false, + }, + hfSpaceName: { + type: 'string', + optional: false, nullable: true, + }, + hfexampleAudioURL: { + type: 'string', + optional: false, nullable: true, + }, + hfexampleText: { + type: 'string', + optional: false, nullable: true, + }, + hfexampleLang: { + type: 'string', + optional: false, nullable: true, + }, + hfslice: { + type: 'string', + optional: false, nullable: true, + }, + hftopK: { + type: 'number', + optional: false, nullable: true, + }, + hftopP: { + type: 'number', + optional: false, nullable: true, + }, + hfTemperature: { + type: 'number', + optional: false, nullable: true, + }, + hfSpeedRate: { + type: 'number', + optional: false, nullable: true, + }, + hfnrm: { + type: 'boolean', + optional: false, nullable: false, + }, + hfdas: { + type: 'boolean', + optional: false, nullable: false, + }, defaultDarkTheme: { type: 'string', optional: false, nullable: true, @@ -588,6 +644,7 @@ export default class extends Endpoint { // eslint- enableEmail: instance.enableEmail, enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, + ttsAvailable: instance.hfAuthKey != null, cacheRemoteFiles: instance.cacheRemoteFiles, cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, pinnedUsers: instance.pinnedUsers, @@ -630,6 +687,19 @@ export default class extends Endpoint { // eslint- objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, + hfAuthKey: instance.hfAuthKey, + hfSpace: instance.hfSpace, + hfSpaceName: instance.hfSpaceName, + hfexampleAudioURL: instance.hfexampleAudioURL, + hfexampleText: instance.hfexampleText, + hfexampleLang: instance.hfexampleLang, + hfslice: instance.hfslice, + hftopK: instance.hftopK, + hftopP: instance.hftopP, + hfTemperature: instance.hfTemperature, + hfSpeedRate: instance.hfSpeedRate, + hfnrm: instance.hfnrm, + hfdas: instance.hfdas, enableIpLogging: instance.enableIpLogging, enableActiveEmailValidation: instance.enableActiveEmailValidation, enableVerifymailApi: instance.enableVerifymailApi, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 38ef0d1de8..fed3c36192 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -98,6 +98,19 @@ export const paramDef = { }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, + hfAuthKey: { type: 'string', nullable: true }, + hfSpace: { type: 'boolean', default: false }, + hfSpaceName: { type: 'string', nullable: true }, + hfexampleAudioURL: { type: 'string', nullable: true }, + hfexampleText: { type: 'string', nullable: true }, + hfexampleLang: { type: 'string', nullable: true }, + hfslice: { type: 'string', default: 'Slice once every 4 sentences', nullable: true }, + hftopK: { type: 'integer', default: 15 }, + hftopP: { type: 'integer', default: 100 }, + hfTemperature: { type: 'integer', default: 100 }, + hfnrm: { type: 'boolean', default: false }, + hfSpeedRate: { type: 'integer', default: 125 }, + hfdas: { type: 'boolean', default: false }, enableEmail: { type: 'boolean' }, email: { type: 'string', nullable: true }, smtpSecure: { type: 'boolean' }, @@ -531,6 +544,82 @@ export default class extends Endpoint { // eslint- set.deeplIsPro = ps.deeplIsPro; } + if (ps.hfAuthKey !== undefined) { + if (ps.hfAuthKey === '') { + set.hfAuthKey = null; + } else { + set.hfAuthKey = ps.hfAuthKey; + } + } + + if (ps.hfSpace !== undefined) { + set.hfSpace = ps.hfSpace; + } + + if (ps.hfSpaceName !== undefined) { + if (ps.hfSpaceName === '') { + set.hfSpaceName = null; + } else { + set.hfSpaceName = ps.hfSpaceName; + } + } + + if (ps.hfexampleAudioURL !== undefined) { + if (ps.hfexampleAudioURL === '') { + set.hfexampleAudioURL = null; + } else { + set.hfexampleAudioURL = ps.hfexampleAudioURL; + } + } + + if (ps.hfexampleText !== undefined) { + if (ps.hfexampleText === '') { + set.hfexampleText = null; + } else { + set.hfexampleText = ps.hfexampleText; + } + } + + if (ps.hfexampleLang !== undefined) { + if (ps.hfexampleLang === '') { + set.hfexampleLang = null; + } else { + set.hfexampleLang = ps.hfexampleLang; + } + } + + if (ps.hfslice !== undefined) { + if (ps.hfslice === '') { + set.hfslice = null; + } else { + set.hfslice = ps.hfslice; + } + } + + if (ps.hftopK !== undefined) { + set.hftopK = ps.hftopK; + } + + if (ps.hftopP !== undefined) { + set.hftopP = ps.hftopP; + } + + if (ps.hfTemperature !== undefined) { + set.hfTemperature = ps.hfTemperature; + } + + if (ps.hfnrm !== undefined) { + set.hfnrm = ps.hfnrm; + } + + if (ps.hfSpeedRate !== undefined) { + set.hfSpeedRate = ps.hfSpeedRate; + } + + if (ps.hfdas !== undefined) { + set.hfdas = ps.hfdas; + } + if (ps.enableIpLogging !== undefined) { set.enableIpLogging = ps.enableIpLogging; } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 0b35005a87..286d50e35f 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -181,6 +181,7 @@ export const paramDef = { preventAiLearning: { type: 'boolean' }, isBot: { type: 'boolean' }, isCat: { type: 'boolean' }, + isVI: { type: 'boolean' }, injectFeaturedNote: { type: 'boolean' }, receiveAnnouncementEmail: { type: 'boolean' }, alwaysMarkNsfw: { type: 'boolean' }, @@ -335,6 +336,7 @@ export default class extends Endpoint { // eslint- if (typeof ps.noCrawle === 'boolean') profileUpdates.noCrawle = ps.noCrawle; if (typeof ps.preventAiLearning === 'boolean') profileUpdates.preventAiLearning = ps.preventAiLearning; if (typeof ps.isCat === 'boolean') updates.isCat = ps.isCat; + if (typeof ps.isVI === 'boolean') updates.isVI = ps.isVI; if (typeof ps.injectFeaturedNote === 'boolean') profileUpdates.injectFeaturedNote = ps.injectFeaturedNote; if (typeof ps.receiveAnnouncementEmail === 'boolean') profileUpdates.receiveAnnouncementEmail = ps.receiveAnnouncementEmail; if (typeof ps.alwaysMarkNsfw === 'boolean') { diff --git a/packages/backend/src/server/api/endpoints/notes/tts.ts b/packages/backend/src/server/api/endpoints/notes/tts.ts new file mode 100644 index 0000000000..3ffbe6863c --- /dev/null +++ b/packages/backend/src/server/api/endpoints/notes/tts.ts @@ -0,0 +1,197 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { Client } from "@gradio/client"; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { MetaService } from '@/core/MetaService.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { GetterService } from '@/server/api/GetterService.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '../../error.js'; + +export const meta = { + tags: ['notes'], + + requireCredential: true, + kind: 'read:account', + + res: { + type: 'string', + optional: true, nullable: false, + contentMediaType: 'audio/flac', + }, + + errors: { + incorrectconfig: { + message: 'Incorrect configuration.', + code: 'INCORRECT_CONFIG', + id: '8d171e60-83b8-11ef-b98c-a7506d6c1de4', + }, + unavailable: { + message: 'Convert of notes unavailable.', + code: 'UNAVAILABLE', + id: '97a0826c-6393-11ef-a650-67972d710975', + }, + noSuchNote: { + message: 'No such note.', + code: 'NO_SUCH_NOTE', + id: 'bea9b03f-36e0-49c5-a4db-627a029f8971', + }, + cannotConvertInvisibleNote: { + message: 'Cannot convert invisible note.', + code: 'CANNOT_CONVERT_INVISIBLE_NOTE', + id: 'f57caae0-6394-11ef-8e2a-d706932c1030', + }, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: { + noteId: { type: 'string', format: 'misskey:id' }, + }, + required: ['noteId'], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + private noteEntityService: NoteEntityService, + private getterService: GetterService, + private metaService: MetaService, + private httpRequestService: HttpRequestService, + private roleService: RoleService, + ) { + // @ts-expect-error: Functionality can be implemented here with minimal modifications. + super(meta, paramDef, async (ps, me) => { + const policies = await this.roleService.getUserPolicies(me.id); + if (!policies.canUseTTS) { + throw new ApiError(meta.errors.unavailable); + } + + const note = await this.getterService.getNote(ps.noteId).catch(err => { + if (err.id === '9725d0ce-ba28-4dde-95a7-2cbb2c15de24') throw new ApiError(meta.errors.noSuchNote); + throw err; + }); + + if (!(await this.noteEntityService.isVisibleForMe(note, me.id))) { + throw new ApiError(meta.errors.cannotConvertInvisibleNote); + } + + if (note.text == null) { + throw new ApiError(meta.errors.cannotConvertInvisibleNote); + } + + const instance = await this.metaService.fetch(); + + if (instance.hfAuthKey == null) { + throw new ApiError(meta.errors.unavailable); + } + + let outofQuota; + + if (instance.hfSpace) { + const langlist = ['Chinese', 'English', 'Japanese', 'Yue', 'Korean', 'Chinese-English Mixed', 'Japanese-English Mixed', 'Yue-English Mixed', 'Korean-English Mixed', 'Multilingual Mixed', 'Multilingual Mixed(Yue)']; + const slicelist = ['No slice', 'Slice once every 4 sentences', 'Slice per 50 characters', 'Slice by Chinese punct', 'Slice by English punct', 'Slice by every punct']; + let exampleAudio; + let app; + + try { + const example = await fetch(instance.hfexampleAudioURL || ''); + exampleAudio = await example.blob(); + } catch (e) { + throw new ApiError(meta.errors.unavailable); + } + + if (((!instance.hfnrm) && (!instance.hfexampleText)) || (!langlist.includes(instance.hfexampleLang || '')) || (!slicelist.includes(instance.hfslice || '')) || (!instance.hfSpaceName) || (!(instance.hfSpeedRate >= 60 && instance.hfSpeedRate <= 165)) || (!(instance.hfTemperature >= 0 && instance.hfTemperature <= 100)) || (!(instance.hftopK >= 0 && instance.hftopK <= 100)) || (!(instance.hftopP >= 0 && instance.hftopP <= 100))) { + throw new ApiError(meta.errors.incorrectconfig); + } + + try { + app = await Client.connect(instance.hfSpaceName, { hf_token: instance.hfAuthKey }); + } catch (e) { + throw new ApiError(meta.errors.unavailable); + } + + let result; + let notcontinue; + + try { + result = await app.predict("/get_tts_wav", [ + exampleAudio, + instance.hfexampleText, + instance.hfexampleLang, + note.text, + "Multilingual Mixed", + instance.hfslice, + instance.hftopK, + instance.hftopP / 100, + instance.hfTemperature / 100, + instance.hfnrm, + instance.hfSpeedRate / 100, + instance.hfdas, + ]); + } catch (e) { + const responseMessage = (e as any).message || ((e as any).original_msg && (e as any).original_msg.message); + + if (responseMessage && responseMessage.includes('You have exceeded your GPU quota')) { + outofQuota = true; + console.info("Fallback to Inference API"); + notcontinue = true; + } else { + throw new ApiError(meta.errors.unavailable); + } + } + + if (!notcontinue) { + const resurl = result.data[0].url; + + const res = await this.httpRequestService.send(resurl, { + method: 'GET', + headers: { + 'Authorization': 'Bearer ' + instance.hfAuthKey, + }, + timeout: 60000, + }); + + const contentType = res.headers.get('Content-Type') || 'application/octet-stream'; + + if (contentType === 'audio/x-wav') { + return res.body; + } else { + throw new ApiError(meta.errors.unavailable); + } + } + } + + if ((!instance.hfSpace) || ((instance.hfSpace) && (outofQuota))) { + const endpoint = 'https://api-inference.huggingface.co/models/suno/bark'; + + const res = await this.httpRequestService.send(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + instance.hfAuthKey, + Accept: 'audio/flac, */*', + }, + body: JSON.stringify({ + inputs: note.text, + }), + timeout: 60000, + }); + + const contentType = res.headers.get('Content-Type') || 'application/octet-stream'; + + if (contentType === 'audio/flac') { + return res.body; + } else { + throw new ApiError(meta.errors.unavailable); + } + } + }); + } +} diff --git a/packages/frontend-shared/js/const.ts b/packages/frontend-shared/js/const.ts index 4fe5cbb205..f4efe05f93 100644 --- a/packages/frontend-shared/js/const.ts +++ b/packages/frontend-shared/js/const.ts @@ -87,6 +87,7 @@ export const ROLE_POLICIES = [ 'canManageAvatarDecorations', 'canSearchNotes', 'canUseTranslator', + 'canUseTTS', 'canHideAds', 'driveCapacityMb', 'alwaysMarkNsfw', diff --git a/packages/frontend/src/components/MkNote.vue b/packages/frontend/src/components/MkNote.vue index 828ad2e872..fbb4990bd2 100644 --- a/packages/frontend/src/components/MkNote.vue +++ b/packages/frontend/src/components/MkNote.vue @@ -84,6 +84,14 @@ SPDX-License-Identifier: AGPL-3.0-only +
+ +
+ + {{ 'From ' + appearNote.id }} + +
+
@@ -167,7 +175,7 @@ SPDX-License-Identifier: AGPL-3.0-only