diff --git a/package.json b/package.json index 90a80cdad3..b604c75f9c 100644 --- a/package.json +++ b/package.json @@ -180,6 +180,7 @@ "markdown-it": "12.0.4", "markdown-it-anchor": "7.1.0", "matter-js": "0.16.1", + "mfm-js": "0.12.0", "mocha": "8.3.2", "moji": "0.5.1", "ms": "2.1.3", @@ -190,7 +191,6 @@ "object-assign-deep": "0.4.0", "os-utils": "0.0.14", "parse5": "6.0.1", - "parsimmon": "1.16.0", "pg": "8.5.1", "portscanner": "2.2.0", "postcss": "8.2.8", diff --git a/src/client/components/mfm.ts b/src/client/components/mfm.ts index 28ac9b8942..b8e948a188 100644 --- a/src/client/components/mfm.ts +++ b/src/client/components/mfm.ts @@ -1,6 +1,5 @@ import { VNode, defineComponent, h } from 'vue'; -import { MfmForest } from '@client/../mfm/prelude'; -import { parse, parsePlain } from '@client/../mfm/parse'; +import * as mfm from 'mfm-js'; import MkUrl from '@client/components/global/url.vue'; import MkLink from '@client/components/link.vue'; import MkMention from '@client/components/mention.vue'; @@ -46,17 +45,17 @@ export default defineComponent({ render() { if (this.text == null || this.text == '') return; - const ast = (this.plain ? parsePlain : parse)(this.text); + const ast = (this.plain ? mfm.parsePlain : mfm.parse)(this.text); const validTime = (t: string | null | undefined) => { if (t == null) return null; return t.match(/^[0-9.]+s$/) ? t : null; }; - const genEl = (ast: MfmForest) => concat(ast.map((token): VNode[] => { - switch (token.node.type) { + const genEl = (ast: mfm.MfmNode[]) => concat(ast.map((token): VNode[] => { + switch (token.type) { case 'text': { - const text = token.node.props.text.replace(/(\r\n|\n|\r)/g, '\n'); + const text = token.props.text.replace(/(\r\n|\n|\r)/g, '\n'); if (!this.plain) { const x = text.split('\n') @@ -83,38 +82,38 @@ export default defineComponent({ } case 'fn': { - // TODO: CSSを文字列で組み立てていくと token.node.props.args.~~~ 経由でCSSインジェクションできるのでよしなにやる + // TODO: CSSを文字列で組み立てていくと token.props.args.~~~ 経由でCSSインジェクションできるのでよしなにやる let style; - switch (token.node.props.name) { + switch (token.props.name) { case 'tada': { style = `font-size: 150%;` + (this.$store.state.animatedMfm ? 'animation: tada 1s linear infinite both;' : ''); break; } case 'jelly': { - const speed = validTime(token.node.props.args.speed) || '1s'; + const speed = validTime(token.props.args.speed) || '1s'; style = (this.$store.state.animatedMfm ? `animation: mfm-rubberBand ${speed} linear infinite both;` : ''); break; } case 'twitch': { - const speed = validTime(token.node.props.args.speed) || '0.5s'; + const speed = validTime(token.props.args.speed) || '0.5s'; style = this.$store.state.animatedMfm ? `animation: mfm-twitch ${speed} ease infinite;` : ''; break; } case 'shake': { - const speed = validTime(token.node.props.args.speed) || '0.5s'; + const speed = validTime(token.props.args.speed) || '0.5s'; style = this.$store.state.animatedMfm ? `animation: mfm-shake ${speed} ease infinite;` : ''; break; } case 'spin': { const direction = - token.node.props.args.left ? 'reverse' : - token.node.props.args.alternate ? 'alternate' : + token.props.args.left ? 'reverse' : + token.props.args.alternate ? 'alternate' : 'normal'; const anime = - token.node.props.args.x ? 'mfm-spinX' : - token.node.props.args.y ? 'mfm-spinY' : + token.props.args.x ? 'mfm-spinX' : + token.props.args.y ? 'mfm-spinY' : 'mfm-spin'; - const speed = validTime(token.node.props.args.speed) || '1.5s'; + const speed = validTime(token.props.args.speed) || '1.5s'; style = this.$store.state.animatedMfm ? `animation: ${anime} ${speed} linear infinite; animation-direction: ${direction};` : ''; break; } @@ -128,8 +127,8 @@ export default defineComponent({ } case 'flip': { const transform = - (token.node.props.args.h && token.node.props.args.v) ? 'scale(-1, -1)' : - token.node.props.args.v ? 'scaleY(-1)' : + (token.props.args.h && token.props.args.v) ? 'scale(-1, -1)' : + token.props.args.v ? 'scaleY(-1)' : 'scaleX(-1)'; style = `transform: ${transform};`; break; @@ -148,12 +147,12 @@ export default defineComponent({ } case 'font': { const family = - token.node.props.args.serif ? 'serif' : - token.node.props.args.monospace ? 'monospace' : - token.node.props.args.cursive ? 'cursive' : - token.node.props.args.fantasy ? 'fantasy' : - token.node.props.args.emoji ? 'emoji' : - token.node.props.args.math ? 'math' : + token.props.args.serif ? 'serif' : + token.props.args.monospace ? 'monospace' : + token.props.args.cursive ? 'cursive' : + token.props.args.fantasy ? 'fantasy' : + token.props.args.emoji ? 'emoji' : + token.props.args.math ? 'math' : null; if (family) style = `font-family: ${family};`; break; @@ -165,7 +164,7 @@ export default defineComponent({ } } if (style == null) { - return h('span', {}, ['[', token.node.props.name, ...genEl(token.children), ']']); + return h('span', {}, ['[', token.props.name, ...genEl(token.children), ']']); } else { return h('span', { style: 'display: inline-block;' + style, @@ -188,7 +187,7 @@ export default defineComponent({ case 'url': { return [h(MkUrl, { key: Math.random(), - url: token.node.props.url, + url: token.props.url, rel: 'nofollow noopener', })]; } @@ -196,7 +195,7 @@ export default defineComponent({ case 'link': { return [h(MkLink, { key: Math.random(), - url: token.node.props.url, + url: token.props.url, rel: 'nofollow noopener', }, genEl(token.children))]; } @@ -204,32 +203,31 @@ export default defineComponent({ case 'mention': { return [h(MkMention, { key: Math.random(), - host: (token.node.props.host == null && this.author && this.author.host != null ? this.author.host : token.node.props.host) || host, - username: token.node.props.username + host: (token.props.host == null && this.author && this.author.host != null ? this.author.host : token.props.host) || host, + username: token.props.username })]; } case 'hashtag': { return [h(MkA, { key: Math.random(), - to: this.isNote ? `/tags/${encodeURIComponent(token.node.props.hashtag)}` : `/explore/tags/${encodeURIComponent(token.node.props.hashtag)}`, + to: this.isNote ? `/tags/${encodeURIComponent(token.props.hashtag)}` : `/explore/tags/${encodeURIComponent(token.props.hashtag)}`, style: 'color:var(--hashtag);' - }, `#${token.node.props.hashtag}`)]; + }, `#${token.props.hashtag}`)]; } case 'blockCode': { return [h(MkCode, { key: Math.random(), - code: token.node.props.code, - lang: token.node.props.lang, + code: token.props.code, + lang: token.props.lang, })]; } case 'inlineCode': { return [h(MkCode, { key: Math.random(), - code: token.node.props.code, - lang: token.node.props.lang, + code: token.props.code, inline: true })]; } @@ -246,10 +244,19 @@ export default defineComponent({ } } - case 'emoji': { + case 'emojiCode': { return [h(MkEmoji, { key: Math.random(), - emoji: token.node.props.name ? `:${token.node.props.name}:` : token.node.props.emoji, + emoji: `:${token.props.name}:`, + customEmojis: this.customEmojis, + normal: this.plain + })]; + } + + case 'unicodeEmoji': { + return [h(MkEmoji, { + key: Math.random(), + emoji: token.props.emoji, customEmojis: this.customEmojis, normal: this.plain })]; @@ -258,7 +265,7 @@ export default defineComponent({ case 'mathInline': { return [h(MkFormula, { key: Math.random(), - formula: token.node.props.formula, + formula: token.props.formula, block: false })]; } @@ -266,7 +273,7 @@ export default defineComponent({ case 'mathBlock': { return [h(MkFormula, { key: Math.random(), - formula: token.node.props.formula, + formula: token.props.formula, block: true })]; } @@ -274,12 +281,12 @@ export default defineComponent({ case 'search': { return [h(MkGoogle, { key: Math.random(), - q: token.node.props.query + q: token.props.query })]; } default: { - console.error('unrecognized ast type:', token.node.type); + console.error('unrecognized ast type:', token.type); return []; } diff --git a/src/client/components/note-detailed.vue b/src/client/components/note-detailed.vue index fb4f9502b3..5124b2a88c 100644 --- a/src/client/components/note-detailed.vue +++ b/src/client/components/note-detailed.vue @@ -120,11 +120,11 @@ </template> <script lang="ts"> -import { computed, defineAsyncComponent, defineComponent, markRaw, ref } from 'vue'; +import { defineAsyncComponent, defineComponent, markRaw } from 'vue'; import { faSatelliteDish, faBolt, faTimes, faBullhorn, faStar, faLink, faExternalLinkSquareAlt, faPlus, faMinus, faRetweet, faReply, faReplyAll, faEllipsisH, faHome, faUnlock, faEnvelope, faThumbtack, faBan, faQuoteRight, faInfoCircle, faBiohazard, faPlug, faExclamationCircle, faPaperclip } from '@fortawesome/free-solid-svg-icons'; import { faCopy, faTrashAlt, faEdit, faEye, faEyeSlash } from '@fortawesome/free-regular-svg-icons'; -import { parse } from '../../mfm/parse'; -import { sum, unique } from '../../prelude/array'; +import * as mfm from 'mfm-js'; +import { sum } from '../../prelude/array'; import XSub from './note.sub.vue'; import XNoteHeader from './note-header.vue'; import XNotePreview from './note-preview.vue'; @@ -141,6 +141,7 @@ import { userPage } from '@client/filters/user'; import * as os from '@client/os'; import { noteActions, noteViewInterruptors } from '@client/store'; import { reactionPicker } from '@client/scripts/reaction-picker'; +import { extractUrlFromMfm } from '@/misc/extract-url-from-mfm'; function markRawAll(...xs) { for (const x of xs) { @@ -252,21 +253,7 @@ export default defineComponent({ urls(): string[] { if (this.appearNote.text) { - const ast = parse(this.appearNote.text); - // TODO: 再帰的にURL要素がないか調べる - const urls = unique(ast - .filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent)) - .map(t => t.node.props.url)); - - // unique without hash - // [ http://a/#1, http://a/#2, http://b/#3 ] => [ http://a/#1, http://b/#3 ] - const removeHash = x => x.replace(/#[^#]*$/, ''); - - return urls.reduce((array, url) => { - const removed = removeHash(url); - if (!array.map(x => removeHash(x)).includes(removed)) array.push(url); - return array; - }, []); + return extractUrlFromMfm(mfm.parse(this.appearNote.text)); } else { return null; } diff --git a/src/client/components/note.vue b/src/client/components/note.vue index b54cadfc80..a656ffc356 100644 --- a/src/client/components/note.vue +++ b/src/client/components/note.vue @@ -102,11 +102,11 @@ </template> <script lang="ts"> -import { computed, defineAsyncComponent, defineComponent, markRaw, ref } from 'vue'; +import { defineAsyncComponent, defineComponent, markRaw } from 'vue'; import { faSatelliteDish, faBolt, faTimes, faBullhorn, faStar, faLink, faExternalLinkSquareAlt, faPlus, faMinus, faRetweet, faReply, faReplyAll, faEllipsisH, faHome, faUnlock, faEnvelope, faThumbtack, faBan, faQuoteRight, faInfoCircle, faBiohazard, faPlug, faExclamationCircle, faPaperclip } from '@fortawesome/free-solid-svg-icons'; import { faCopy, faTrashAlt, faEdit, faEye, faEyeSlash } from '@fortawesome/free-regular-svg-icons'; -import { parse } from '../../mfm/parse'; -import { sum, unique } from '../../prelude/array'; +import * as mfm from 'mfm-js'; +import { sum } from '../../prelude/array'; import XSub from './note.sub.vue'; import XNoteHeader from './note-header.vue'; import XNotePreview from './note-preview.vue'; @@ -123,6 +123,7 @@ import { userPage } from '@client/filters/user'; import * as os from '@client/os'; import { noteActions, noteViewInterruptors } from '@client/store'; import { reactionPicker } from '@client/scripts/reaction-picker'; +import { extractUrlFromMfm } from '@/misc/extract-url-from-mfm'; function markRawAll(...xs) { for (const x of xs) { @@ -238,21 +239,7 @@ export default defineComponent({ urls(): string[] { if (this.appearNote.text) { - const ast = parse(this.appearNote.text); - // TODO: 再帰的にURL要素がないか調べる - const urls = unique(ast - .filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent)) - .map(t => t.node.props.url)); - - // unique without hash - // [ http://a/#1, http://a/#2, http://b/#3 ] => [ http://a/#1, http://b/#3 ] - const removeHash = x => x.replace(/#[^#]*$/, ''); - - return urls.reduce((array, url) => { - const removed = removeHash(url); - if (!array.map(x => removeHash(x)).includes(removed)) array.push(url); - return array; - }, []); + return extractUrlFromMfm(mfm.parse(this.appearNote.text)); } else { return null; } diff --git a/src/client/components/page/page.text.vue b/src/client/components/page/page.text.vue index 491c62be26..580c5a93bf 100644 --- a/src/client/components/page/page.text.vue +++ b/src/client/components/page/page.text.vue @@ -9,8 +9,8 @@ import { TextBlock } from '@client/scripts/hpml/block'; import { Hpml } from '@client/scripts/hpml/evaluator'; import { defineAsyncComponent, defineComponent, PropType } from 'vue'; -import { parse } from '../../../mfm/parse'; -import { unique } from '../../../prelude/array'; +import * as mfm from 'mfm-js'; +import { extractUrlFromMfm } from '@/misc/extract-url-from-mfm'; export default defineComponent({ components: { @@ -34,11 +34,7 @@ export default defineComponent({ computed: { urls(): string[] { if (this.text) { - const ast = parse(this.text); - // TODO: 再帰的にURL要素がないか調べる - return unique(ast - .filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent)) - .map(t => t.node.props.url)); + return extractUrlFromMfm(mfm.parse(this.text)); } else { return []; } diff --git a/src/client/components/post-form.vue b/src/client/components/post-form.vue index 7d2355c190..13e5c0f433 100644 --- a/src/client/components/post-form.vue +++ b/src/client/components/post-form.vue @@ -58,7 +58,7 @@ import insertTextAtCursor from 'insert-text-at-cursor'; import { length } from 'stringz'; import { toASCII } from 'punycode'; import XNotePreview from './note-preview.vue'; -import { parse } from '../../mfm/parse'; +import * as mfm from 'mfm-js'; import { host, url } from '@client/config'; import { erase, unique } from '../../prelude/array'; import extractMentions from '@/misc/extract-mentions'; @@ -229,7 +229,7 @@ export default defineComponent({ } if (this.reply && this.reply.text != null) { - const ast = parse(this.reply.text); + const ast = mfm.parse(this.reply.text); for (const x of extractMentions(ast)) { const mention = x.host ? `@${x.username}@${toASCII(x.host)}` : `@${x.username}`; @@ -580,7 +580,7 @@ export default defineComponent({ this.deleteDraft(); this.$emit('posted'); if (this.text && this.text != '') { - const hashtags = parse(this.text).filter(x => x.node.type === 'hashtag').map(x => x.node.props.hashtag); + const hashtags = mfm.parse(this.text).filter(x => x.type === 'hashtag').map(x => x.props.hashtag); const history = JSON.parse(localStorage.getItem('hashtags') || '[]') as string[]; localStorage.setItem('hashtags', JSON.stringify(unique(hashtags.concat(history)))); } diff --git a/src/client/pages/about-misskey.vue b/src/client/pages/about-misskey.vue index e9e2e15573..72b94968df 100644 --- a/src/client/pages/about-misskey.vue +++ b/src/client/pages/about-misskey.vue @@ -40,6 +40,7 @@ <FormLink to="https://github.com/rinsuki" external>@rinsuki</FormLink> <FormLink to="https://github.com/Xeltica" external>@Xeltica</FormLink> <FormLink to="https://github.com/u1-liquid" external>@u1-liquid</FormLink> + <FormLink to="https://github.com/marihachi" external>@marihachi</FormLink> <template #caption><MkLink url="https://github.com/misskey-dev/misskey/graphs/contributors">{{ $ts._aboutMisskey.allContributors }}</MkLink></template> </FormGroup> <FormGroup> diff --git a/src/client/pages/messaging/messaging-room.message.vue b/src/client/pages/messaging/messaging-room.message.vue index a6d142bd34..3755bc2b5c 100644 --- a/src/client/pages/messaging/messaging-room.message.vue +++ b/src/client/pages/messaging/messaging-room.message.vue @@ -37,8 +37,8 @@ <script lang="ts"> import { defineComponent } from 'vue'; -import { parse } from '../../../mfm/parse'; -import { unique } from '../../../prelude/array'; +import * as mfm from 'mfm-js'; +import { extractUrlFromMfm } from '@/misc/extract-url-from-mfm'; import MkUrlPreview from '@client/components/url-preview.vue'; import * as os from '@client/os'; @@ -60,10 +60,7 @@ export default defineComponent({ }, urls(): string[] { if (this.message.text) { - const ast = parse(this.message.text); - return unique(ast - .filter(t => ((t.node.type === 'url' || t.node.type === 'link') && t.node.props.url && !t.node.props.silent)) - .map(t => t.node.props.url)); + return extractUrlFromMfm(mfm.parse(this.message.text)); } else { return []; } diff --git a/src/client/ui/chat/note.vue b/src/client/ui/chat/note.vue index 4afd7989e1..f6789f214d 100644 --- a/src/client/ui/chat/note.vue +++ b/src/client/ui/chat/note.vue @@ -101,11 +101,11 @@ </template> <script lang="ts"> -import { computed, defineAsyncComponent, defineComponent, markRaw, ref } from 'vue'; +import { defineAsyncComponent, defineComponent, markRaw } from 'vue'; import { faSatelliteDish, faBolt, faTimes, faBullhorn, faStar, faLink, faExternalLinkSquareAlt, faPlus, faMinus, faRetweet, faReply, faReplyAll, faEllipsisH, faHome, faUnlock, faEnvelope, faThumbtack, faBan, faQuoteRight, faInfoCircle, faBiohazard, faPlug, faExclamationCircle, faPaperclip } from '@fortawesome/free-solid-svg-icons'; import { faCopy, faTrashAlt, faEdit, faEye, faEyeSlash } from '@fortawesome/free-regular-svg-icons'; -import { parse } from '../../../mfm/parse'; -import { sum, unique } from '../../../prelude/array'; +import * as mfm from 'mfm-js'; +import { sum } from '../../../prelude/array'; import XSub from './note.sub.vue'; import XNoteHeader from './note-header.vue'; import XNotePreview from './note-preview.vue'; @@ -122,6 +122,7 @@ import { userPage } from '@client/filters/user'; import * as os from '@client/os'; import { noteActions, noteViewInterruptors } from '@client/store'; import { reactionPicker } from '@client/scripts/reaction-picker'; +import { extractUrlFromMfm } from '@/misc/extract-url-from-mfm'; function markRawAll(...xs) { for (const x of xs) { @@ -238,21 +239,7 @@ export default defineComponent({ urls(): string[] { if (this.appearNote.text) { - const ast = parse(this.appearNote.text); - // TODO: 再帰的にURL要素がないか調べる - const urls = unique(ast - .filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent)) - .map(t => t.node.props.url)); - - // unique without hash - // [ http://a/#1, http://a/#2, http://b/#3 ] => [ http://a/#1, http://b/#3 ] - const removeHash = x => x.replace(/#[^#]*$/, ''); - - return urls.reduce((array, url) => { - const removed = removeHash(url); - if (!array.map(x => removeHash(x)).includes(removed)) array.push(url); - return array; - }, []); + return extractUrlFromMfm(mfm.parse(this.appearNote.text)); } else { return null; } diff --git a/src/client/ui/chat/post-form.vue b/src/client/ui/chat/post-form.vue index 5bb1a04d58..e5f4132c4b 100644 --- a/src/client/ui/chat/post-form.vue +++ b/src/client/ui/chat/post-form.vue @@ -53,7 +53,7 @@ import { faEyeSlash, faLaughSquint } from '@fortawesome/free-regular-svg-icons'; import insertTextAtCursor from 'insert-text-at-cursor'; import { length } from 'stringz'; import { toASCII } from 'punycode'; -import { parse } from '../../../mfm/parse'; +import * as mfm from 'mfm-js'; import { host, url } from '@client/config'; import { erase, unique } from '../../../prelude/array'; import extractMentions from '@/misc/extract-mentions'; @@ -216,7 +216,7 @@ export default defineComponent({ } if (this.reply && this.reply.text != null) { - const ast = parse(this.reply.text); + const ast = mfm.parse(this.reply.text); for (const x of extractMentions(ast)) { const mention = x.host ? `@${x.username}@${toASCII(x.host)}` : `@${x.username}`; @@ -567,7 +567,7 @@ export default defineComponent({ this.deleteDraft(); this.$emit('posted'); if (this.text && this.text != '') { - const hashtags = parse(this.text).filter(x => x.node.type === 'hashtag').map(x => x.node.props.hashtag); + const hashtags = mfm.parse(this.text).filter(x => x.type === 'hashtag').map(x => x.props.hashtag); const history = JSON.parse(localStorage.getItem('hashtags') || '[]') as string[]; localStorage.setItem('hashtags', JSON.stringify(unique(hashtags.concat(history)))); } diff --git a/src/mfm/from-html.ts b/src/mfm/from-html.ts index 0b4f9b8945..4c8e2dbec8 100644 --- a/src/mfm/from-html.ts +++ b/src/mfm/from-html.ts @@ -1,7 +1,9 @@ import * as parse5 from 'parse5'; import treeAdapter = require('parse5/lib/tree-adapters/default'); import { URL } from 'url'; -import { urlRegex, urlRegexFull } from './prelude'; + +const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; +const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; export function fromHtml(html: string, hashtagNames?: string[]): string { const dom = parse5.parseFragment(html); diff --git a/src/mfm/language.ts b/src/mfm/language.ts deleted file mode 100644 index bad7b10a0d..0000000000 --- a/src/mfm/language.ts +++ /dev/null @@ -1,191 +0,0 @@ -import * as P from 'parsimmon'; -import { createLeaf, createTree, urlRegex } from './prelude'; -import { takeWhile, cumulativeSum } from '../prelude/array'; -import parseAcct from '@/misc/acct/parse'; -import { toUnicode } from 'punycode'; -import { emojiRegex } from '@/misc/emoji-regex'; - -export function removeOrphanedBrackets(s: string): string { - const openBrackets = ['(', '「', '[']; - const closeBrackets = [')', '」', ']']; - const xs = cumulativeSum(s.split('').map(c => { - if (openBrackets.includes(c)) return 1; - if (closeBrackets.includes(c)) return -1; - return 0; - })); - const firstOrphanedCloseBracket = xs.findIndex(x => x < 0); - if (firstOrphanedCloseBracket !== -1) return s.substr(0, firstOrphanedCloseBracket); - const lastMatched = xs.lastIndexOf(0); - return s.substr(0, lastMatched + 1); -} - -export const mfmLanguage = P.createLanguage({ - root: r => P.alt(r.block, r.inline).atLeast(1), - plain: r => P.alt(r.emoji, r.text).atLeast(1), - block: r => P.alt( - r.quote, - r.search, - r.blockCode, - r.mathBlock, - r.center, - ), - startOfLine: () => P((input, i) => { - if (i === 0 || input[i] === '\n' || input[i - 1] === '\n') { - return P.makeSuccess(i, null); - } else { - return P.makeFailure(i, 'not newline'); - } - }), - quote: r => r.startOfLine.then(P((input, i) => { - const text = input.substr(i); - if (!text.match(/^>[\s\S]+?/)) return P.makeFailure(i, 'not a quote'); - const quote = takeWhile(line => line.startsWith('>'), text.split('\n')); - const qInner = quote.join('\n').replace(/^>/gm, '').replace(/^ /gm, ''); - if (qInner === '') return P.makeFailure(i, 'not a quote'); - const contents = r.root.tryParse(qInner); - return P.makeSuccess(i + quote.join('\n').length + 1, createTree('quote', contents, {})); - })), - search: r => r.startOfLine.then(P((input, i) => { - const text = input.substr(i); - const match = text.match(/^(.+?)( | )(検索|\[検索\]|Search|\[Search\])(\n|$)/i); - if (!match) return P.makeFailure(i, 'not a search'); - return P.makeSuccess(i + match[0].length, createLeaf('search', { query: match[1], content: match[0].trim() })); - })), - blockCode: r => r.startOfLine.then(P((input, i) => { - const text = input.substr(i); - const match = text.match(/^```(.+?)?\n([\s\S]+?)\n```(\n|$)/i); - if (!match) return P.makeFailure(i, 'not a blockCode'); - return P.makeSuccess(i + match[0].length, createLeaf('blockCode', { code: match[2], lang: match[1] ? match[1].trim() : null })); - })), - inline: r => P.alt( - r.big, - r.bold, - r.small, - r.italic, - r.strike, - r.inlineCode, - r.mathInline, - r.mention, - r.hashtag, - r.url, - r.link, - r.emoji, - r.fn, - r.text - ), - // TODO: そのうち消す - big: r => P.regexp(/^\*\*\*([\s\S]+?)\*\*\*/, 1).map(x => createTree('fn', r.inline.atLeast(1).tryParse(x), { - name: 'tada', - args: {} - })), - bold: r => { - const asterisk = P.regexp(/\*\*([\s\S]+?)\*\*/, 1); - const underscore = P.regexp(/__([a-zA-Z0-9\s]+?)__/, 1); - return P.alt(asterisk, underscore).map(x => createTree('bold', r.inline.atLeast(1).tryParse(x), {})); - }, - small: r => P.regexp(/<small>([\s\S]+?)<\/small>/, 1).map(x => createTree('small', r.inline.atLeast(1).tryParse(x), {})), - italic: r => { - const xml = P.regexp(/<i>([\s\S]+?)<\/i>/, 1); - const underscore = P((input, i) => { - const text = input.substr(i); - const match = text.match(/^(\*|_)([a-zA-Z0-9]+?[\s\S]*?)\1/); - if (!match) return P.makeFailure(i, 'not a italic'); - if (input[i - 1] != null && input[i - 1] != ' ' && input[i - 1] != '\n') return P.makeFailure(i, 'not a italic'); - return P.makeSuccess(i + match[0].length, match[2]); - }); - - return P.alt(xml, underscore).map(x => createTree('italic', r.inline.atLeast(1).tryParse(x), {})); - }, - strike: r => P.regexp(/~~([^\n~]+?)~~/, 1).map(x => createTree('strike', r.inline.atLeast(1).tryParse(x), {})), - center: r => r.startOfLine.then(P.regexp(/<center>([\s\S]+?)<\/center>/, 1).map(x => createTree('center', r.inline.atLeast(1).tryParse(x), {}))), - inlineCode: () => P.regexp(/`([^´\n]+?)`/, 1).map(x => createLeaf('inlineCode', { code: x })), - mathBlock: r => r.startOfLine.then(P.regexp(/\\\[([\s\S]+?)\\\]/, 1).map(x => createLeaf('mathBlock', { formula: x.trim() }))), - mathInline: () => P.regexp(/\\\((.+?)\\\)/, 1).map(x => createLeaf('mathInline', { formula: x })), - mention: () => { - return P((input, i) => { - const text = input.substr(i); - const match = text.match(/^@\w([\w-]*\w)?(?:@[\w.\-]+\w)?/); - if (!match) return P.makeFailure(i, 'not a mention'); - if (input[i - 1] != null && input[i - 1].match(/[a-z0-9]/i)) return P.makeFailure(i, 'not a mention'); - return P.makeSuccess(i + match[0].length, match[0]); - }).map(x => { - const { username, host } = parseAcct(x.substr(1)); - const canonical = host != null ? `@${username}@${toUnicode(host)}` : x; - return createLeaf('mention', { canonical, username, host, acct: x }); - }); - }, - hashtag: () => P((input, i) => { - const text = input.substr(i); - const match = text.match(/^#([^\s.,!?'"#:\/\[\]【】]+)/i); - if (!match) return P.makeFailure(i, 'not a hashtag'); - let hashtag = match[1]; - hashtag = removeOrphanedBrackets(hashtag); - if (hashtag.match(/^(\u20e3|\ufe0f)/)) return P.makeFailure(i, 'not a hashtag'); - if (hashtag.match(/^[0-9]+$/)) return P.makeFailure(i, 'not a hashtag'); - if (input[i - 1] != null && input[i - 1].match(/[a-z0-9]/i)) return P.makeFailure(i, 'not a hashtag'); - if (Array.from(hashtag || '').length > 128) return P.makeFailure(i, 'not a hashtag'); - return P.makeSuccess(i + ('#' + hashtag).length, createLeaf('hashtag', { hashtag: hashtag })); - }), - url: () => { - return P((input, i) => { - const text = input.substr(i); - const match = text.match(urlRegex); - let url: string; - if (!match) { - const match = text.match(/^<(https?:\/\/.*?)>/); - if (!match) { - return P.makeFailure(i, 'not a url'); - } - url = match[1]; - i += 2; - } else { - url = match[0]; - } - url = removeOrphanedBrackets(url); - url = url.replace(/[.,]*$/, ''); - return P.makeSuccess(i + url.length, url); - }).map(x => createLeaf('url', { url: x })); - }, - link: r => { - return P.seqObj( - ['silent', P.string('?').fallback(null).map(x => x != null)] as any, - P.string('['), ['text', P.regexp(/[^\n\[\]]+/)] as any, P.string(']'), - P.string('('), ['url', r.url] as any, P.string(')'), - ).map((x: any) => { - return createTree('link', r.inline.atLeast(1).tryParse(x.text), { - silent: x.silent, - url: x.url.node.props.url - }); - }); - }, - emoji: () => { - const name = P.regexp(/:([a-z0-9_+-]+):/i, 1).map(x => createLeaf('emoji', { name: x })); - const code = P.regexp(emojiRegex).map(x => createLeaf('emoji', { emoji: x })); - return P.alt(name, code); - }, - fn: r => { - return P.seqObj( - P.string('['), ['fn', P.regexp(/[^\s\n\[\]]+/)] as any, P.string(' '), P.optWhitespace, ['text', P.regexp(/[^\n\[\]]+/)] as any, P.string(']'), - ).map((x: any) => { - let name = x.fn; - const args = {}; - const separator = x.fn.indexOf('.'); - if (separator > -1) { - name = x.fn.substr(0, separator); - for (const arg of x.fn.substr(separator + 1).split(',')) { - const kv = arg.split('='); - if (kv.length === 1) { - args[kv[0]] = true; - } else { - args[kv[0]] = kv[1]; - } - } - } - return createTree('fn', r.inline.atLeast(1).tryParse(x.text), { - name, - args - }); - }); - }, - text: () => P.any.map(x => createLeaf('text', { text: x })) -}); diff --git a/src/mfm/normalize.ts b/src/mfm/normalize.ts deleted file mode 100644 index a0f0702096..0000000000 --- a/src/mfm/normalize.ts +++ /dev/null @@ -1,31 +0,0 @@ -import * as A from '../prelude/array'; -import * as S from '../prelude/string'; -import { MfmForest, MfmTree } from './prelude'; -import { createTree, createLeaf } from '../prelude/tree'; - -function isEmptyTextTree(t: MfmTree): boolean { - return t.node.type === 'text' && t.node.props.text === ''; -} - -function concatTextTrees(ts: MfmForest): MfmTree { - return createLeaf({ type: 'text', props: { text: S.concat(ts.map(x => x.node.props.text)) } }); -} - -function concatIfTextTrees(ts: MfmForest): MfmForest { - return ts[0].node.type === 'text' ? [concatTextTrees(ts)] : ts; -} - -function concatConsecutiveTextTrees(ts: MfmForest): MfmForest { - const us = A.concat(A.groupOn(t => t.node.type, ts).map(concatIfTextTrees)); - return us.map(t => createTree(t.node, concatConsecutiveTextTrees(t.children))); -} - -function removeEmptyTextNodes(ts: MfmForest): MfmForest { - return ts - .filter(t => !isEmptyTextTree(t)) - .map(t => createTree(t.node, removeEmptyTextNodes(t.children))); -} - -export function normalize(ts: MfmForest): MfmForest { - return removeEmptyTextNodes(concatConsecutiveTextTrees(ts)); -} diff --git a/src/mfm/parse.ts b/src/mfm/parse.ts deleted file mode 100644 index c628042f12..0000000000 --- a/src/mfm/parse.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { mfmLanguage } from './language'; -import { MfmForest } from './prelude'; -import { normalize } from './normalize'; - -export function parse(source: string | null): MfmForest | null { - if (source == null || source === '') { - return null; - } - - return normalize(mfmLanguage.root.tryParse(source)); -} - -export function parsePlain(source: string | null): MfmForest | null { - if (source == null || source === '') { - return null; - } - - return normalize(mfmLanguage.plain.tryParse(source)); -} diff --git a/src/mfm/prelude.ts b/src/mfm/prelude.ts deleted file mode 100644 index a8b52eb315..0000000000 --- a/src/mfm/prelude.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Tree } from '../prelude/tree'; -import * as T from '../prelude/tree'; - -type Node<T, P> = { type: T, props: P }; - -export type MentionNode = Node<'mention', { - canonical: string, - username: string, - host: string, - acct: string -}>; - -export type HashtagNode = Node<'hashtag', { - hashtag: string -}>; - -export type EmojiNode = Node<'emoji', { - name: string -}>; - -export type MfmNode = - MentionNode | - HashtagNode | - EmojiNode | - Node<string, any>; - -export type MfmTree = Tree<MfmNode>; - -export type MfmForest = MfmTree[]; - -export function createLeaf(type: string, props: any): MfmTree { - return T.createLeaf({ type, props }); -} - -export function createTree(type: string, children: MfmForest, props: any): MfmTree { - return T.createTree({ type, props }, children); -} - -export const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; -export const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; diff --git a/src/mfm/to-html.ts b/src/mfm/to-html.ts index 66015d539f..aa39443c64 100644 --- a/src/mfm/to-html.ts +++ b/src/mfm/to-html.ts @@ -1,12 +1,12 @@ import { JSDOM } from 'jsdom'; +import * as mfm from 'mfm-js'; import config from '@/config'; import { intersperse } from '../prelude/array'; -import { MfmForest, MfmTree } from './prelude'; import { IMentionedRemoteUsers } from '../models/entities/note'; import { wellKnownServices } from '../well-known-services'; -export function toHtml(tokens: MfmForest | null, mentionedRemoteUsers: IMentionedRemoteUsers = []) { - if (tokens == null) { +export function toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = []) { + if (nodes == null) { return null; } @@ -14,95 +14,101 @@ export function toHtml(tokens: MfmForest | null, mentionedRemoteUsers: IMentione const doc = window.document; - function appendChildren(children: MfmForest, targetElement: any): void { - for (const child of children.map(t => handlers[t.node.type](t))) targetElement.appendChild(child); + function appendChildren(children: mfm.MfmNode[], targetElement: any): void { + if (children) { + for (const child of children.map(x => (handlers as any)[x.type](x))) targetElement.appendChild(child); + } } - const handlers: { [key: string]: (token: MfmTree) => any } = { - bold(token) { + const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any } = { + bold(node) { const el = doc.createElement('b'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - small(token) { + small(node) { const el = doc.createElement('small'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - strike(token) { + strike(node) { const el = doc.createElement('del'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - italic(token) { + italic(node) { const el = doc.createElement('i'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - fn(token) { + fn(node) { const el = doc.createElement('i'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - blockCode(token) { + blockCode(node) { const pre = doc.createElement('pre'); const inner = doc.createElement('code'); - inner.textContent = token.node.props.code; + inner.textContent = node.props.code; pre.appendChild(inner); return pre; }, - center(token) { + center(node) { const el = doc.createElement('div'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - emoji(token) { - return doc.createTextNode(token.node.props.emoji ? token.node.props.emoji : `\u200B:${token.node.props.name}:\u200B`); + emojiCode(node) { + return doc.createTextNode(`\u200B:${node.props.name}:\u200B`); }, - hashtag(token) { + unicodeEmoji(node) { + return doc.createTextNode(node.props.emoji); + }, + + hashtag(node) { const a = doc.createElement('a'); - a.href = `${config.url}/tags/${token.node.props.hashtag}`; - a.textContent = `#${token.node.props.hashtag}`; + a.href = `${config.url}/tags/${node.props.hashtag}`; + a.textContent = `#${node.props.hashtag}`; a.setAttribute('rel', 'tag'); return a; }, - inlineCode(token) { + inlineCode(node) { const el = doc.createElement('code'); - el.textContent = token.node.props.code; + el.textContent = node.props.code; return el; }, - mathInline(token) { + mathInline(node) { const el = doc.createElement('code'); - el.textContent = token.node.props.formula; + el.textContent = node.props.formula; return el; }, - mathBlock(token) { + mathBlock(node) { const el = doc.createElement('code'); - el.textContent = token.node.props.formula; + el.textContent = node.props.formula; return el; }, - link(token) { + link(node) { const a = doc.createElement('a'); - a.href = token.node.props.url; - appendChildren(token.children, a); + a.href = node.props.url; + appendChildren(node.children, a); return a; }, - mention(token) { + mention(node) { const a = doc.createElement('a'); - const { username, host, acct } = token.node.props; + const { username, host, acct } = node.props; const wellKnown = wellKnownServices.find(x => x[0] === host); if (wellKnown) { a.href = wellKnown[1](username); @@ -115,39 +121,39 @@ export function toHtml(tokens: MfmForest | null, mentionedRemoteUsers: IMentione return a; }, - quote(token) { + quote(node) { const el = doc.createElement('blockquote'); - appendChildren(token.children, el); + appendChildren(node.children, el); return el; }, - text(token) { + text(node) { const el = doc.createElement('span'); - const nodes = (token.node.props.text as string).split(/\r\n|\r|\n/).map(x => doc.createTextNode(x) as Node); + const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => doc.createTextNode(x)); - for (const x of intersperse<Node | 'br'>('br', nodes)) { + for (const x of intersperse<FIXME | 'br'>('br', nodes)) { el.appendChild(x === 'br' ? doc.createElement('br') : x); } return el; }, - url(token) { + url(node) { const a = doc.createElement('a'); - a.href = token.node.props.url; - a.textContent = token.node.props.url; + a.href = node.props.url; + a.textContent = node.props.url; return a; }, - search(token) { + search(node) { const a = doc.createElement('a'); - a.href = `https://www.google.com/search?q=${token.node.props.query}`; - a.textContent = token.node.props.content; + a.href = `https://www.google.com/search?q=${node.props.query}`; + a.textContent = node.props.content; return a; } }; - appendChildren(tokens, doc.body); + appendChildren(nodes, doc.body); return `<p>${doc.body.innerHTML}</p>`; } diff --git a/src/mfm/to-string.ts b/src/mfm/to-string.ts deleted file mode 100644 index 347c94c247..0000000000 --- a/src/mfm/to-string.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { MfmForest, MfmTree } from './prelude'; -import { nyaize } from '@/misc/nyaize'; - -export type RestoreOptions = { - doNyaize?: boolean; -}; - -export function toString(tokens: MfmForest | null, opts?: RestoreOptions): string { - - if (tokens === null) return ''; - - function appendChildren(children: MfmForest, opts?: RestoreOptions): string { - return children.map(t => handlers[t.node.type](t, opts)).join(''); - } - - const handlers: { [key: string]: (token: MfmTree, opts?: RestoreOptions) => string } = { - bold(token, opts) { - return `**${appendChildren(token.children, opts)}**`; - }, - - small(token, opts) { - return `<small>${appendChildren(token.children, opts)}</small>`; - }, - - strike(token, opts) { - return `~~${appendChildren(token.children, opts)}~~`; - }, - - italic(token, opts) { - return `<i>${appendChildren(token.children, opts)}</i>`; - }, - - fn(token, opts) { - const name = token.node.props?.name; - const args = token.node.props?.args || {}; - const argsStr = Object.entries(args).map(([k, v]) => v === true ? k : `${k}=${v}`).join(','); - return `[${name}${argsStr !== '' ? '.' + argsStr : ''} ${appendChildren(token.children, opts)}]`; - }, - - blockCode(token) { - return `\`\`\`${token.node.props.lang || ''}\n${token.node.props.code}\n\`\`\`\n`; - }, - - center(token, opts) { - return `<center>${appendChildren(token.children, opts)}</center>`; - }, - - emoji(token) { - return (token.node.props.emoji ? token.node.props.emoji : `:${token.node.props.name}:`); - }, - - hashtag(token) { - return `#${token.node.props.hashtag}`; - }, - - inlineCode(token) { - return `\`${token.node.props.code}\``; - }, - - mathInline(token) { - return `\\(${token.node.props.formula}\\)`; - }, - - mathBlock(token) { - return `\\[${token.node.props.formula}\\]`; - }, - - link(token, opts) { - if (token.node.props.silent) { - return `?[${appendChildren(token.children, opts)}](${token.node.props.url})`; - } else { - return `[${appendChildren(token.children, opts)}](${token.node.props.url})`; - } - }, - - mention(token) { - return token.node.props.canonical; - }, - - quote(token) { - return `${appendChildren(token.children, {doNyaize: false}).replace(/^/gm,'>').trim()}\n`; - }, - - text(token, opts) { - return (opts && opts.doNyaize) ? nyaize(token.node.props.text) : token.node.props.text; - }, - - url(token) { - return `<${token.node.props.url}>`; - }, - - search(token, opts) { - const query = token.node.props.query; - return `${(opts && opts.doNyaize ? nyaize(query) : query)} [search]\n`; - } - }; - - return appendChildren(tokens, { doNyaize: (opts && opts.doNyaize) || false }).trim(); -} diff --git a/src/misc/extract-custom-emojis-from-mfm.ts b/src/misc/extract-custom-emojis-from-mfm.ts new file mode 100644 index 0000000000..f1477a79f0 --- /dev/null +++ b/src/misc/extract-custom-emojis-from-mfm.ts @@ -0,0 +1,18 @@ +import * as mfm from 'mfm-js'; +import { unique } from '@/prelude/array'; + +export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] { + const emojiNodes = [] as mfm.MfmEmojiCode[]; + + function scan(nodes: mfm.MfmNode[]) { + for (const node of nodes) { + if (node.type === 'emojiCode') emojiNodes.push(node); + else if (node.children) scan(node.children); + } + } + + scan(nodes); + + const emojis = emojiNodes.filter(x => x.props.name.length <= 100).map(x => x.props.name!); + return unique(emojis); +} diff --git a/src/misc/extract-emojis.ts b/src/misc/extract-emojis.ts deleted file mode 100644 index 2c57e9a8aa..0000000000 --- a/src/misc/extract-emojis.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { EmojiNode, MfmForest } from '../mfm/prelude'; -import { preorderF } from '../prelude/tree'; -import { unique } from '../prelude/array'; - -export default function(mfmForest: MfmForest): string[] { - const emojiNodes = preorderF(mfmForest).filter(x => x.type === 'emoji') as EmojiNode[]; - const emojis = emojiNodes.filter(x => x.props.name && x.props.name.length <= 100).map(x => x.props.name); - return unique(emojis); -} diff --git a/src/misc/extract-hashtags.ts b/src/misc/extract-hashtags.ts index 36b2296a76..9961755ccd 100644 --- a/src/misc/extract-hashtags.ts +++ b/src/misc/extract-hashtags.ts @@ -1,9 +1,18 @@ -import { HashtagNode, MfmForest } from '../mfm/prelude'; -import { preorderF } from '../prelude/tree'; -import { unique } from '../prelude/array'; +import * as mfm from 'mfm-js'; +import { unique } from '@/prelude/array'; + +export default function(nodes: mfm.MfmNode[]): string[] { + const hashtagNodes = [] as mfm.MfmHashtag[]; + + function scan(nodes: mfm.MfmNode[]) { + for (const node of nodes) { + if (node.type === 'hashtag') hashtagNodes.push(node); + else if (node.children) scan(node.children); + } + } + + scan(nodes); -export default function(mfmForest: MfmForest): string[] { - const hashtagNodes = preorderF(mfmForest).filter(x => x.type === 'hashtag') as HashtagNode[]; const hashtags = hashtagNodes.map(x => x.props.hashtag); return unique(hashtags); } diff --git a/src/misc/extract-mentions.ts b/src/misc/extract-mentions.ts index 72330d31e1..a9d4b378f3 100644 --- a/src/misc/extract-mentions.ts +++ b/src/misc/extract-mentions.ts @@ -1,10 +1,19 @@ // test is located in test/extract-mentions -import { MentionNode, MfmForest } from '../mfm/prelude'; -import { preorderF } from '../prelude/tree'; +import * as mfm from 'mfm-js'; -export default function(mfmForest: MfmForest): MentionNode['props'][] { +export default function(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { // TODO: 重複を削除 - const mentionNodes = preorderF(mfmForest).filter(x => x.type === 'mention') as MentionNode[]; + const mentionNodes = [] as mfm.MfmMention[]; + + function scan(nodes: mfm.MfmNode[]) { + for (const node of nodes) { + if (node.type === 'mention') mentionNodes.push(node); + else if (node.children) scan(node.children); + } + } + + scan(nodes); + return mentionNodes.map(x => x.props); } diff --git a/src/misc/extract-url-from-mfm.ts b/src/misc/extract-url-from-mfm.ts new file mode 100644 index 0000000000..aa7f5f2540 --- /dev/null +++ b/src/misc/extract-url-from-mfm.ts @@ -0,0 +1,34 @@ +import * as mfm from 'mfm-js'; +import { unique } from '@/prelude/array'; + +// unique without hash +// [ http://a/#1, http://a/#2, http://b/#3 ] => [ http://a/#1, http://b/#3 ] +const removeHash = (x: string) => x.replace(/#[^#]*$/, ''); + +export function extractUrlFromMfm(nodes: mfm.MfmNode[], respectSilentFlag = true): string[] { + const urlNodes = [] as (mfm.MfmUrl | mfm.MfmLink)[]; + + function scan(nodes: mfm.MfmNode[]) { + for (const node of nodes) { + if (node.type === 'url') { + urlNodes.push(node); + } else if (node.type === 'link') { + if (!respectSilentFlag || !node.props.silent) { + urlNodes.push(node); + } + } else if (node.children) { + scan(node.children); + } + } + } + + scan(nodes); + + const urls = unique(urlNodes.map(x => x.props.url)); + + return urls.reduce((array, url) => { + const removed = removeHash(url); + if (!array.map(x => removeHash(x)).includes(removed)) array.push(url); + return array; + }, [] as string[]); +} diff --git a/src/models/repositories/note.ts b/src/models/repositories/note.ts index 3642a03c2c..cdf4841918 100644 --- a/src/models/repositories/note.ts +++ b/src/models/repositories/note.ts @@ -1,12 +1,12 @@ import { EntityRepository, Repository, In } from 'typeorm'; +import * as mfm from 'mfm-js'; import { Note } from '../entities/note'; import { User } from '../entities/user'; import { Users, PollVotes, DriveFiles, NoteReactions, Followings, Polls, Channels } from '..'; import { SchemaType } from '@/misc/schema'; +import { nyaize } from '@/misc/nyaize'; import { awaitAll } from '../../prelude/await-all'; import { convertLegacyReaction, convertLegacyReactions, decodeReaction } from '@/misc/reaction-lib'; -import { toString } from '../../mfm/to-string'; -import { parse } from '../../mfm/parse'; import { NoteReaction } from '../entities/note-reaction'; import { aggregateNoteEmojis, populateEmojis, prefetchEmojis } from '@/misc/populate-emojis'; @@ -223,8 +223,13 @@ export class NoteRepository extends Repository<Note> { }); if (packed.user.isCat && packed.text) { - const tokens = packed.text ? parse(packed.text) : []; - packed.text = toString(tokens, { doNyaize: true }); + const tokens = packed.text ? mfm.parse(packed.text) : []; + mfm.inspect(tokens, node => { + if (node.type === 'text') { + node.props.text = nyaize(node.props.text); + } + }); + packed.text = mfm.toString(tokens); } if (!opts.skipHide) { diff --git a/src/prelude/tree.ts b/src/prelude/tree.ts deleted file mode 100644 index 519234a0b0..0000000000 --- a/src/prelude/tree.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { concat, sum } from './array'; - -export type Tree<T> = { - node: T, - children: Forest<T>; -}; - -export type Forest<T> = Tree<T>[]; - -export function createLeaf<T>(node: T): Tree<T> { - return { node, children: [] }; -} - -export function createTree<T>(node: T, children: Forest<T>): Tree<T> { - return { node, children }; -} - -export function hasChildren<T>(t: Tree<T>): boolean { - return t.children.length !== 0; -} - -export function preorder<T>(t: Tree<T>): T[] { - return [t.node, ...preorderF(t.children)]; -} - -export function preorderF<T>(ts: Forest<T>): T[] { - return concat(ts.map(preorder)); -} - -export function countNodes<T>(t: Tree<T>): number { - return preorder(t).length; -} - -export function countNodesF<T>(ts: Forest<T>): number { - return sum(ts.map(countNodes)); -} diff --git a/src/remote/activitypub/misc/get-note-html.ts b/src/remote/activitypub/misc/get-note-html.ts index 6990a4ae5e..683860d9cc 100644 --- a/src/remote/activitypub/misc/get-note-html.ts +++ b/src/remote/activitypub/misc/get-note-html.ts @@ -1,9 +1,9 @@ +import * as mfm from 'mfm-js'; import { Note } from '../../../models/entities/note'; import { toHtml } from '../../../mfm/to-html'; -import { parse } from '../../../mfm/parse'; export default function(note: Note) { - let html = toHtml(parse(note.text), JSON.parse(note.mentionedRemoteUsers)); + let html = note.text ? toHtml(mfm.parse(note.text), JSON.parse(note.mentionedRemoteUsers)) : null; if (html == null) html = '<p>.</p>'; return html; diff --git a/src/remote/activitypub/renderer/person.ts b/src/remote/activitypub/renderer/person.ts index e4e8f24f10..91b91bff92 100644 --- a/src/remote/activitypub/renderer/person.ts +++ b/src/remote/activitypub/renderer/person.ts @@ -1,10 +1,10 @@ import { URL } from 'url'; +import * as mfm from 'mfm-js'; import renderImage from './image'; import renderKey from './key'; import config from '@/config'; import { ILocalUser } from '../../../models/entities/user'; import { toHtml } from '../../../mfm/to-html'; -import { parse } from '../../../mfm/parse'; import { getEmojis } from './note'; import renderEmoji from './emoji'; import { IIdentifier } from '../models/identifier'; @@ -66,7 +66,7 @@ export async function renderPerson(user: ILocalUser) { url: `${config.url}/@${user.username}`, preferredUsername: user.username, name: user.name, - summary: toHtml(parse(profile.description)), + summary: profile.description ? toHtml(mfm.parse(profile.description)) : null, icon: avatar ? renderImage(avatar) : null, image: banner ? renderImage(banner) : null, tag, diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts index 0554fe76fb..2c20da41c5 100644 --- a/src/server/api/endpoints/i/update.ts +++ b/src/server/api/endpoints/i/update.ts @@ -1,11 +1,11 @@ import $ from 'cafy'; +import * as mfm from 'mfm-js'; import { ID } from '@/misc/cafy-id'; import { publishMainStream, publishUserEvent } from '../../../../services/stream'; import acceptAllFollowRequests from '../../../../services/following/requests/accept-all'; import { publishToFollowers } from '../../../../services/i/update'; import define from '../../define'; -import { parse, parsePlain } from '../../../../mfm/parse'; -import extractEmojis from '@/misc/extract-emojis'; +import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm'; import extractHashtags from '@/misc/extract-hashtags'; import * as langmap from 'langmap'; import { updateUsertags } from '../../../../services/update-hashtag'; @@ -291,13 +291,13 @@ export default define(meta, async (ps, _user, token) => { const newDescription = profileUpdates.description === undefined ? profile.description : profileUpdates.description; if (newName != null) { - const tokens = parsePlain(newName); - emojis = emojis.concat(extractEmojis(tokens!)); + const tokens = mfm.parsePlain(newName); + emojis = emojis.concat(extractCustomEmojisFromMfm(tokens!)); } if (newDescription != null) { - const tokens = parse(newDescription); - emojis = emojis.concat(extractEmojis(tokens!)); + const tokens = mfm.parse(newDescription); + emojis = emojis.concat(extractCustomEmojisFromMfm(tokens!)); tags = extractHashtags(tokens!).map(tag => normalizeForSearch(tag)).splice(0, 32); } diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 64d5513ecc..125285f34a 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -1,3 +1,4 @@ +import * as mfm from 'mfm-js'; import es from '../../db/elasticsearch'; import { publishMainStream, publishNotesStream } from '../stream'; import DeliverManager from '../../remote/activitypub/deliver-manager'; @@ -5,7 +6,6 @@ import renderNote from '../../remote/activitypub/renderer/note'; import renderCreate from '../../remote/activitypub/renderer/create'; import renderAnnounce from '../../remote/activitypub/renderer/announce'; import { renderActivity } from '../../remote/activitypub/renderer'; -import { parse } from '../../mfm/parse'; import { resolveUser } from '../../remote/resolve-user'; import config from '@/config'; import { updateHashtags } from '../update-hashtag'; @@ -13,7 +13,7 @@ import { concat } from '../../prelude/array'; import insertNoteUnread from './unread'; import { registerOrFetchInstanceDoc } from '../register-or-fetch-instance-doc'; import extractMentions from '@/misc/extract-mentions'; -import extractEmojis from '@/misc/extract-emojis'; +import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm'; import extractHashtags from '@/misc/extract-hashtags'; import { Note, IMentionedRemoteUsers } from '../../models/entities/note'; import { Mutings, Users, NoteWatchings, Notes, Instances, UserProfiles, Antennas, Followings, MutedNotes, Channels, ChannelFollowings } from '../../models'; @@ -182,17 +182,17 @@ export default async (user: { id: User['id']; username: User['username']; host: // Parse MFM if needed if (!tags || !emojis || !mentionedUsers) { - const tokens = data.text ? parse(data.text)! : []; - const cwTokens = data.cw ? parse(data.cw)! : []; + const tokens = data.text ? mfm.parse(data.text)! : []; + const cwTokens = data.cw ? mfm.parse(data.cw)! : []; const choiceTokens = data.poll && data.poll.choices - ? concat(data.poll.choices.map(choice => parse(choice)!)) + ? concat(data.poll.choices.map(choice => mfm.parse(choice)!)) : []; const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens); tags = data.apHashtags || extractHashtags(combinedTokens); - emojis = data.apEmojis || extractEmojis(combinedTokens); + emojis = data.apEmojis || extractCustomEmojisFromMfm(combinedTokens); mentionedUsers = data.apMentions || await extractMentionedUsers(user, combinedTokens); } @@ -604,7 +604,7 @@ function incNotesCountOfUser(user: { id: User['id']; }) { .execute(); } -async function extractMentionedUsers(user: { host: User['host']; }, tokens: ReturnType<typeof parse>): Promise<User[]> { +async function extractMentionedUsers(user: { host: User['host']; }, tokens: mfm.MfmNode[]): Promise<User[]> { if (tokens == null) return []; const mentions = extractMentions(tokens); diff --git a/test/extract-mentions.ts b/test/extract-mentions.ts index 00f736ae1c..68b59f7f55 100644 --- a/test/extract-mentions.ts +++ b/test/extract-mentions.ts @@ -1,7 +1,7 @@ import * as assert from 'assert'; import extractMentions from '../src/misc/extract-mentions'; -import { parse } from '../src/mfm/parse'; +import { parse } from 'mfm-js'; describe('Extract mentions', () => { it('simple', () => { @@ -10,17 +10,14 @@ describe('Extract mentions', () => { assert.deepStrictEqual(mentions, [{ username: 'foo', acct: '@foo', - canonical: '@foo', host: null }, { username: 'bar', acct: '@bar', - canonical: '@bar', host: null }, { username: 'baz', acct: '@baz', - canonical: '@baz', host: null }]); }); @@ -31,17 +28,14 @@ describe('Extract mentions', () => { assert.deepStrictEqual(mentions, [{ username: 'foo', acct: '@foo', - canonical: '@foo', host: null }, { username: 'bar', acct: '@bar', - canonical: '@bar', host: null }, { username: 'baz', acct: '@baz', - canonical: '@baz', host: null }]); }); diff --git a/test/mfm.ts b/test/mfm.ts index 0a120f96e1..3f997878b5 100644 --- a/test/mfm.ts +++ b/test/mfm.ts @@ -9,1152 +9,22 @@ */ import * as assert from 'assert'; +import * as mfm from 'mfm-js'; -import { parse, parsePlain } from '../src/mfm/parse'; import { toHtml } from '../src/mfm/to-html'; import { fromHtml } from '../src/mfm/from-html'; -import { toString } from '../src/mfm/to-string'; -import { createTree as tree, createLeaf as leaf, MfmTree } from '../src/mfm/prelude'; -import { removeOrphanedBrackets } from '../src/mfm/language'; -function text(text: string): MfmTree { - return leaf('text', { text }); -} - -describe('createLeaf', () => { - it('creates leaf', () => { - assert.deepStrictEqual(leaf('text', { text: 'abc' }), { - node: { - type: 'text', - props: { - text: 'abc' - } - }, - children: [], - }); - }); -}); - -describe('createTree', () => { - it('creates tree', () => { - const t = tree('tree', [ - leaf('left', { a: 2 }), - leaf('right', { b: 'hi' }) - ], { - c: 4 - }); - assert.deepStrictEqual(t, { - node: { - type: 'tree', - props: { - c: 4 - } - }, - children: [ - leaf('left', { a: 2 }), - leaf('right', { b: 'hi' }) - ], - }); - }); -}); - -describe('removeOrphanedBrackets', () => { - it('single (contained)', () => { - const input = '(foo)'; - const expected = '(foo)'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); +describe('toHtml', () => { + it('br', () => { + const input = 'foo\nbar\nbaz'; + const output = '<p><span>foo<br>bar<br>baz</span></p>'; + assert.equal(toHtml(mfm.parse(input)), output); }); - it('single (head)', () => { - const input = '(foo)bar'; - const expected = '(foo)bar'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('single (tail)', () => { - const input = 'foo(bar)'; - const expected = 'foo(bar)'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('a', () => { - const input = '(foo'; - const expected = ''; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('b', () => { - const input = ')foo'; - const expected = ''; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('nested', () => { - const input = 'foo(「(bar)」)'; - const expected = 'foo(「(bar)」)'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('no brackets', () => { - const input = 'foo'; - const expected = 'foo'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('with foreign bracket (single)', () => { - const input = 'foo(bar))'; - const expected = 'foo(bar)'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('with foreign bracket (open)', () => { - const input = 'foo(bar'; - const expected = 'foo'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('with foreign bracket (close)', () => { - const input = 'foo)bar'; - const expected = 'foo'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('with foreign bracket (close and open)', () => { - const input = 'foo)(bar'; - const expected = 'foo'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('various bracket type', () => { - const input = 'foo「(bar)」('; - const expected = 'foo「(bar)」'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); - - it('intersected', () => { - const input = 'foo(「)」'; - const expected = 'foo(「)」'; - const actual = removeOrphanedBrackets(input); - assert.deepStrictEqual(actual, expected); - }); -}); - -describe('MFM', () => { - it('can be analyzed', () => { - const tokens = parse('@himawari @hima_sub@namori.net お腹ペコい :cat: #yryr'); - assert.deepStrictEqual(tokens, [ - leaf('mention', { - acct: '@himawari', - canonical: '@himawari', - username: 'himawari', - host: null - }), - text(' '), - leaf('mention', { - acct: '@hima_sub@namori.net', - canonical: '@hima_sub@namori.net', - username: 'hima_sub', - host: 'namori.net' - }), - text(' お腹ペコい '), - leaf('emoji', { name: 'cat' }), - text(' '), - leaf('hashtag', { hashtag: 'yryr' }), - ]); - }); - - describe('elements', () => { - describe('bold', () => { - it('simple', () => { - const tokens = parse('**foo**'); - assert.deepStrictEqual(tokens, [ - tree('bold', [ - text('foo') - ], {}), - ]); - }); - - it('with other texts', () => { - const tokens = parse('bar**foo**bar'); - assert.deepStrictEqual(tokens, [ - text('bar'), - tree('bold', [ - text('foo') - ], {}), - text('bar'), - ]); - }); - - it('with underscores', () => { - const tokens = parse('__foo__'); - assert.deepStrictEqual(tokens, [ - tree('bold', [ - text('foo') - ], {}), - ]); - }); - - it('with underscores (ensure it allows alphabet only)', () => { - const tokens = parse('(=^・__________・^=)'); - assert.deepStrictEqual(tokens, [ - text('(=^・__________・^=)') - ]); - }); - - it('mixed syntax', () => { - const tokens = parse('**foo__'); - assert.deepStrictEqual(tokens, [ - text('**foo__'), - ]); - }); - - it('mixed syntax', () => { - const tokens = parse('__foo**'); - assert.deepStrictEqual(tokens, [ - text('__foo**'), - ]); - }); - }); - - it('small', () => { - const tokens = parse('<small>smaller</small>'); - assert.deepStrictEqual(tokens, [ - tree('small', [ - text('smaller') - ], {}), - ]); - }); - - describe('mention', () => { - it('local', () => { - const tokens = parse('@himawari foo'); - assert.deepStrictEqual(tokens, [ - leaf('mention', { - acct: '@himawari', - canonical: '@himawari', - username: 'himawari', - host: null - }), - text(' foo') - ]); - }); - - it('remote', () => { - const tokens = parse('@hima_sub@namori.net foo'); - assert.deepStrictEqual(tokens, [ - leaf('mention', { - acct: '@hima_sub@namori.net', - canonical: '@hima_sub@namori.net', - username: 'hima_sub', - host: 'namori.net' - }), - text(' foo') - ]); - }); - - it('remote punycode', () => { - const tokens = parse('@hima_sub@xn--q9j5bya.xn--zckzah foo'); - assert.deepStrictEqual(tokens, [ - leaf('mention', { - acct: '@hima_sub@xn--q9j5bya.xn--zckzah', - canonical: '@hima_sub@なもり.テスト', - username: 'hima_sub', - host: 'xn--q9j5bya.xn--zckzah' - }), - text(' foo') - ]); - }); - - it('ignore', () => { - const tokens = parse('idolm@ster'); - assert.deepStrictEqual(tokens, [ - text('idolm@ster') - ]); - - const tokens2 = parse('@a\n@b\n@c'); - assert.deepStrictEqual(tokens2, [ - leaf('mention', { - acct: '@a', - canonical: '@a', - username: 'a', - host: null - }), - text('\n'), - leaf('mention', { - acct: '@b', - canonical: '@b', - username: 'b', - host: null - }), - text('\n'), - leaf('mention', { - acct: '@c', - canonical: '@c', - username: 'c', - host: null - }) - ]); - - const tokens3 = parse('**x**@a'); - assert.deepStrictEqual(tokens3, [ - tree('bold', [ - text('x') - ], {}), - leaf('mention', { - acct: '@a', - canonical: '@a', - username: 'a', - host: null - }) - ]); - - const tokens4 = parse('@\n@v\n@veryverylongusername'); - assert.deepStrictEqual(tokens4, [ - text('@\n'), - leaf('mention', { - acct: '@v', - canonical: '@v', - username: 'v', - host: null - }), - text('\n'), - leaf('mention', { - acct: '@veryverylongusername', - canonical: '@veryverylongusername', - username: 'veryverylongusername', - host: null - }), - ]); - }); - }); - - describe('hashtag', () => { - it('simple', () => { - const tokens = parse('#alice'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'alice' }) - ]); - }); - - it('after line break', () => { - const tokens = parse('foo\n#alice'); - assert.deepStrictEqual(tokens, [ - text('foo\n'), - leaf('hashtag', { hashtag: 'alice' }) - ]); - }); - - it('with text', () => { - const tokens = parse('Strawberry Pasta #alice'); - assert.deepStrictEqual(tokens, [ - text('Strawberry Pasta '), - leaf('hashtag', { hashtag: 'alice' }) - ]); - }); - - it('with text (zenkaku)', () => { - const tokens = parse('こんにちは#世界'); - assert.deepStrictEqual(tokens, [ - text('こんにちは'), - leaf('hashtag', { hashtag: '世界' }) - ]); - }); - - it('ignore comma and period', () => { - const tokens = parse('Foo #bar, baz #piyo.'); - assert.deepStrictEqual(tokens, [ - text('Foo '), - leaf('hashtag', { hashtag: 'bar' }), - text(', baz '), - leaf('hashtag', { hashtag: 'piyo' }), - text('.'), - ]); - }); - - it('ignore exclamation mark', () => { - const tokens = parse('#Foo!'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'Foo' }), - text('!'), - ]); - }); - - it('ignore colon', () => { - const tokens = parse('#Foo:'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'Foo' }), - text(':'), - ]); - }); - - it('ignore single quote', () => { - const tokens = parse('#foo\''); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'foo' }), - text('\''), - ]); - }); - - it('ignore double quote', () => { - const tokens = parse('#foo"'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'foo' }), - text('"'), - ]); - }); - - it('ignore square brackets', () => { - const tokens = parse('#foo]'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'foo' }), - text(']'), - ]); - }); - - it('ignore 】', () => { - const tokens = parse('#foo】'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'foo' }), - text('】'), - ]); - }); - - it('allow including number', () => { - const tokens = parse('#foo123'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'foo123' }), - ]); - }); - - it('with brackets', () => { - const tokens1 = parse('(#foo)'); - assert.deepStrictEqual(tokens1, [ - text('('), - leaf('hashtag', { hashtag: 'foo' }), - text(')'), - ]); - - const tokens2 = parse('「#foo」'); - assert.deepStrictEqual(tokens2, [ - text('「'), - leaf('hashtag', { hashtag: 'foo' }), - text('」'), - ]); - }); - - it('with mixed brackets', () => { - const tokens = parse('「#foo(bar)」'); - assert.deepStrictEqual(tokens, [ - text('「'), - leaf('hashtag', { hashtag: 'foo(bar)' }), - text('」'), - ]); - }); - - it('with brackets (space before)', () => { - const tokens1 = parse('(bar #foo)'); - assert.deepStrictEqual(tokens1, [ - text('(bar '), - leaf('hashtag', { hashtag: 'foo' }), - text(')'), - ]); - - const tokens2 = parse('「bar #foo」'); - assert.deepStrictEqual(tokens2, [ - text('「bar '), - leaf('hashtag', { hashtag: 'foo' }), - text('」'), - ]); - }); - - it('disallow number only', () => { - const tokens = parse('#123'); - assert.deepStrictEqual(tokens, [ - text('#123'), - ]); - }); - - it('disallow number only (with brackets)', () => { - const tokens = parse('(#123)'); - assert.deepStrictEqual(tokens, [ - text('(#123)'), - ]); - }); - - it('ignore slash', () => { - const tokens = parse('#foo/bar'); - assert.deepStrictEqual(tokens, [ - leaf('hashtag', { hashtag: 'foo' }), - text('/bar'), - ]); - }); - - it('ignore Keycap Number Sign (U+0023 + U+20E3)', () => { - const tokens = parse('#⃣'); - assert.deepStrictEqual(tokens, [ - leaf('emoji', { emoji: '#⃣' }) - ]); - }); - - it('ignore Keycap Number Sign (U+0023 + U+FE0F + U+20E3)', () => { - const tokens = parse('#️⃣'); - assert.deepStrictEqual(tokens, [ - leaf('emoji', { emoji: '#️⃣' }) - ]); - }); - }); - - describe('quote', () => { - it('basic', () => { - const tokens1 = parse('> foo'); - assert.deepStrictEqual(tokens1, [ - tree('quote', [ - text('foo') - ], {}) - ]); - - const tokens2 = parse('>foo'); - assert.deepStrictEqual(tokens2, [ - tree('quote', [ - text('foo') - ], {}) - ]); - }); - - it('series', () => { - const tokens = parse('> foo\n\n> bar'); - assert.deepStrictEqual(tokens, [ - tree('quote', [ - text('foo') - ], {}), - text('\n'), - tree('quote', [ - text('bar') - ], {}), - ]); - }); - - it('trailing line break', () => { - const tokens1 = parse('> foo\n'); - assert.deepStrictEqual(tokens1, [ - tree('quote', [ - text('foo') - ], {}), - ]); - - const tokens2 = parse('> foo\n\n'); - assert.deepStrictEqual(tokens2, [ - tree('quote', [ - text('foo') - ], {}), - text('\n') - ]); - }); - - it('multiline', () => { - const tokens1 = parse('>foo\n>bar'); - assert.deepStrictEqual(tokens1, [ - tree('quote', [ - text('foo\nbar') - ], {}) - ]); - - const tokens2 = parse('> foo\n> bar'); - assert.deepStrictEqual(tokens2, [ - tree('quote', [ - text('foo\nbar') - ], {}) - ]); - }); - - it('multiline with trailing line break', () => { - const tokens1 = parse('> foo\n> bar\n'); - assert.deepStrictEqual(tokens1, [ - tree('quote', [ - text('foo\nbar') - ], {}), - ]); - - const tokens2 = parse('> foo\n> bar\n\n'); - assert.deepStrictEqual(tokens2, [ - tree('quote', [ - text('foo\nbar') - ], {}), - text('\n') - ]); - }); - - it('with before and after texts', () => { - const tokens = parse('before\n> foo\nafter'); - assert.deepStrictEqual(tokens, [ - text('before\n'), - tree('quote', [ - text('foo') - ], {}), - text('after'), - ]); - }); - - it('multiple quotes', () => { - const tokens = parse('> foo\nbar\n\n> foo\nbar\n\n> foo\nbar'); - assert.deepStrictEqual(tokens, [ - tree('quote', [ - text('foo') - ], {}), - text('bar\n\n'), - tree('quote', [ - text('foo') - ], {}), - text('bar\n\n'), - tree('quote', [ - text('foo') - ], {}), - text('bar'), - ]); - }); - - it('require line break before ">"', () => { - const tokens = parse('foo>bar'); - assert.deepStrictEqual(tokens, [ - text('foo>bar'), - ]); - }); - - it('nested', () => { - const tokens = parse('>> foo\n> bar'); - assert.deepStrictEqual(tokens, [ - tree('quote', [ - tree('quote', [ - text('foo') - ], {}), - text('bar') - ], {}) - ]); - }); - - it('trim line breaks', () => { - const tokens = parse('foo\n\n>a\n>>b\n>>\n>>>\n>>>c\n>>>\n>d\n\n'); - assert.deepStrictEqual(tokens, [ - text('foo\n\n'), - tree('quote', [ - text('a\n'), - tree('quote', [ - text('b\n\n'), - tree('quote', [ - text('\nc\n') - ], {}) - ], {}), - text('d') - ], {}), - text('\n'), - ]); - }); - }); - - describe('url', () => { - it('simple', () => { - const tokens = parse('https://example.com'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://example.com' }) - ]); - }); - - it('ignore trailing period', () => { - const tokens = parse('https://example.com.'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://example.com' }), - text('.') - ]); - }); - - it('ignore trailing periods', () => { - const tokens = parse('https://example.com...'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://example.com' }), - text('...') - ]); - }); - - it('with comma', () => { - const tokens = parse('https://example.com/foo?bar=a,b'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://example.com/foo?bar=a,b' }) - ]); - }); - - it('ignore trailing comma', () => { - const tokens = parse('https://example.com/foo, bar'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://example.com/foo' }), - text(', bar') - ]); - }); - - it('with brackets', () => { - const tokens = parse('https://example.com/foo(bar)'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://example.com/foo(bar)' }) - ]); - }); - - it('ignore parent brackets', () => { - const tokens = parse('(https://example.com/foo)'); - assert.deepStrictEqual(tokens, [ - text('('), - leaf('url', { url: 'https://example.com/foo' }), - text(')') - ]); - }); - - it('ignore parent []', () => { - const tokens = parse('foo [https://example.com/foo] bar'); - assert.deepStrictEqual(tokens, [ - text('foo ['), - leaf('url', { url: 'https://example.com/foo' }), - text('] bar') - ]); - }); - - it('ignore parent brackets 2', () => { - const tokens = parse('(foo https://example.com/foo)'); - assert.deepStrictEqual(tokens, [ - text('(foo '), - leaf('url', { url: 'https://example.com/foo' }), - text(')') - ]); - }); - - it('ignore parent brackets with internal brackets', () => { - const tokens = parse('(https://example.com/foo(bar))'); - assert.deepStrictEqual(tokens, [ - text('('), - leaf('url', { url: 'https://example.com/foo(bar)' }), - text(')') - ]); - }); - - it('ignore non-ascii characters contained url without angle brackets', () => { - const tokens = parse('https://大石泉すき.example.com'); - assert.deepStrictEqual(tokens, [ - text('https://大石泉すき.example.com') - ]); - }); - - it('match non-ascii characters contained url with angle brackets', () => { - const tokens = parse('<https://大石泉すき.example.com>'); - assert.deepStrictEqual(tokens, [ - leaf('url', { url: 'https://大石泉すき.example.com' }) - ]); - }); - }); - - describe('link', () => { - it('simple', () => { - const tokens = parse('[foo](https://example.com)'); - assert.deepStrictEqual(tokens, [ - tree('link', [ - text('foo') - ], { url: 'https://example.com', silent: false }) - ]); - }); - - it('simple (with silent flag)', () => { - const tokens = parse('?[foo](https://example.com)'); - assert.deepStrictEqual(tokens, [ - tree('link', [ - text('foo') - ], { url: 'https://example.com', silent: true }) - ]); - }); - - it('in text', () => { - const tokens = parse('before[foo](https://example.com)after'); - assert.deepStrictEqual(tokens, [ - text('before'), - tree('link', [ - text('foo') - ], { url: 'https://example.com', silent: false }), - text('after'), - ]); - }); - - it('with brackets', () => { - const tokens = parse('[foo](https://example.com/foo(bar))'); - assert.deepStrictEqual(tokens, [ - tree('link', [ - text('foo') - ], { url: 'https://example.com/foo(bar)', silent: false }) - ]); - }); - - it('with parent brackets', () => { - const tokens = parse('([foo](https://example.com/foo(bar)))'); - assert.deepStrictEqual(tokens, [ - text('('), - tree('link', [ - text('foo') - ], { url: 'https://example.com/foo(bar)', silent: false }), - text(')') - ]); - }); - }); - - it('emoji', () => { - const tokens1 = parse(':cat:'); - assert.deepStrictEqual(tokens1, [ - leaf('emoji', { name: 'cat' }) - ]); - - const tokens2 = parse(':cat::cat::cat:'); - assert.deepStrictEqual(tokens2, [ - leaf('emoji', { name: 'cat' }), - leaf('emoji', { name: 'cat' }), - leaf('emoji', { name: 'cat' }) - ]); - - const tokens3 = parse('🍎'); - assert.deepStrictEqual(tokens3, [ - leaf('emoji', { emoji: '🍎' }) - ]); - }); - - describe('block code', () => { - it('simple', () => { - const tokens = parse('```\nvar x = "Strawberry Pasta";\n```'); - assert.deepStrictEqual(tokens, [ - leaf('blockCode', { code: 'var x = "Strawberry Pasta";', lang: null }) - ]); - }); - - it('can specify language', () => { - const tokens = parse('``` json\n{ "x": 42 }\n```'); - assert.deepStrictEqual(tokens, [ - leaf('blockCode', { code: '{ "x": 42 }', lang: 'json' }) - ]); - }); - - it('require line break before "```"', () => { - const tokens = parse('before```\nfoo\n```'); - assert.deepStrictEqual(tokens, [ - text('before'), - leaf('inlineCode', { code: '`' }), - text('\nfoo\n'), - leaf('inlineCode', { code: '`' }) - ]); - }); - - it('series', () => { - const tokens = parse('```\nfoo\n```\n```\nbar\n```\n```\nbaz\n```'); - assert.deepStrictEqual(tokens, [ - leaf('blockCode', { code: 'foo', lang: null }), - leaf('blockCode', { code: 'bar', lang: null }), - leaf('blockCode', { code: 'baz', lang: null }), - ]); - }); - - it('ignore internal marker', () => { - const tokens = parse('```\naaa```bbb\n```'); - assert.deepStrictEqual(tokens, [ - leaf('blockCode', { code: 'aaa```bbb', lang: null }) - ]); - }); - - it('trim after line break', () => { - const tokens = parse('```\nfoo\n```\nbar'); - assert.deepStrictEqual(tokens, [ - leaf('blockCode', { code: 'foo', lang: null }), - text('bar') - ]); - }); - }); - - describe('inline code', () => { - it('simple', () => { - const tokens = parse('`var x = "Strawberry Pasta";`'); - assert.deepStrictEqual(tokens, [ - leaf('inlineCode', { code: 'var x = "Strawberry Pasta";' }) - ]); - }); - - it('disallow line break', () => { - const tokens = parse('`foo\nbar`'); - assert.deepStrictEqual(tokens, [ - text('`foo\nbar`') - ]); - }); - - it('disallow ´', () => { - const tokens = parse('`foo´bar`'); - assert.deepStrictEqual(tokens, [ - text('`foo´bar`') - ]); - }); - }); - - it('mathInline', () => { - const fomula = 'x = {-b \\pm \\sqrt{b^2-4ac} \\over 2a}'; - const content = `\\(${fomula}\\)`; - const tokens = parse(content); - assert.deepStrictEqual(tokens, [ - leaf('mathInline', { formula: fomula }) - ]); - }); - - describe('mathBlock', () => { - it('simple', () => { - const fomula = 'x = {-b \\pm \\sqrt{b^2-4ac} \\over 2a}'; - const content = `\\[\n${fomula}\n\\]`; - const tokens = parse(content); - assert.deepStrictEqual(tokens, [ - leaf('mathBlock', { formula: fomula }) - ]); - }); - }); - - it('search', () => { - const tokens1 = parse('a b c 検索'); - assert.deepStrictEqual(tokens1, [ - leaf('search', { content: 'a b c 検索', query: 'a b c' }) - ]); - - const tokens2 = parse('a b c Search'); - assert.deepStrictEqual(tokens2, [ - leaf('search', { content: 'a b c Search', query: 'a b c' }) - ]); - - const tokens3 = parse('a b c search'); - assert.deepStrictEqual(tokens3, [ - leaf('search', { content: 'a b c search', query: 'a b c' }) - ]); - - const tokens4 = parse('a b c SEARCH'); - assert.deepStrictEqual(tokens4, [ - leaf('search', { content: 'a b c SEARCH', query: 'a b c' }) - ]); - }); - - describe('center', () => { - it('simple', () => { - const tokens = parse('<center>foo</center>'); - assert.deepStrictEqual(tokens, [ - tree('center', [ - text('foo') - ], {}), - ]); - }); - }); - - describe('strike', () => { - it('simple', () => { - const tokens = parse('~~foo~~'); - assert.deepStrictEqual(tokens, [ - tree('strike', [ - text('foo') - ], {}), - ]); - }); - - // https://misskey.io/notes/7u1kv5dmia - it('ignore internal tilde', () => { - const tokens = parse('~~~~~'); - assert.deepStrictEqual(tokens, [ - text('~~~~~') - ]); - }); - }); - - describe('italic', () => { - it('<i>', () => { - const tokens = parse('<i>foo</i>'); - assert.deepStrictEqual(tokens, [ - tree('italic', [ - text('foo') - ], {}), - ]); - }); - - it('underscore', () => { - const tokens = parse('_foo_'); - assert.deepStrictEqual(tokens, [ - tree('italic', [ - text('foo') - ], {}), - ]); - }); - - it('simple with asterix', () => { - const tokens = parse('*foo*'); - assert.deepStrictEqual(tokens, [ - tree('italic', [ - text('foo') - ], {}), - ]); - }); - - it('exlude emotes', () => { - const tokens = parse('*.*'); - assert.deepStrictEqual(tokens, [ - text('*.*'), - ]); - }); - - it('mixed', () => { - const tokens = parse('_foo*'); - assert.deepStrictEqual(tokens, [ - text('_foo*'), - ]); - }); - - it('mixed', () => { - const tokens = parse('*foo_'); - assert.deepStrictEqual(tokens, [ - text('*foo_'), - ]); - }); - - it('ignore snake_case string', () => { - const tokens = parse('foo_bar_baz'); - assert.deepStrictEqual(tokens, [ - text('foo_bar_baz'), - ]); - }); - - it('require spaces', () => { - const tokens = parse('4日目_L38b a_b'); - assert.deepStrictEqual(tokens, [ - text('4日目_L38b a_b'), - ]); - }); - - it('newline sandwich', () => { - const tokens = parse('foo\n_bar_\nbaz'); - assert.deepStrictEqual(tokens, [ - text('foo\n'), - tree('italic', [ - text('bar') - ], {}), - text('\nbaz'), - ]); - }); - }); - }); - - describe('plainText', () => { - it('text', () => { - const tokens = parsePlain('foo'); - assert.deepStrictEqual(tokens, [ - text('foo'), - ]); - }); - - it('emoji', () => { - const tokens = parsePlain(':foo:'); - assert.deepStrictEqual(tokens, [ - leaf('emoji', { name: 'foo' }) - ]); - }); - - it('emoji in text', () => { - const tokens = parsePlain('foo:bar:baz'); - assert.deepStrictEqual(tokens, [ - text('foo'), - leaf('emoji', { name: 'bar' }), - text('baz'), - ]); - }); - - it('disallow other syntax', () => { - const tokens = parsePlain('foo **bar** baz'); - assert.deepStrictEqual(tokens, [ - text('foo **bar** baz'), - ]); - }); - }); - - describe('toHtml', () => { - it('br', () => { - const input = 'foo\nbar\nbaz'; - const output = '<p><span>foo<br>bar<br>baz</span></p>'; - assert.equal(toHtml(parse(input)), output); - }); - - it('br alt', () => { - const input = 'foo\r\nbar\rbaz'; - const output = '<p><span>foo<br>bar<br>baz</span></p>'; - assert.equal(toHtml(parse(input)), output); - }); - }); - - it('code block with quote', () => { - const tokens = parse('> foo\n```\nbar\n```'); - assert.deepStrictEqual(tokens, [ - tree('quote', [ - text('foo') - ], {}), - leaf('blockCode', { code: 'bar', lang: null }) - ]); - }); - - it('quote between two code blocks', () => { - const tokens = parse('```\nbefore\n```\n> foo\n```\nafter\n```'); - assert.deepStrictEqual(tokens, [ - leaf('blockCode', { code: 'before', lang: null }), - tree('quote', [ - text('foo') - ], {}), - leaf('blockCode', { code: 'after', lang: null }) - ]); - }); - - describe('toString', () => { - it('太字', () => { - assert.deepStrictEqual(toString(parse('**太字**')), '**太字**'); - }); - it('中央揃え', () => { - assert.deepStrictEqual(toString(parse('<center>中央揃え</center>')), '<center>中央揃え</center>'); - }); - it('打ち消し線', () => { - assert.deepStrictEqual(toString(parse('~~打ち消し線~~')), '~~打ち消し線~~'); - }); - it('小さい字', () => { - assert.deepStrictEqual(toString(parse('<small>小さい字</small>')), '<small>小さい字</small>'); - }); - it('コードブロック', () => { - assert.deepStrictEqual(toString(parse('```\nコードブロック\n```')), '```\nコードブロック\n```'); - }); - it('インラインコード', () => { - assert.deepStrictEqual(toString(parse('`インラインコード`')), '`インラインコード`'); - }); - it('引用行', () => { - assert.deepStrictEqual(toString(parse('>引用行')), '>引用行'); - }); - it('検索', () => { - assert.deepStrictEqual(toString(parse('検索 [search]')), '検索 [search]'); - }); - it('リンク', () => { - assert.deepStrictEqual(toString(parse('[リンク](http://example.com)')), '[リンク](http://example.com)'); - }); - it('詳細なしリンク', () => { - assert.deepStrictEqual(toString(parse('?[詳細なしリンク](http://example.com)')), '?[詳細なしリンク](http://example.com)'); - }); - it('インライン数式', () => { - assert.deepStrictEqual(toString(parse('\\(インライン数式\\)')), '\\(インライン数式\\)'); - }); - it('ブロック数式', () => { - assert.deepStrictEqual(toString(parse('\\\[\nブロック数式\n\]\\')), '\\\[\nブロック数式\n\]\\'); - }); + it('br alt', () => { + const input = 'foo\r\nbar\rbaz'; + const output = '<p><span>foo<br>bar<br>baz</span></p>'; + assert.equal(toHtml(mfm.parse(input)), output); }); }); diff --git a/yarn.lock b/yarn.lock index c85ab3b9c6..02fcf960d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6608,6 +6608,13 @@ methods@^1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= +mfm-js@0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/mfm-js/-/mfm-js-0.12.0.tgz#47be2fdb18869b9e55576fffcc159d0417c670db" + integrity sha512-u0IyIMwzsGsOGmctRXcOdWYsh9LWHKHqX+XCBfPjORX+1DCBdonaO6pryOawns6z16Xvus2yZk0KMMqWt2TotQ== + dependencies: + twemoji-parser "13.0.x" + micromatch@^3.0.4, micromatch@^3.1.4: version "3.1.10" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23" @@ -7494,11 +7501,6 @@ parseurl@^1.3.2: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== -parsimmon@1.16.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/parsimmon/-/parsimmon-1.16.0.tgz#2834e3db645b6a855ab2ea14fbaad10d82867e0f" - integrity sha512-tekGDz2Lny27SQ/5DzJdIK0lqsWwZ667SCLFIDCxaZM7VNgQjyKLbaL7FYPKpbjdxNAXFV/mSxkq5D2fnkW4pA== - pascalcase@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14" @@ -10521,6 +10523,11 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0: resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= +twemoji-parser@13.0.x: + version "13.0.0" + resolved "https://registry.yarnpkg.com/twemoji-parser/-/twemoji-parser-13.0.0.tgz#bd9d1b98474f1651dc174696b45cabefdfa405af" + integrity sha512-zMaGdskpH8yKjT2RSE/HwE340R4Fm+fbie4AaqjDa4H/l07YUmAvxkSfNl6awVWNRRQ0zdzLQ8SAJZuY5MgstQ== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"