Implement remote account resolution

src/api/endpoints/drive/files/create.ts

@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import { validateFileName, pack } from '../../../models/drive-file';
-import create from '../../../common/add-file-to-drive';
+import create from '../../../common/drive/add-file';
 
 /**
  * Create a file

src/api/endpoints/drive/files/upload_from_url.ts

@@ -1,16 +1,9 @@
 /**
  * Module dependencies
  */
-import * as URL from 'url';
 import $ from 'cafy';
-import { validateFileName, pack } from '../../../models/drive-file';
-import create from '../../../common/add-file-to-drive';
-import * as debug from 'debug';
-import * as tmp from 'tmp';
-import * as fs from 'fs';
-import * as request from 'request';
-
-const log = debug('misskey:endpoint:upload_from_url');
+import { pack } from '../../../models/drive-file';
+import uploadFromUrl from '../../../common/drive/upload_from_url';
 
 /**
  * Create a file from a URL
@@ -25,42 +18,9 @@ module.exports = async (params, user): Promise<any> => {
 	const [url, urlErr] = $(params.url).string().$;
 	if (urlErr) throw 'invalid url param';
 
-	let name = URL.parse(url).pathname.split('/').pop();
-	if (!validateFileName(name)) {
-		name = null;
-	}
-
 	// Get 'folder_id' parameter
 	const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
 	if (folderIdErr) throw 'invalid folder_id param';
 
-	// Create temp file
-	const path = await new Promise((res: (string) => void, rej) => {
-		tmp.file((e, path) => {
-			if (e) return rej(e);
-			res(path);
-		});
-	});
-
-	// write content at URL to temp file
-	await new Promise((res, rej) => {
-		const writable = fs.createWriteStream(path);
-		request(url)
-			.on('error', rej)
-			.on('end', () => {
-				writable.close();
-				res(path);
-			})
-			.pipe(writable)
-			.on('error', rej);
-	});
-
-	const driveFile = await create(user, path, name, null, folderId);
-
-	// clean-up
-	fs.unlink(path, (e) => {
-		if (e) log(e.stack);
-	});
-
-	return pack(driveFile);
+	return pack(await uploadFromUrl(url, user, folderId));
 };

src/api/endpoints/posts/create.ts

@@ -5,7 +5,7 @@ import $ from 'cafy';
 import deepEqual = require('deep-equal');
 import parse from '../../common/text';
 import { default as Post, IPost, isValidText } from '../../models/post';
-import { default as User, IUser } from '../../models/user';
+import { default as User, ILocalAccount, IUser } from '../../models/user';
 import { default as Channel, IChannel } from '../../models/channel';
 import Following from '../../models/following';
 import Mute from '../../models/mute';
@@ -16,6 +16,8 @@ import { pack } from '../../models/post';
 import notify from '../../common/notify';
 import watch from '../../common/watch-post';
 import event, { pushSw, publishChannelStream } from '../../event';
+import getAcct from '../../../common/user/get-acct';
+import parseAcct from '../../../common/user/parse-acct';
 import config from '../../../conf';
 
 /**
@@ -390,7 +392,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 			});
 
 		// この投稿をWatchする
-		if (user.account.settings.auto_watch !== false) {
+		if ((user.account as ILocalAccount).settings.auto_watch !== false) {
 			watch(user._id, reply);
 		}
 
@@ -477,7 +479,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 		// Extract an '@' mentions
 		const atMentions = tokens
 			.filter(t => t.type == 'mention')
-			.map(m => m.username)
+			.map(getAcct)
 			// Drop dupulicates
 			.filter((v, i, s) => s.indexOf(v) == i);
 
@@ -486,9 +488,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 			// Fetch mentioned user
 			// SELECT _id
 			const mentionee = await User
-				.findOne({
-					username_lower: mention.toLowerCase()
-				}, { _id: true });
+				.findOne(parseAcct(mention), { _id: true });
 
 			// When mentioned user not found
 			if (mentionee == null) return;

src/api/endpoints/username/available.ts

@@ -19,6 +19,7 @@ module.exports = async (params) => new Promise(async (res, rej) => {
 	// Get exist
 	const exist = await User
 		.count({
+			host: null,
 			username_lower: username.toLowerCase()
 		}, {
 			limit: 1

src/api/endpoints/users/posts.ts

@@ -2,6 +2,7 @@
  * Module dependencies
  */
 import $ from 'cafy';
+import getHostLower from '../../common/get-host-lower';
 import Post, { pack } from '../../models/post';
 import User from '../../models/user';
 
@@ -22,7 +23,15 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 	if (usernameErr) return rej('invalid username param');
 
 	if (userId === undefined && username === undefined) {
-		return rej('user_id or username is required');
+		return rej('user_id or pair of username and host is required');
+	}
+
+	// Get 'host' parameter
+	const [host, hostErr] = $(params.host).optional.string().$;
+	if (hostErr) return rej('invalid host param');
+
+	if (userId === undefined && host === undefined) {
+		return rej('user_id or pair of username and host is required');
 	}
 
 	// Get 'include_replies' parameter
@@ -60,7 +69,7 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 
 	const q = userId !== undefined
 		? { _id: userId }
-		: { username_lower: username.toLowerCase() } ;
+		: { username_lower: username.toLowerCase(), host_lower: getHostLower(host) } ;
 
 	// Lookup user
 	const user = await User.findOne(q, {

src/api/endpoints/users/recommendation.ts

@@ -30,9 +30,15 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 			_id: {
 				$nin: followingIds
 			},
-			'account.last_used_at': {
-				$gte: new Date(Date.now() - ms('7days'))
-			}
+			$or: [
+				{
+					'account.last_used_at': {
+						$gte: new Date(Date.now() - ms('7days'))
+					}
+				}, {
+					host: { $not: null }
+				}
+			]
 		}, {
 			limit: limit,
 			skip: offset,

src/api/endpoints/users/show.ts

@@ -2,7 +2,49 @@
  * Module dependencies
  */
 import $ from 'cafy';
-import User, { pack } from '../../models/user';
+import { JSDOM } from 'jsdom';
+import { toUnicode, toASCII } from 'punycode';
+import uploadFromUrl from '../../common/drive/upload_from_url';
+import User, { pack, validateUsername, isValidName, isValidDescription } from '../../models/user';
+const request = require('request-promise-native');
+const WebFinger = require('webfinger.js');
+
+const webFinger = new WebFinger({});
+
+async function getCollectionCount(url) {
+	if (!url) {
+		return null;
+	}
+
+	try {
+		const collection = await request({ url, json: true });
+		return collection ? collection.totalItems : null;
+	} catch (exception) {
+		return null;
+	}
+}
+
+function findUser(q) {
+	return User.findOne(q, {
+		fields: {
+			data: false
+		}
+	});
+}
+
+function webFingerAndVerify(query, verifier) {
+	return new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
+		if (error) {
+			return rej(error);
+		}
+
+		if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
+			return rej('WebFinger verfification failed');
+		}
+
+		res(result.object);
+	}));
+}
 
 /**
  * Show a user
@@ -12,6 +54,8 @@ import User, { pack } from '../../models/user';
  * @return {Promise<any>}
  */
 module.exports = (params, me) => new Promise(async (res, rej) => {
+	let user;
+
 	// Get 'user_id' parameter
 	const [userId, userIdErr] = $(params.user_id).optional.id().$;
 	if (userIdErr) return rej('invalid user_id param');
@@ -20,23 +64,142 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 	const [username, usernameErr] = $(params.username).optional.string().$;
 	if (usernameErr) return rej('invalid username param');
 
-	if (userId === undefined && username === undefined) {
-		return rej('user_id or username is required');
+	// Get 'host' parameter
+	const [host, hostErr] = $(params.host).optional.string().$;
+	if (hostErr) return rej('invalid username param');
+
+	if (userId === undefined && typeof username !== 'string') {
+		return rej('user_id or pair of username and host is required');
 	}
 
-	const q = userId !== undefined
-		? { _id: userId }
-		: { username_lower: username.toLowerCase() };
-
 	// Lookup user
-	const user = await User.findOne(q, {
-		fields: {
-			data: false
-		}
-	});
+	if (typeof host === 'string') {
+		const username_lower = username.toLowerCase();
+		const host_lower_ascii = toASCII(host).toLowerCase();
+		const host_lower = toUnicode(host_lower_ascii);
 
-	if (user === null) {
-		return rej('user not found');
+		user = await findUser({ username_lower, host_lower });
+
+		if (user === null) {
+			const acct_lower = `${username_lower}@${host_lower_ascii}`;
+			let activityStreams;
+			let finger;
+			let followers_count;
+			let following_count;
+			let likes_count;
+			let posts_count;
+
+			if (!validateUsername(username)) {
+				return rej('username validation failed');
+			}
+
+			try {
+				finger = await webFingerAndVerify(acct_lower, acct_lower);
+			} catch (exception) {
+				return rej('WebFinger lookup failed');
+			}
+
+			const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
+			if (!self) {
+				return rej('WebFinger has no reference to self representation');
+			}
+
+			try {
+				activityStreams = await request({
+					url: self.href,
+					headers: {
+						Accept: 'application/activity+json, application/ld+json'
+					},
+					json: true
+				});
+			} catch (exception) {
+				return rej('failed to retrieve ActivityStreams representation');
+			}
+
+			if (!(activityStreams &&
+				(Array.isArray(activityStreams['@context']) ?
+					activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') :
+					activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') &&
+				activityStreams.type === 'Person' &&
+				typeof activityStreams.preferredUsername === 'string' &&
+				activityStreams.preferredUsername.toLowerCase() === username_lower &&
+				isValidName(activityStreams.name) &&
+				isValidDescription(activityStreams.summary)
+			)) {
+				return rej('failed ActivityStreams validation');
+			}
+
+			try {
+				[followers_count, following_count, likes_count, posts_count] = await Promise.all([
+					getCollectionCount(activityStreams.followers),
+					getCollectionCount(activityStreams.following),
+					getCollectionCount(activityStreams.liked),
+					getCollectionCount(activityStreams.outbox),
+					webFingerAndVerify(activityStreams.id, acct_lower),
+				]);
+			} catch (exception) {
+				return rej('failed to fetch assets');
+			}
+
+			const summaryDOM = JSDOM.fragment(activityStreams.summary);
+
+			// Create user
+			user = await User.insert({
+				avatar_id: null,
+				banner_id: null,
+				created_at: new Date(),
+				description: summaryDOM.textContent,
+				followers_count,
+				following_count,
+				name: activityStreams.name,
+				posts_count,
+				likes_count,
+				liked_count: 0,
+				drive_capacity: 1073741824, // 1GB
+				username: username,
+				username_lower,
+				host: toUnicode(finger.subject.replace(/^.*?@/, '')),
+				host_lower,
+				account: {
+					uri: activityStreams.id,
+				},
+			});
+
+			const [icon, image] = await Promise.all([
+				activityStreams.icon,
+				activityStreams.image,
+			].map(async image => {
+				if (!image || image.type !== 'Image') {
+					return { _id: null };
+				}
+
+				try {
+					return await uploadFromUrl(image.url, user);
+				} catch (exception) {
+					return { _id: null };
+				}
+			}));
+
+			User.update({ _id: user._id }, {
+				$set: {
+					avatar_id: icon._id,
+					banner_id: image._id,
+				},
+			});
+
+			user.avatar_id = icon._id;
+			user.banner_id = icon._id;
+		}
+	} else {
+		const q = userId !== undefined
+			? { _id: userId }
+			: { username_lower: username.toLowerCase(), host: null };
+
+		user = await findUser(q);
+
+		if (user === null) {
+			return rej('user not found');
+		}
 	}
 
 	// Send response