Oh yeah, I was playing with the follow app.

git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@4368 ed5197a5-7fde-0310-b194-c3ffbd925b24

apps/follow/app.js

@@ -1,73 +1,163 @@
-var g_following_cache = {};
+let g_about_cache = {};
-var g_following_deep_cache = {};
-var g_about_cache = {};
 
-async function following(db, id) {
+async function query(sql, args) {
-	if (g_following_cache[id]) {
+	let result = [];
-		return g_following_cache[id];
+	await ssb.sqlAsync(sql, args, function(row) {
-	}
+		result.push(row);
-	var o = await db.get(id + ":following");
-	const k_version = 5;
-	var f = o ? JSON.parse(o) : o;
-	if (!f || f.version != k_version) {
-		f = {users: [], sequence: 0, version: k_version};
-	}
-	f.users = new Set(f.users);
-	await ssb.sqlAsync(
-		"SELECT "+
-		"  sequence, "+
-		"  json_extract(content, '$.contact') AS contact, "+
-		"  json_extract(content, '$.following') AS following "+
-		"FROM messages "+
-		"WHERE "+
-		"  author = ?1 AND "+
-		"  sequence > ?2 AND "+
-		"  json_extract(content, '$.type') = 'contact' "+
-		"UNION SELECT MAX(sequence) AS sequence, NULL, NULL FROM messages WHERE author = ?1 "+
-		"ORDER BY sequence",
-		[id, f.sequence],
-		function(row) {
-			if (row.following) {
-				f.users.add(row.contact);
-			} else {
-				f.users.delete(row.contact);
-			}
-			f.sequence = row.sequence;
 	});
-	var as_set = f.users;
+	return result;
-	f.users = Array.from(f.users).sort();
-	var j = JSON.stringify(f);
-	if (o != j) {
-		await db.set(id + ":following", j);
-	}
-	f.users = as_set;
-	g_following_cache[id] = f.users;
-	return f.users;
 }
 
-async function followingDeep(db, seed_ids, depth) {
+async function contacts_internal(id, last_row_id, following, max_row_id) {
-	if (depth <= 0) {
+	let result = Object.assign({}, following[id] || {});
-		return seed_ids;
+	result.following = result.following || {};
+	result.blocking = result.blocking || {};
+	let contacts = await query(
+		`
+				SELECT content FROM messages
+				WHERE author = ? AND
+				rowid > ? AND
+				rowid <= ? AND
+				json_extract(content, '$.type') = 'contact'
+				ORDER BY sequence
+			`,
+		[id, last_row_id, max_row_id]);
+	for (let row of contacts) {
+		let contact = JSON.parse(row.content);
+		if (contact.following === true) {
+			result.following[contact.contact] = true;
+		} else if (contact.following === false) {
+			delete result.following[contact.contact];
+		} else if (contact.blocking === true) {
+			result.blocking[contact.contact] = true;
+		} else if (contact.blocking === false) {
+			delete result.blocking[contact.contact];
 		}
-	var key = JSON.stringify([seed_ids, depth]);
-	if (g_following_deep_cache[key]) {
-		return g_following_deep_cache[key];
 	}
-	var f = await Promise.all(seed_ids.map(x => following(db, x).then(x => [...x])));
+	following[id] = result;
-	var ids = [].concat(...f);
+	return result;
-	var x = await followingDeep(db, [...new Set(ids)].sort(), depth - 1);
+}
-	x = [...new Set([].concat(...x, ...seed_ids))].sort();
+
-	g_following_deep_cache[key] = x;
+async function contact(id, last_row_id, following, max_row_id) {
-	return x;
+	return await contacts_internal(id, last_row_id, following, max_row_id);
+}
+
+async function following_deep_internal(ids, depth, blocking, last_row_id, following, max_row_id) {
+	let contacts = await Promise.all([...new Set(ids)].map(x => contact(x, last_row_id, following, max_row_id)));
+	let result = {};
+	for (let i = 0; i < ids.length; i++) {
+		let id = ids[i];
+		let contact = contacts[i];
+		let all_blocking = Object.assign({}, contact.blocking, blocking);
+		let found = Object.keys(contact.following).filter(y => !all_blocking[y]);
+		let deeper = depth > 1 ? await following_deep_internal(found, depth - 1, all_blocking, last_row_id, following, max_row_id) : [];
+		result[id] = [id, ...found, ...deeper];
+	}
+	return [...new Set(Object.values(result).flat())];
+}
+
+async function following_deep(ids, depth, blocking) {
+	let db = await database('cache');
+	const k_cache_version = 5;
+	let cache = await db.get('following');
+	cache = cache ? JSON.parse(cache) : {};
+	if (cache.version !== k_cache_version) {
+		cache = {
+			version: k_cache_version,
+			following: {},
+			last_row_id: 0,
+		};
+	}
+	let max_row_id = (await query(`
+			SELECT MAX(rowid) AS max_row_id FROM messages
+		`, []))[0].max_row_id;
+	let result = await following_deep_internal(ids, depth, blocking, cache.last_row_id, cache.following, max_row_id);
+	cache.last_row_id = max_row_id;
+	let store = JSON.stringify(cache);
+	await db.set('following', store);
+	return result;
+}
+
+async function fetch_about(db, ids, users) {
+	const k_cache_version = 1;
+	let cache = await db.get('about');
+	cache = cache ? JSON.parse(cache) : {};
+	if (cache.version !== k_cache_version) {
+		cache = {
+			version: k_cache_version,
+			about: {},
+			last_row_id: 0,
+		};
+	}
+	let max_row_id = 0;
+	await ssb.sqlAsync(`
+			SELECT MAX(rowid) AS max_row_id FROM messages
+		`,
+		[],
+		function(row) {
+			max_row_id = row.max_row_id;
+		});
+	for (let id of Object.keys(cache.about)) {
+		if (ids.indexOf(id) == -1) {
+			delete cache.about[id];
+		}
+	}
+
+	let abouts = [];
+	await ssb.sqlAsync(
+		`
+				SELECT
+					messages.*
+				FROM
+					messages,
+					json_each(?1) AS following
+				WHERE
+					messages.author = following.value AND
+					messages.rowid > ?3 AND
+					messages.rowid <= ?4 AND
+					json_extract(messages.content, '$.type') = 'about'
+				UNION
+				SELECT
+					messages.*
+				FROM
+					messages,
+					json_each(?2) AS following
+				WHERE
+					messages.author = following.value AND
+					messages.rowid <= ?4 AND
+					json_extract(messages.content, '$.type') = 'about'
+				ORDER BY messages.author, messages.sequence
+			`,
+		[
+			JSON.stringify(ids.filter(id => cache.about[id])),
+			JSON.stringify(ids.filter(id => !cache.about[id])),
+			cache.last_row_id,
+			max_row_id,
+		]);
+	for (let about of abouts) {
+		let content = JSON.parse(about.content);
+		if (content.about === about.author) {
+			delete content.type;
+			delete content.about;
+			cache.about[about.author] = Object.assign(cache.about[about.author] || {}, content);
+		}
+	}
+	cache.last_row_id = max_row_id;
+	await db.set('about', JSON.stringify(cache));
+	users = users || {};
+	for (let id of Object.keys(cache.about)) {
+		users[id] = Object.assign(users[id] || {}, cache.about[id]);
+	}
+	return Object.assign({}, users);
 }
 
 async function getAbout(db, id) {
 	if (g_about_cache[id]) {
 		return g_about_cache[id];
 	}
-	var o = await db.get(id + ":about");
+	let o = await db.get(id + ":about");
 	const k_version = 4;
-	var f = o ? JSON.parse(o) : o;
+	let f = o ? JSON.parse(o) : o;
 	if (!f || f.version != k_version) {
 		f = {about: {}, sequence: 0, version: k_version};
 	}
@@ -87,7 +177,7 @@ async function getAbout(db, id) {
 		function(row) {
 			f.sequence = row.sequence;
 			if (row.content) {
-				var about = {};
+				let about = {};
 				try {
 					about = JSON.parse(row.content);
 				} catch {
@@ -97,7 +187,7 @@ async function getAbout(db, id) {
 				f.about = Object.assign(f.about, about);
 			}
 		});
-	var j = JSON.stringify(f);
+	let j = JSON.stringify(f);
 	if (o != j) {
 		await db.set(id + ":about", j);
 	}
@@ -108,7 +198,7 @@ async function getAbout(db, id) {
 async function getSize(db, id) {
 	let size = 0;
 	await ssb.sqlAsync(
-		"SELECT (SUM(LENGTH(content)) + SUM(LENGTH(author)) + SUM(LENGTH(id))) AS size FROM messages WHERE author = ?1",
+		"SELECT (LENGTH(author) * COUNT(*) + SUM(LENGTH(content)) + SUM(LENGTH(id))) AS size FROM messages WHERE author = ?1",
 		[id],
 		function (row) {
 			size += row.size;
@@ -116,6 +206,25 @@ async function getSize(db, id) {
 	return size;
 }
 
+
+async function getSizes(ids) {
+	let sizes = {};
+	await ssb.sqlAsync(
+		`
+			SELECT
+				author,
+				(SUM(LENGTH(content)) + SUM(LENGTH(author)) + SUM(LENGTH(messages.id))) AS size
+			FROM messages
+			JOIN json_each(?) AS ids ON author = ids.value
+			GROUP BY author
+		`,
+		[JSON.stringify(ids)],
+		function (row) {
+			sizes[row.author] = row.size;
+		});
+	return sizes;
+}
+
 function niceSize(bytes) {
 	let value = bytes;
 	let unit = 'B';
@@ -131,27 +240,28 @@ function niceSize(bytes) {
 	return Math.round(value * 10) / 10 + ' ' + unit;
 }
 
-async function buildTree(db, root, indent, depth) {
+function escape(value) {
-	var f = await following(db, root);
+	return value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
-	var result = indent + '[' + f.size + '] ' + '<a target="_top" href="../index/#' + root + '">' + ((await getAbout(db, root)).name || root) + '</a> ' + niceSize(await getSize(db, root)) + '\n';
-	if (depth > 0) {
-		for (let next of f) {
-			result += await buildTree(db, next, indent + '  ', depth - 1);
-		}
-	}
-	return result;
 }
 
 async function main() {
 	await app.setDocument('<pre style="color: #fff">building...</pre>');
-	var db = await database('ssb');
+	let db = await database('ssb');
-	var whoami = await ssb.getIdentities();
+	let whoami = await ssb.getIdentities();
-	var tree = '';
+	let tree = '';
-	for (let id of whoami) {
+	await app.setDocument(`<pre style="color: #fff">Enumerating followed users...</pre>`);
-		await app.setDocument(`<pre style="color: #fff">building... ${id}</pre>`);
+	let following = await following_deep(whoami, 2, {});
-		tree += await buildTree(db, id, '', 2);
+	await app.setDocument(`<pre style="color: #fff">Getting names and sizes...</pre>`);
+	let [about, sizes] = await Promise.all([
+		fetch_about(db, following, {}),
+		getSizes(following),
+	]);
+	await app.setDocument(`<pre style="color: #fff">Finishing...</pre>`);
+	following.sort((a, b) => ((sizes[b] ?? 0) - (sizes[a] ?? 0)));
+	for (let id of following) {
+		tree += `<li><a href="/~core/ssb/#${id}">${escape(about[id]?.name ?? id)}</a> ${niceSize(sizes[id] ?? 0)}</li>\n`;
 	}
-	await app.setDocument('<pre style="color: #fff">FOLLOWING:\n' + tree + '</pre>');
+	await app.setDocument('<!DOCTYPE html>\n<html>\n<body style="color: #fff"><h1>Following</h1>\n<ul>' + tree + '</ul>\n</body>\n</html>');
 }
 
 main();