ssb: Switch to a more brute force about-gathering approach. I think if I start with this and avoid querying all known accounts up-front, we will be plenty fast.

apps/ssb.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🦀",
-	"previous": "&gO8ganUDJtyGtfKUPRTbK2d34GJXhHKvHIZheF/lXI4=.sha256"
+	"previous": "&2TUIbhXFJWMc6NTZt2Q0RB004ufTD+6CuaC5i2DdpsU=.sha256"
 }

apps/ssb/tf-app.js

@@ -152,137 +152,36 @@ class TfElement extends LitElement {
 
 	async fetch_about(following, users) {
 		let ids = Object.keys(following).sort();
-		const k_cache_version = 1;
+		console.log('loading about for', ids.length, 'accounts');
-		let cache = await tfrpc.rpc.databaseGet('about');
-		let original_cache = cache;
-		cache = cache ? JSON.parse(cache) : {};
-		if (cache.version !== k_cache_version) {
-			cache = {
-				version: k_cache_version,
-				about: {},
-				last_row_id: 0,
-			};
-		}
-		let max_row_id = (
-			await tfrpc.rpc.query(
-				`
-					SELECT MAX(rowid) AS max_row_id FROM messages
-				`,
-				[]
-			)
-		)[0].max_row_id;
-		for (let id of Object.keys(cache.about)) {
-			if (ids.indexOf(id) == -1) {
-				delete cache.about[id];
-			}
-		}
-
-		const k_account_chunk_size = 4096;
-		const k_chunk_size = 1024;
-		let min_row_id = 0;
-		console.log(
-			'loading about for',
-			ids.length,
-			'accounts',
-			cache.last_row_id,
-			'=>',
-			max_row_id
-		);
 		try {
-			while (true) {
+			let rows = await tfrpc.rpc.query(
-				let max_seen;
+				`
-				for (
+					SELECT all_abouts.author, json(json_group_object(all_abouts.key, all_abouts.value)) AS about
-					let account_chunk = 0;
+					FROM (
-					account_chunk < ids.length;
+						SELECT
-					account_chunk += k_account_chunk_size
+							messages.author,
-				) {
+							fields.key,
-					let ids_chunk = ids.slice(
+							RANK() OVER (PARTITION BY messages.author, fields.key ORDER BY messages.sequence DESC) AS rank,
-						account_chunk,
+							fields.value
-						account_chunk + k_account_chunk_size
+						FROM messages JOIN json_each(messages.content) AS fields
-					);
+						WHERE
-					let abouts = await tfrpc.rpc.query(
+							messages.content ->> '$.type' = 'about' AND
-						`
+							messages.content ->> '$.about' = messages.author AND
-							WITH
+							NOT fields.key IN ('about', 'type')) all_abouts
-								past AS (
+					WHERE rank = 1
-									SELECT
+					GROUP BY all_abouts.author
-										messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
+				`,
-									FROM
+				[JSON.stringify(ids)]
-										messages
+			);
-									WHERE
+			users = users || {};
-										messages.rowid > ?3 AND
+			for (let row of rows) {
-										messages.rowid <= ?4 AND
+				users[row.author] = Object.assign(
-										messages.content ->> 'type' = 'about'
+					{follow_depth: following[row.author]?.d},
-								),
+					users[row.author] || {},
-								current AS (
+					JSON.parse(row.about)
-									SELECT
-										messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
-									FROM
-										messages
-									WHERE
-										messages.rowid > ?6 AND
-										messages.rowid <= ?4 AND
-										messages.content ->> 'type' = 'about'
-								)
-								SELECT * FROM past
-								JOIN json_each(?1) AS following
-								ON past.author = following.value
-								UNION SELECT * FROM current
-								JOIN json_each(?2) AS following
-								ON current.author = following.value
-								ORDER BY rowid LIMIT ?5
-						`,
-						[
-							JSON.stringify(ids_chunk.filter((id) => cache.about[id])),
-							JSON.stringify(ids_chunk.filter((id) => !cache.about[id])),
-							cache.last_row_id,
-							max_row_id,
-							k_chunk_size,
-							min_row_id,
-						]
-					);
-					for (let about of abouts) {
-						let content = JSON.parse(about.content);
-						if (content.about === about.author) {
-							delete content.type;
-							delete content.about;
-							cache.about[about.author] = Object.assign(
-								cache.about[about.author] || {},
-								content
-							);
-						}
-						max_seen = about.rowid;
-					}
-					console.log(account_chunk, '/', ids.length, 'accounts');
-				}
-				console.log(
-					'cache =',
-					cache.last_row_id,
-					'seen =',
-					max_seen,
-					'max =',
-					max_row_id
 				);
-				cache.last_row_id = Math.max(cache.last_row_id, max_seen ?? max_row_id);
-				min_row_id = Math.max(min_row_id, max_seen ?? max_row_id);
-				let new_cache = JSON.stringify(cache);
-				if (new_cache !== original_cache) {
-					let start_time = new Date();
-					tfrpc.rpc.databaseSet('about', new_cache).then(function () {
-						console.log('saving about took', (new Date() - start_time) / 1000);
-					});
-				}
-				users = users || {};
-				for (let id of Object.keys(cache.about)) {
-					users[id] = Object.assign(
-						{follow_depth: following[id]?.d},
-						users[id] || {},
-						cache.about[id]
-					);
-				}
-				if (cache.last_row_id >= max_row_id) {
-					break;
-				}
 			}
+			console.log('updated users');
 		} catch (e) {
 			console.log(e);
 		}

apps/ssb/tf-tab-news.js

@@ -98,7 +98,9 @@ class TfTabNewsElement extends LitElement {
 
 	unread_status(channel) {
 		if (channel === undefined) {
-			if (Object.keys(this.channels_unread).some(x => this.unread_status(x))) {
+			if (
+				Object.keys(this.channels_unread).some((x) => this.unread_status(x))
+			) {
 				return '✉️ ';
 			}
 		} else if (