From 5a818d2119a8b9e34dc6b316333a1efd67bb68c3 Mon Sep 17 00:00:00 2001
From: Cory McWilliams <cory@unprompted.com>
Date: Wed, 9 Apr 2025 22:08:33 -0400
Subject: [PATCH] ssb: More incremental profile loading?

---
 apps/ssb.json      |   2 +-
 apps/ssb/tf-app.js | 127 +++++++++++++++++++++++++--------------------
 2 files changed, 73 insertions(+), 56 deletions(-)

diff --git a/apps/ssb.json b/apps/ssb.json
index 40568eb5..99bc2a71 100644
--- a/apps/ssb.json
+++ b/apps/ssb.json
@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🦀",
-	"previous": "&VE+OD1O5xdeiqipvbxXpmatUHu7dTbliuQpFnuZgsGQ=.sha256"
+	"previous": "&h9uzh3gVWy1KWIu6wFYCrBZkVmnQAi7QLAj5iIXffFM=.sha256"
 }
diff --git a/apps/ssb/tf-app.js b/apps/ssb/tf-app.js
index a3debd56..ef784825 100644
--- a/apps/ssb/tf-app.js
+++ b/apps/ssb/tf-app.js
@@ -175,64 +175,81 @@ class TfElement extends LitElement {
 			}
 		}
 
+		const k_chunk_size = 1024;
+		let min_row_id = 0;
+		console.log('loading about for', ids.length, 'accounts', cache.last_row_id, '=>', max_row_id);
 		try {
-			let abouts = await tfrpc.rpc.query(
-				`
-					SELECT
-						messages.author, json(messages.content) AS content, messages.sequence
-					FROM
-						messages,
-						json_each(?1) AS following
-					WHERE
-						messages.author = following.value AND
-						messages.content ->> 'type' = 'about' AND
-						messages.rowid > ?3 AND
-						messages.rowid <= ?4
-					UNION
-					SELECT
-						messages.author, json(messages.content) AS content, messages.sequence
-					FROM
-						messages,
-						json_each(?2) AS following
-					WHERE
-						messages.author = following.value AND
-						messages.content ->> 'type' = 'about' AND
-						messages.rowid <= ?4
-					ORDER BY messages.author, messages.sequence
-				`,
-				[
-					JSON.stringify(ids.filter((id) => cache.about[id])),
-					JSON.stringify(ids.filter((id) => !cache.about[id])),
-					cache.last_row_id,
-					max_row_id,
-				]
-			);
-			for (let about of abouts) {
-				let content = JSON.parse(about.content);
-				if (content.about === about.author) {
-					delete content.type;
-					delete content.about;
-					cache.about[about.author] = Object.assign(
-						cache.about[about.author] || {},
-						content
+			while (true) {
+				let abouts = await tfrpc.rpc.query(
+					`
+						SELECT * FROM (
+							SELECT
+								messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
+							FROM
+								messages,
+								json_each(?1) AS following
+							WHERE
+								messages.author = following.value AND
+								messages.content ->> 'type' = 'about' AND
+								messages.rowid > ?3 AND
+								messages.rowid <= ?4
+							UNION
+							SELECT
+								messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
+							FROM
+								messages,
+								json_each(?2) AS following
+							WHERE
+								messages.author = following.value AND
+								messages.content ->> 'type' = 'about' AND
+								messages.rowid > ?6 AND
+								messages.rowid <= ?4
+						)
+						ORDER BY rowid LIMIT ?5
+					`,
+					[
+						JSON.stringify(ids.filter((id) => cache.about[id])),
+						JSON.stringify(ids.filter((id) => !cache.about[id])),
+						cache.last_row_id,
+						max_row_id,
+						k_chunk_size,
+						min_row_id,
+					]
+				);
+				let max_seen;
+				for (let about of abouts) {
+					let content = JSON.parse(about.content);
+					if (content.about === about.author) {
+						delete content.type;
+						delete content.about;
+						cache.about[about.author] = Object.assign(
+							cache.about[about.author] || {},
+							content
+						);
+					}
+					max_seen = about.rowid;
+				}
+				console.log('cache =', cache.last_row_id, 'seen =', max_seen, 'max =', max_row_id);
+				cache.last_row_id = Math.max(cache.last_row_id, max_seen ?? max_row_id);
+				min_row_id = Math.max(min_row_id, max_seen ?? max_row_id);
+				let new_cache = JSON.stringify(cache);
+				if (new_cache !== original_cache) {
+					let start_time = new Date();
+					tfrpc.rpc.databaseSet('about', new_cache).then(function () {
+						console.log('saving about took', (new Date() - start_time) / 1000);
+					});
+				}
+				users = users || {};
+				for (let id of Object.keys(cache.about)) {
+					users[id] = Object.assign(
+						{follow_depth: following[id]?.d},
+						users[id] || {},
+						cache.about[id]
 					);
 				}
-			}
-			cache.last_row_id = max_row_id;
-			let new_cache = JSON.stringify(cache);
-			if (new_cache !== original_cache) {
-				let start_time = new Date();
-				tfrpc.rpc.databaseSet('about', new_cache).then(function () {
-					console.log('saving about took', (new Date() - start_time) / 1000);
-				});
-			}
-			users = users || {};
-			for (let id of Object.keys(cache.about)) {
-				users[id] = Object.assign(
-					{follow_depth: following[id]?.d},
-					users[id] || {},
-					cache.about[id]
-				);
+				if (cache.last_row_id >= max_row_id) {
+					break;
+				}
 			}
 		} catch (e) {
 			console.log(e);