From f1ced31f69d868d1213cf260d153bba80399597a Mon Sep 17 00:00:00 2001
From: Cory McWilliams <cory@unprompted.com>
Date: Wed, 7 May 2025 17:52:02 -0400
Subject: [PATCH] ssb: Faster loads by encouraging the right queries with CTEs
 in fetch_about.

---
 apps/ssb.json      |   2 +-
 apps/ssb/tf-app.js | 101 ++++++++++++++++++++++++---------------------
 2 files changed, 56 insertions(+), 47 deletions(-)

diff --git a/apps/ssb.json b/apps/ssb.json
index df89f563..518c0d6a 100644
--- a/apps/ssb.json
+++ b/apps/ssb.json
@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🦀",
-	"previous": "&YeFLcHjD39FzsciGDfXuLd0hPzu4Rrp7QlMA7NaUCio=.sha256"
+	"previous": "&RkHUl/WRe0UpvTB2W6cw0f4AeupL/2PEI1pXg5PDXBg=.sha256"
 }
diff --git a/apps/ssb/tf-app.js b/apps/ssb/tf-app.js
index 3421f097..41eb5b36 100644
--- a/apps/ssb/tf-app.js
+++ b/apps/ssb/tf-app.js
@@ -177,6 +177,7 @@ class TfElement extends LitElement {
 			}
 		}
 
+		const k_account_chunk_size = 4096;
 		const k_chunk_size = 1024;
 		let min_row_id = 0;
 		console.log(
@@ -189,54 +190,62 @@ class TfElement extends LitElement {
 		);
 		try {
 			while (true) {
-				let abouts = await tfrpc.rpc.query(
-					`
-						SELECT * FROM (
-							SELECT
-								messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
-							FROM
-								messages,
-								json_each(?1) AS following
-							WHERE
-								messages.author = following.value AND
-								messages.content ->> 'type' = 'about' AND
-								messages.rowid > ?3 AND
-								messages.rowid <= ?4
-							UNION
-							SELECT
-								messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
-							FROM
-								messages,
-								json_each(?2) AS following
-							WHERE
-								messages.author = following.value AND
-								messages.content ->> 'type' = 'about' AND
-								messages.rowid > ?6 AND
-								messages.rowid <= ?4
-						)
-						ORDER BY rowid LIMIT ?5
-					`,
-					[
-						JSON.stringify(ids.filter((id) => cache.about[id])),
-						JSON.stringify(ids.filter((id) => !cache.about[id])),
-						cache.last_row_id,
-						max_row_id,
-						k_chunk_size,
-						min_row_id,
-					]
-				);
 				let max_seen;
-				for (let about of abouts) {
-					let content = JSON.parse(about.content);
-					if (content.about === about.author) {
-						delete content.type;
-						delete content.about;
-						cache.about[about.author] = Object.assign(
-							cache.about[about.author] || {},
-							content
-						);
+				for (let account_chunk = 0; account_chunk < ids.length; account_chunk += k_account_chunk_size) {
+					let ids_chunk = ids.slice(account_chunk, account_chunk + k_account_chunk_size);
+					let abouts = await tfrpc.rpc.query(
+						`
+							WITH
+								past AS (
+									SELECT
+										messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
+									FROM
+										messages
+									WHERE
+										messages.rowid > ?3 AND
+										messages.rowid <= ?4 AND
+										messages.content ->> 'type' = 'about'
+								),
+								current AS (
+									SELECT
+										messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
+									FROM
+										messages
+									WHERE
+										messages.rowid > ?6 AND
+										messages.rowid <= ?4 AND
+										messages.content ->> 'type' = 'about'
+								)
+								SELECT * FROM past
+								JOIN json_each(?1) AS following
+								ON past.author = following.value
+								UNION SELECT * FROM current
+								JOIN json_each(?2) AS following
+								ON current.author = following.value
+								ORDER BY rowid LIMIT ?5
+						`,
+						[
+							JSON.stringify(ids_chunk.filter((id) => cache.about[id])),
+							JSON.stringify(ids_chunk.filter((id) => !cache.about[id])),
+							cache.last_row_id,
+							max_row_id,
+							k_chunk_size,
+							min_row_id,
+						]
+					);
+					for (let about of abouts) {
+						let content = JSON.parse(about.content);
+						if (content.about === about.author) {
+							delete content.type;
+							delete content.about;
+							cache.about[about.author] = Object.assign(
+								cache.about[about.author] || {},
+								content
+							);
+						}
+						max_seen = about.rowid;
 					}
-					max_seen = about.rowid;
+					console.log(account_chunk, '/', ids.length, 'accounts');
 				}
 				console.log(
 					'cache =',