From 7f252e79b6e3a3453fdc2c00120221ca6b0a89c5 Mon Sep 17 00:00:00 2001 From: Cory McWilliams Date: Wed, 9 Apr 2025 18:50:14 -0400 Subject: [PATCH] ssb: Faster channel loads. --- apps/ssb.json | 2 +- apps/ssb/tf-tab-news-feed.js | 120 +++++++++++++++++------------------ src/ssb.db.c | 3 +- 3 files changed, 63 insertions(+), 62 deletions(-) diff --git a/apps/ssb.json b/apps/ssb.json index 181f31cd..5a65323d 100644 --- a/apps/ssb.json +++ b/apps/ssb.json @@ -1,5 +1,5 @@ { "type": "tildefriends-app", "emoji": "🦀", - "previous": "&shP+cVoCoktYoePzAG2wj1vCx/eGXZPnoP1MDHT3d6g=.sha256" + "previous": "&r6gBoGYTO1yQXHLBPut0+iNd3NNFKPO/LXNV+F5+46M=.sha256" } diff --git a/apps/ssb/tf-tab-news-feed.js b/apps/ssb/tf-tab-news-feed.js index 9729d9bb..103b48a4 100644 --- a/apps/ssb/tf-tab-news-feed.js +++ b/apps/ssb/tf-tab-news-feed.js @@ -46,6 +46,53 @@ class TfTabNewsFeedElement extends LitElement { : this.hash.substring(1); } + async _fetch_related_messages(messages) { + let refs = await tfrpc.rpc.query( + ` + WITH + news AS ( + SELECT value AS id FROM json_each(?) + ) + SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id + UNION + SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id + `, + [JSON.stringify(messages.map((x) => x.id))] + ); + let related_messages = await tfrpc.rpc.query( + ` + SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature + FROM messages + JOIN json_each(?2) refs ON messages.id = refs.value + JOIN json_each(?1) AS following ON messages.author = following.value + `, + [JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))] + ); + let combined = [].concat(messages, related_messages); + let refs2 = await tfrpc.rpc.query( + ` + WITH + news AS ( + SELECT value AS id FROM json_each(?) + ) + SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id + UNION + SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id + `, + [JSON.stringify(combined.map((x) => x.id))] + ); + return [].concat(combined, await tfrpc.rpc.query( + ` + SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature + FROM messages + JOIN json_each(?2) refs ON messages.id = refs.value + JOIN json_each(?1) AS following ON messages.author = following.value + WHERE messages.content ->> 'type' = 'vote' + `, + [JSON.stringify(this.following), JSON.stringify(refs2.map((x) => x.ref))] + )); + } + async fetch_messages(start_time, end_time) { this.time_loading = [start_time, end_time]; let result; @@ -107,7 +154,8 @@ class TfTabNewsFeedElement extends LitElement { [this.hash.substring(1)] ); } else if (this.hash.startsWith('##')) { - result = await tfrpc.rpc.query( + let t0 = new Date(); + let initial_messages = await tfrpc.rpc.query( ` WITH all_news AS ( @@ -120,20 +168,11 @@ class TfTabNewsFeedElement extends LitElement { FROM messages_fts(?5) JOIN messages ON messages.rowid = messages_fts.rowid JOIN json_each(?1) AS following ON messages.author = following.value - JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4), + JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4 + ), news AS (SELECT * FROM all_news WHERE (?2 IS NULL OR all_news.timestamp >= ?2) AND all_news.timestamp < ?3 ORDER BY all_news.timestamp DESC LIMIT 20) - SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM news - JOIN messages_refs ON news.id = messages_refs.ref - JOIN messages ON messages_refs.message = messages.id - UNION - SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM news - JOIN messages_refs ON news.id = messages_refs.message - JOIN messages ON messages_refs.ref = messages.id - UNION SELECT TRUE AS is_primary, news.* FROM news `, [ @@ -144,6 +183,12 @@ class TfTabNewsFeedElement extends LitElement { '"#' + this.hash.substring(2).replace('"', '""') + '"', ] ); + let t1 = new Date(); + result = await this._fetch_related_messages(initial_messages); + let t2 = new Date(); + console.log( + `load of ${result.length} rows took ${(t2 - t0) / 1000} (${(t1 - t0) / 1000} to find ${initial_messages.length} initial messages, ${(t2 - t1) / 1000} to find ${result.length} total messages) following=${this.following.length} st=${start_time} et=${end_time}` + ); } else if (this.hash == '#🔐') { result = await tfrpc.rpc.query( ` @@ -160,7 +205,7 @@ class TfTabNewsFeedElement extends LitElement { result = (await this.decrypt(result)).filter((x) => x.decrypted); } else { let t0 = new Date(); - result = await tfrpc.rpc.query( + let initial_messages = await tfrpc.rpc.query( ` WITH all_news AS ( @@ -177,56 +222,11 @@ class TfTabNewsFeedElement extends LitElement { `, [JSON.stringify(this.following), start_time, end_time] ); - let news_length = result.length; let t1 = new Date(); - let refs = await tfrpc.rpc.query( - ` - WITH - news AS ( - SELECT value AS id FROM json_each(?) - ) - SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id - UNION - SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id - `, - [JSON.stringify(result.map((x) => x.id))] - ); + result = await this._fetch_related_messages(initial_messages); let t2 = new Date(); - let related_messages = await tfrpc.rpc.query( - ` - SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM messages - JOIN json_each(?2) refs ON messages.id = refs.value - JOIN json_each(?1) AS following ON messages.author = following.value - `, - [JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))] - ); - result = [].concat(result, related_messages); - refs = await tfrpc.rpc.query( - ` - WITH - news AS ( - SELECT value AS id FROM json_each(?) - ) - SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id - UNION - SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id - `, - [JSON.stringify(result.map((x) => x.id))] - ); - result = [].concat(result, await tfrpc.rpc.query( - ` - SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM messages - JOIN json_each(?2) refs ON messages.id = refs.value - JOIN json_each(?1) AS following ON messages.author = following.value - WHERE messages.content ->> 'type' = 'vote' - `, - [JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))] - )); - let t3 = new Date(); console.log( - `load of ${result.length} rows took ${(t3 - t0) / 1000} (${(t1 - t0) / 1000} to find ${news_length} messages, ${(t2 - t1) / 1000} to find ${refs.length} refs, and ${(t3 - t2) / 1000} to find ${related_messages.length} related messages) following=${this.following.length} st=${start_time} et=${end_time}` + `load of ${result.length} rows took ${(t2 - t0) / 1000} (${(t1 - t0) / 1000} to find ${initial_messages.length} initial messages, ${(t2 - t1) / 1000} to find ${result.length} total messages) following=${this.following.length} st=${start_time} et=${end_time}` ); } this.time_loading = undefined; diff --git a/src/ssb.db.c b/src/ssb.db.c index 31ca7104..497ecce1 100644 --- a/src/ssb.db.c +++ b/src/ssb.db.c @@ -183,12 +183,13 @@ void tf_ssb_db_init(tf_ssb_t* ssb) _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_timestamp_index ON messages (content ->> 'type', timestamp)"); _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_size_by_author_index ON messages (author, length(content))"); _tf_ssb_db_exec( - db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_root_rowid_index ON messages (author, content ->> 'type', content ->> 'channel', content ->> 'root')"); + db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_root_timestamp_index ON messages (author, timestamp, content ->> 'type', content ->> 'channel', content ->> 'root')"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_author_id_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_by_author_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_timestamp_author_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_id_index"); + _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_root_rowid_index"); _tf_ssb_db_exec(db, "CREATE TABLE IF NOT EXISTS blobs (" " id TEXT PRIMARY KEY,"