From 18cf058af35fd43a56c8d3ec45319ae389935f00 Mon Sep 17 00:00:00 2001 From: Cory McWilliams Date: Tue, 21 Jan 2025 20:53:23 -0500 Subject: [PATCH] ssb: Experimenting with the news query strategy. --- apps/ssb.json | 2 +- apps/ssb/tf-tab-news-feed.js | 104 +++++++++++++++-------------------- 2 files changed, 46 insertions(+), 60 deletions(-) diff --git a/apps/ssb.json b/apps/ssb.json index 6c272ef9..55e1eb14 100644 --- a/apps/ssb.json +++ b/apps/ssb.json @@ -1,5 +1,5 @@ { "type": "tildefriends-app", "emoji": "🦀", - "previous": "&ZKixAk7qZ3OfnjvCoyT1MC7qYVpUz0ngaAozr11xihs=.sha256" + "previous": "&QQhMV7DY8GfmrAfjLFhOA2VScoGXlSA0MW520Pigrjg=.sha256" } diff --git a/apps/ssb/tf-tab-news-feed.js b/apps/ssb/tf-tab-news-feed.js index 1dbaa204..db8a3197 100644 --- a/apps/ssb/tf-tab-news-feed.js +++ b/apps/ssb/tf-tab-news-feed.js @@ -57,8 +57,7 @@ class TfTabNewsFeedElement extends LitElement { JOIN json_each(?2) AS following ON messages.author = following.value WHERE messages.author != ?1 AND - messages.timestamp >= ?3 AND - messages.timestamp < ?4 + ?3 IS NULL OR (messages.timestamp >= ?3 AND messages.timestamp < ?4) ORDER BY timestamp DESC limit 20) SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM mentions @@ -77,22 +76,21 @@ class TfTabNewsFeedElement extends LitElement { } else if (this.hash.startsWith('#@')) { result = await tfrpc.rpc.query( ` - WITH mine AS (SELECT rowid, id, previous, author, sequence, timestamp, hash, json(content) AS content, signature - FROM messages - WHERE messages.author = ? - ORDER BY sequence DESC) + WITH + mine AS (SELECT rowid, id, previous, author, sequence, timestamp, hash, json(content) AS content, signature + FROM messages + WHERE messages.author = ?), + selected AS ( + SELECT * FROM mine + WHERE ?2 IS NULL OR (mine.timestamp >= 2 AND mine.timestamp < ?3) + ORDER BY sequence DESC LIMIT 20 + ) SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM mine - JOIN messages_refs ON mine.id = messages_refs.ref + FROM selected + JOIN messages_refs ON selected.id = messages_refs.ref JOIN messages ON messages_refs.message = messages.id - WHERE - messages.timestamp >= ?2 AND - messages.timestamp < ?3 UNION - SELECT * FROM mine - WHERE - mine.timestamp >= ?2 AND - mine.timestamp < ?3 + SELECT * FROM selected `, [this.hash.substring(1), start_time, end_time] ); @@ -117,14 +115,21 @@ class TfTabNewsFeedElement extends LitElement { promises.push( tfrpc.rpc.query( ` - WITH news AS (SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM messages - JOIN json_each(?) AS following ON messages.author = following.value - WHERE - messages.timestamp >= ? AND - messages.timestamp < ? AND - messages.content ->> 'channel' = ? - ORDER BY messages.timestamp DESC) + WITH + all_news AS ( + SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature + FROM messages + JOIN json_each(?) AS following ON messages.author = following.value + WHERE messages.content ->> 'channel' = ? + UNION + SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature + FROM messages_fts(?5) + JOIN messages ON messages.rowid = messages_fts.rowid + JOIN json_each(?1) AS following ON messages.author = following.value + JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4), + news AS (SELECT * FROM all_news + WHERE ?2 IS NULL OR (all_news.timestamp >= ?2 AND all_news.timestamp < ?3) + ORDER BY all_news.timestamp DESC LIMIT 20) SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM news JOIN messages_refs ON news.id = messages_refs.ref @@ -135,15 +140,6 @@ class TfTabNewsFeedElement extends LitElement { JOIN messages_refs ON news.id = messages_refs.message JOIN messages ON messages_refs.ref = messages.id UNION - SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM messages_fts(?5) - JOIN messages ON messages.rowid = messages_fts.rowid - JOIN json_each(?1) AS following ON messages.author = following.value - JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4 - WHERE - messages.timestamp >= ?2 AND - messages.timestamp < ?3 - UNION SELECT news.* FROM news `, [ @@ -164,10 +160,9 @@ class TfTabNewsFeedElement extends LitElement { FROM messages JOIN json_each(?1) AS following ON messages.author = following.value WHERE - messages.timestamp >= ?2 AND - messages.timestamp < ?3 AND + ?2 IS NULL OR (messages.timestamp >= ?2 AND messages.timestamp < ?3) AND json(messages.content) LIKE '"%' - ORDER BY sequence DESC + ORDER BY sequence DESC LIMIT 20 `, [JSON.stringify(this.following), start_time, end_time] ); @@ -179,11 +174,17 @@ class TfTabNewsFeedElement extends LitElement { promises.push( tfrpc.rpc.query( ` - WITH news AS (SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature - FROM messages - JOIN json_each(?) AS following ON messages.author = following.value - WHERE messages.timestamp >= ? AND messages.timestamp < ? - ORDER BY messages.timestamp DESC) + WITH + all_news AS ( + SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature + FROM messages + JOIN json_each(?) AS following ON messages.author = following.value + WHERE timestamp >= 0 AND timestamp < ?3), + news AS ( + SELECT * FROM all_news + WHERE ?2 IS NULL or (all_news.timestamp >= ?2 AND all_news.timestamp < ?3) + ORDER BY timestamp DESC LIMIT 20 + ) SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM news JOIN messages_refs ON news.id = messages_refs.ref @@ -313,6 +314,7 @@ class TfTabNewsFeedElement extends LitElement { } async load_messages() { + let start_time = new Date(); let self = this; this.loading++; let messages = []; @@ -327,37 +329,21 @@ class TfTabNewsFeedElement extends LitElement { this.start_time = start_time; this.time_range = [this.start_time, now + 24 * 60 * 60 * 1000]; messages = await this.fetch_messages( - this.time_range[0], + null, this.time_range[1] ); this.update_time_range_from_messages( messages.filter( - (x) => - x.timestamp >= this.time_range[0] && - x.timestamp < this.time_range[1] + (x) => x.timestamp < this.time_range[1] ) ); messages = await this.decrypt(messages); - if (!messages.length) { - let more = []; - while (!more.length && start_time >= 0) { - let last_start_time = start_time; - start_time = last_start_time - 7 * 24 * 60 * 60 * 1000; - more = await this.fetch_messages(start_time, last_start_time); - this.update_time_range_from_messages( - more.filter( - (x) => x.timestamp >= start_time && x.timestamp < last_start_time - ) - ); - } - messages = await this.decrypt([...more, ...this.messages]); - } } finally { this.loading--; } this.messages = this.merge_messages(this.messages, messages); this.time_loading = undefined; - console.log(`loading messages done for ${self.whoami}`); + console.log(`loading messages done for ${self.whoami} in ${(new Date() - start_time) / 1000}s`); } mark_all_read() {