cory/tildefriends
5
6
Fork 2
Code Issues 35 Pull Requests Actions Releases 31 Wiki Activity

ssb: Faster channel loads.

Browse Source
This commit is contained in:
Cory McWilliams 2025-04-09 18:50:14 -04:00
parent ba2bb17638
commit 7f252e79b6
3 changed files with 63 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/ssb.json
View File

@ -1,5 +1,5 @@
{ {
"type": "tildefriends-app", "type": "tildefriends-app",
"emoji": "🦀", "emoji": "🦀",
"previous": "&shP+cVoCoktYoePzAG2wj1vCx/eGXZPnoP1MDHT3d6g=.sha256" "previous": "&r6gBoGYTO1yQXHLBPut0+iNd3NNFKPO/LXNV+F5+46M=.sha256"
} }

120
apps/ssb/tf-tab-news-feed.js
View File

@ -46,6 +46,53 @@ class TfTabNewsFeedElement extends LitElement {
: this.hash.substring(1); : this.hash.substring(1);
} }
async _fetch_related_messages(messages) {
let refs = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(messages.map((x) => x.id))]
);
let related_messages = await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
`,
[JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))]
);
let combined = [].concat(messages, related_messages);
let refs2 = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(combined.map((x) => x.id))]
);
return [].concat(combined, await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
WHERE messages.content ->> 'type' = 'vote'
`,
[JSON.stringify(this.following), JSON.stringify(refs2.map((x) => x.ref))]
));
}
async fetch_messages(start_time, end_time) { async fetch_messages(start_time, end_time) {
this.time_loading = [start_time, end_time]; this.time_loading = [start_time, end_time];
let result; let result;
@ -107,7 +154,8 @@ class TfTabNewsFeedElement extends LitElement {
[this.hash.substring(1)] [this.hash.substring(1)]
); );
} else if (this.hash.startsWith('##')) { } else if (this.hash.startsWith('##')) {
result = await tfrpc.rpc.query( let t0 = new Date();
let initial_messages = await tfrpc.rpc.query(
` `
WITH WITH
all_news AS ( all_news AS (
@ -120,20 +168,11 @@ class TfTabNewsFeedElement extends LitElement {
FROM messages_fts(?5) FROM messages_fts(?5)
JOIN messages ON messages.rowid = messages_fts.rowid JOIN messages ON messages.rowid = messages_fts.rowid
JOIN json_each(?1) AS following ON messages.author = following.value JOIN json_each(?1) AS following ON messages.author = following.value
JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4), JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4
),
news AS (SELECT * FROM all_news news AS (SELECT * FROM all_news
WHERE (?2 IS NULL OR all_news.timestamp >= ?2) AND all_news.timestamp < ?3 WHERE (?2 IS NULL OR all_news.timestamp >= ?2) AND all_news.timestamp < ?3
ORDER BY all_news.timestamp DESC LIMIT 20) ORDER BY all_news.timestamp DESC LIMIT 20)
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM news
JOIN messages_refs ON news.id = messages_refs.ref
JOIN messages ON messages_refs.message = messages.id
UNION
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM news
JOIN messages_refs ON news.id = messages_refs.message
JOIN messages ON messages_refs.ref = messages.id
UNION
SELECT TRUE AS is_primary, news.* FROM news SELECT TRUE AS is_primary, news.* FROM news
`, `,
[ [
@ -144,6 +183,12 @@ class TfTabNewsFeedElement extends LitElement {
'"#' + this.hash.substring(2).replace('"', '""') + '"', '"#' + this.hash.substring(2).replace('"', '""') + '"',
] ]
); );
let t1 = new Date();
result = await this._fetch_related_messages(initial_messages);
let t2 = new Date();
console.log(
`load of ${result.length} rows took ${(t2 - t0) / 1000} (${(t1 - t0) / 1000} to find ${initial_messages.length} initial messages, ${(t2 - t1) / 1000} to find ${result.length} total messages) following=${this.following.length} st=${start_time} et=${end_time}`
);
} else if (this.hash == '#🔐') { } else if (this.hash == '#🔐') {
result = await tfrpc.rpc.query( result = await tfrpc.rpc.query(
` `
@ -160,7 +205,7 @@ class TfTabNewsFeedElement extends LitElement {
result = (await this.decrypt(result)).filter((x) => x.decrypted); result = (await this.decrypt(result)).filter((x) => x.decrypted);
} else { } else {
let t0 = new Date(); let t0 = new Date();
result = await tfrpc.rpc.query( let initial_messages = await tfrpc.rpc.query(
` `
WITH WITH
all_news AS ( all_news AS (
@ -177,56 +222,11 @@ class TfTabNewsFeedElement extends LitElement {
`, `,
[JSON.stringify(this.following), start_time, end_time] [JSON.stringify(this.following), start_time, end_time]
); );
let news_length = result.length;
let t1 = new Date(); let t1 = new Date();
let refs = await tfrpc.rpc.query( result = await this._fetch_related_messages(initial_messages);
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(result.map((x) => x.id))]
);
let t2 = new Date(); let t2 = new Date();
let related_messages = await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
`,
[JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))]
);
result = [].concat(result, related_messages);
refs = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(result.map((x) => x.id))]
);
result = [].concat(result, await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
WHERE messages.content ->> 'type' = 'vote'
`,
[JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))]
));
let t3 = new Date();
console.log( console.log(
`load of ${result.length} rows took ${(t3 - t0) / 1000} (${(t1 - t0) / 1000} to find ${news_length} messages, ${(t2 - t1) / 1000} to find ${refs.length} refs, and ${(t3 - t2) / 1000} to find ${related_messages.length} related messages) following=${this.following.length} st=${start_time} et=${end_time}` `load of ${result.length} rows took ${(t2 - t0) / 1000} (${(t1 - t0) / 1000} to find ${initial_messages.length} initial messages, ${(t2 - t1) / 1000} to find ${result.length} total messages) following=${this.following.length} st=${start_time} et=${end_time}`
); );
} }
this.time_loading = undefined; this.time_loading = undefined;

3
src/ssb.db.c
View File

@ -183,12 +183,13 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_timestamp_index ON messages (content ->> 'type', timestamp)"); _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_timestamp_index ON messages (content ->> 'type', timestamp)");
_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_size_by_author_index ON messages (author, length(content))"); _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_size_by_author_index ON messages (author, length(content))");
_tf_ssb_db_exec( _tf_ssb_db_exec(
db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_root_rowid_index ON messages (author, content ->> 'type', content ->> 'channel', content ->> 'root')"); db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_root_timestamp_index ON messages (author, timestamp, content ->> 'type', content ->> 'channel', content ->> 'root')");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_author_id_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_author_id_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_by_author_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_by_author_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_timestamp_author_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_timestamp_author_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_id_index"); _tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_id_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_root_rowid_index");
_tf_ssb_db_exec(db, _tf_ssb_db_exec(db,
"CREATE TABLE IF NOT EXISTS blobs (" "CREATE TABLE IF NOT EXISTS blobs ("
" id TEXT PRIMARY KEY," " id TEXT PRIMARY KEY,"