cory/tildefriends
5
6
Fork 2
Code Issues 35 Pull Requests Actions Releases 31 Wiki Activity

ssb: Faster channel loads.

Browse Source
This commit is contained in:
Cory McWilliams 2025-04-09 18:50:14 -04:00
parent ba2bb17638
commit 7f252e79b6
3 changed files with 63 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/ssb.json
View File

@ -1,5 +1,5 @@
{
"type": "tildefriends-app",
"emoji": "🦀",
"previous": "&shP+cVoCoktYoePzAG2wj1vCx/eGXZPnoP1MDHT3d6g=.sha256"
"previous": "&r6gBoGYTO1yQXHLBPut0+iNd3NNFKPO/LXNV+F5+46M=.sha256"
}

120
apps/ssb/tf-tab-news-feed.js
View File

@ -46,6 +46,53 @@ class TfTabNewsFeedElement extends LitElement {
: this.hash.substring(1);
}
async _fetch_related_messages(messages) {
let refs = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(messages.map((x) => x.id))]
);
let related_messages = await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
`,
[JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))]
);
let combined = [].concat(messages, related_messages);
let refs2 = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(combined.map((x) => x.id))]
);
return [].concat(combined, await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
WHERE messages.content ->> 'type' = 'vote'
`,
[JSON.stringify(this.following), JSON.stringify(refs2.map((x) => x.ref))]
));
}
async fetch_messages(start_time, end_time) {
this.time_loading = [start_time, end_time];
let result;
@ -107,7 +154,8 @@ class TfTabNewsFeedElement extends LitElement {
[this.hash.substring(1)]
);
} else if (this.hash.startsWith('##')) {
result = await tfrpc.rpc.query(
let t0 = new Date();
let initial_messages = await tfrpc.rpc.query(
`
WITH
all_news AS (
@ -120,20 +168,11 @@ class TfTabNewsFeedElement extends LitElement {
FROM messages_fts(?5)
JOIN messages ON messages.rowid = messages_fts.rowid
JOIN json_each(?1) AS following ON messages.author = following.value
JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4),
JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4
),
news AS (SELECT * FROM all_news
WHERE (?2 IS NULL OR all_news.timestamp >= ?2) AND all_news.timestamp < ?3
ORDER BY all_news.timestamp DESC LIMIT 20)
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM news
JOIN messages_refs ON news.id = messages_refs.ref
JOIN messages ON messages_refs.message = messages.id
UNION
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM news
JOIN messages_refs ON news.id = messages_refs.message
JOIN messages ON messages_refs.ref = messages.id
UNION
SELECT TRUE AS is_primary, news.* FROM news
`,
[
@ -144,6 +183,12 @@ class TfTabNewsFeedElement extends LitElement {
'"#' + this.hash.substring(2).replace('"', '""') + '"',
]
);
let t1 = new Date();
result = await this._fetch_related_messages(initial_messages);
let t2 = new Date();
console.log(
`load of ${result.length} rows took ${(t2 - t0) / 1000} (${(t1 - t0) / 1000} to find ${initial_messages.length} initial messages, ${(t2 - t1) / 1000} to find ${result.length} total messages) following=${this.following.length} st=${start_time} et=${end_time}`
);
} else if (this.hash == '#🔐') {
result = await tfrpc.rpc.query(
`
@ -160,7 +205,7 @@ class TfTabNewsFeedElement extends LitElement {
result = (await this.decrypt(result)).filter((x) => x.decrypted);
} else {
let t0 = new Date();
result = await tfrpc.rpc.query(
let initial_messages = await tfrpc.rpc.query(
`
WITH
all_news AS (
@ -177,56 +222,11 @@ class TfTabNewsFeedElement extends LitElement {
`,
[JSON.stringify(this.following), start_time, end_time]
);
let news_length = result.length;
let t1 = new Date();
let refs = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(result.map((x) => x.id))]
);
result = await this._fetch_related_messages(initial_messages);
let t2 = new Date();
let related_messages = await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
`,
[JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))]
);
result = [].concat(result, related_messages);
refs = await tfrpc.rpc.query(
`
WITH
news AS (
SELECT value AS id FROM json_each(?)
)
SELECT refs_out.ref AS ref FROM messages_refs refs_out JOIN news ON refs_out.message = news.id
UNION
SELECT refs_in.message AS ref FROM messages_refs refs_in JOIN news ON refs_in.ref = news.id
`,
[JSON.stringify(result.map((x) => x.id))]
);
result = [].concat(result, await tfrpc.rpc.query(
`
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages
JOIN json_each(?2) refs ON messages.id = refs.value
JOIN json_each(?1) AS following ON messages.author = following.value
WHERE messages.content ->> 'type' = 'vote'
`,
[JSON.stringify(this.following), JSON.stringify(refs.map((x) => x.ref))]
));
let t3 = new Date();
console.log(
`load of ${result.length} rows took ${(t3 - t0) / 1000} (${(t1 - t0) / 1000} to find ${news_length} messages, ${(t2 - t1) / 1000} to find ${refs.length} refs, and ${(t3 - t2) / 1000} to find ${related_messages.length} related messages) following=${this.following.length} st=${start_time} et=${end_time}`
`load of ${result.length} rows took ${(t2 - t0) / 1000} (${(t1 - t0) / 1000} to find ${initial_messages.length} initial messages, ${(t2 - t1) / 1000} to find ${result.length} total messages) following=${this.following.length} st=${start_time} et=${end_time}`
);
}
this.time_loading = undefined;

3
src/ssb.db.c
View File

@ -183,12 +183,13 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_timestamp_index ON messages (content ->> 'type', timestamp)");
_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_size_by_author_index ON messages (author, length(content))");
_tf_ssb_db_exec(
db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_root_rowid_index ON messages (author, content ->> 'type', content ->> 'channel', content ->> 'root')");
db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_root_timestamp_index ON messages (author, timestamp, content ->> 'type', content ->> 'channel', content ->> 'root')");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_author_id_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_by_author_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_timestamp_author_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_id_index");
_tf_ssb_db_exec(db, "DROP INDEX IF EXISTS messages_type_author_channel_root_rowid_index");
_tf_ssb_db_exec(db,
"CREATE TABLE IF NOT EXISTS blobs ("
" id TEXT PRIMARY KEY,"