cory/tildefriends
5
6
Fork 2
Code Issues34 Pull Requests Actions Releases30 Wiki Activity

ssb: I made blob_wants_view too slow. Let's make a real table updated by triggers.

Browse Source
This commit is contained in:
Cory McWilliams 2025-03-12 18:17:11 -04:00
parent 98b4c7cf04
commit 3660577a23
2 changed files with 38 additions and 10 deletions

46
src/ssb.db.c

@ -290,28 +290,56 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
_tf_ssb_db_exec(db, "DROP VIEW IF EXISTS blob_wants_view"); _tf_ssb_db_exec(db, "DROP VIEW IF EXISTS blob_wants_view");
_tf_ssb_db_exec(db, _tf_ssb_db_exec(db,
"CREATE VIEW IF NOT EXISTS blob_wants_view (id, timestamp) AS " "CREATE VIEW IF NOT EXISTS blob_wants_view (source, id, timestamp) AS "
" WITH RECURSIVE " " WITH RECURSIVE "
" wanted1 AS ( " " wanted1 AS ( "
" SELECT messages_refs.ref AS id, messages.timestamp AS timestamp " " SELECT messages_refs.message AS source, messages_refs.ref AS id, messages.timestamp AS timestamp "
" FROM messages_refs " " FROM messages_refs "
" JOIN messages ON messages.id = messages_refs.message " " JOIN messages ON messages.id = messages_refs.message "
" UNION " " UNION "
" SELECT messages_refs.ref AS id, unixepoch() * 1000 AS timestamp " " SELECT messages_refs.message AS source, messages_refs.ref AS id, unixepoch() * 1000 AS timestamp "
" FROM messages_refs " " FROM messages_refs "
" JOIN messages ON messages.id = messages_refs.message " " JOIN messages ON messages.id = messages_refs.message "
" WHERE messages.content ->> 'type' = 'about' " " WHERE messages.content ->> 'type' = 'about' "
" ), " " ), "
" wanted(id, timestamp) AS ( " " wanted(source, id, timestamp) AS ( "
" SELECT wanted1.id AS id, wanted1.timestamp AS timestamp FROM wanted1 " " SELECT wanted1.source AS source, wanted1.id AS id, wanted1.timestamp AS timestamp FROM wanted1 "
" UNION " " UNION "
" SELECT br.ref AS id, wanted.timestamp AS timestamp FROM wanted JOIN blobs_refs br ON br.blob = wanted.id " " SELECT wanted.source AS source, br.ref AS id, wanted.timestamp AS timestamp FROM wanted JOIN blobs_refs br ON br.blob = wanted.id "
" ) " " ) "
" SELECT wanted.id, wanted.timestamp FROM wanted " " SELECT wanted.source, wanted.id, wanted.timestamp FROM wanted "
" LEFT OUTER JOIN blobs ON wanted.id = blobs.id " " LEFT OUTER JOIN blobs ON wanted.id = blobs.id "
" WHERE blobs.id IS NULL " " WHERE blobs.id IS NULL "
" AND LENGTH(wanted.id) = 52 " " AND LENGTH(wanted.id) = 52 "
" AND wanted.id LIKE '&%.sha256'"); " AND wanted.id LIKE '&%.sha256'");
if (!_tf_ssb_db_has_rows(db, "PRAGMA table_list('blob_wants_cache')"))
{
tf_printf("Populating blob_wants_cache...\n");
_tf_ssb_db_exec(db, "BEGIN TRANSACTION");
_tf_ssb_db_exec(db, "CREATE TABLE IF NOT EXISTS blob_wants_cache (source TEXT, id TEXT, timestamp REAL, UNIQUE(source, id))");
_tf_ssb_db_exec(
db, "INSERT INTO blob_wants_cache SELECT * FROM blob_wants_view WHERE true ON CONFLICT(source, id) DO UPDATE SET timestamp = MAX(timestamp, excluded.timestamp)");
_tf_ssb_db_exec(db, "COMMIT TRANSACTION");
tf_printf("Done.\n");
}
_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS blob_wants_cache_id_idx ON blob_wants_cache (id)");
_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS blob_wants_cache_timestamp_id_idx ON blob_wants_cache (timestamp, id)");
_tf_ssb_db_exec(db,
"CREATE TRIGGER IF NOT EXISTS messages_ai_blob_wants_cache AFTER INSERT ON messages_refs BEGIN "
"INSERT INTO blob_wants_cache (source, id, timestamp) "
"SELECT messages.id, new.ref, messages.timestamp FROM messages WHERE messages.id = new.message AND "
"LENGTH(new.ref) = 52 AND new.ref LIKE '&%.sha256' "
"ON CONFLICT (source, id) DO NOTHING; END");
_tf_ssb_db_exec(db,
"CREATE TRIGGER IF NOT EXISTS blobs_ai_blob_wants_cache AFTER INSERT ON blobs_refs BEGIN "
"INSERT INTO blob_wants_cache (source, id, timestamp) "
"SELECT messages.id, new.ref, messages.timestamp FROM messages "
"JOIN blob_wants_cache bwc ON bwc.source = messages.id AND bwc.id = new.blob "
"ON CONFLICT (source, id) DO NOTHING; END");
_tf_ssb_db_exec(db,
"CREATE TRIGGER IF NOT EXISTS messages_ad_blob_wants_cache AFTER DELETE ON messages BEGIN "
"DELETE FROM blob_wants_cache WHERE blob_wants_cache.source = old.id; END");
bool need_add_flags = true; bool need_add_flags = true;
bool need_convert_timestamp_to_real = false; bool need_convert_timestamp_to_real = false;
@ -1039,8 +1067,8 @@ int tf_ssb_sqlite_authorizer(void* user_data, int action_code, const char* arg0,
result = SQLITE_OK; result = SQLITE_OK;
break; break;
case SQLITE_READ: case SQLITE_READ:
result = (strcmp(arg0, "blob_wants_view") == 0 || strcmp(arg0, "json_each") == 0 || strcmp(arg0, "json_tree") == 0 || strcmp(arg0, "messages") == 0 || result = (strcmp(arg0, "blob_wants_cache") == 0 || strcmp(arg0, "blob_wants_view") == 0 || strcmp(arg0, "json_each") == 0 || strcmp(arg0, "json_tree") == 0 ||
strcmp(arg0, "messages_stats") == 0 || strcmp(arg0, "messages_fts") == 0 || strcmp(arg0, "messages_fts_idx") == 0 || strcmp(arg0, "messages") == 0 || strcmp(arg0, "messages_stats") == 0 || strcmp(arg0, "messages_fts") == 0 || strcmp(arg0, "messages_fts_idx") == 0 ||
strcmp(arg0, "messages_fts_config") == 0 || strcmp(arg0, "messages_refs") == 0 || strcmp(arg0, "messages_refs_message_idx") == 0 || strcmp(arg0, "messages_fts_config") == 0 || strcmp(arg0, "messages_refs") == 0 || strcmp(arg0, "messages_refs_message_idx") == 0 ||
strcmp(arg0, "messages_refs_ref_idx") == 0 || strcmp(arg0, "sqlite_master") == 0 || false) strcmp(arg0, "messages_refs_ref_idx") == 0 || strcmp(arg0, "sqlite_master") == 0 || false)
? SQLITE_OK ? SQLITE_OK

2
src/ssb.rpc.c

@ -232,7 +232,7 @@ static void _tf_ssb_request_blob_wants_work(tf_ssb_connection_t* connection, voi
db = tf_ssb_acquire_db_reader(ssb); db = tf_ssb_acquire_db_reader(ssb);
sqlite3_stmt* statement; sqlite3_stmt* statement;
if (sqlite3_prepare(db, "SELECT id FROM blob_wants_view WHERE id > ? AND timestamp > ? ORDER BY id LIMIT ?", -1, &statement, NULL) == SQLITE_OK) if (sqlite3_prepare(db, "SELECT id FROM blob_wants_cache WHERE id > ? AND timestamp > ? ORDER BY id LIMIT ?", -1, &statement, NULL) == SQLITE_OK)
{ {
if (sqlite3_bind_text(statement, 1, blob_wants->last_id, -1, NULL) == SQLITE_OK && sqlite3_bind_int64(statement, 2, timestamp) == SQLITE_OK && if (sqlite3_bind_text(statement, 1, blob_wants->last_id, -1, NULL) == SQLITE_OK && sqlite3_bind_int64(statement, 2, timestamp) == SQLITE_OK &&
sqlite3_bind_int(statement, 3, tf_countof(work->out_id)) == SQLITE_OK) sqlite3_bind_int(statement, 3, tf_countof(work->out_id)) == SQLITE_OK)