From 591642efb3b26f3c16b2eca09b209366cdf2ecf6 Mon Sep 17 00:00:00 2001 From: Cory McWilliams Date: Wed, 28 Feb 2024 20:01:52 -0500 Subject: [PATCH] Convert messages.content to JSONB. This is a very disruptive change. --- apps/journal.json | 2 +- apps/sneaker.json | 3 ++- apps/ssb.json | 2 +- apps/ssb/tf-app.js | 4 ++-- apps/ssb/tf-tab-news-feed.js | 20 ++++++++++---------- apps/wiki.json | 2 +- src/ssb.db.c | 29 +++++++++++++++++++++++------ src/ssb.rpc.c | 2 +- 8 files changed, 41 insertions(+), 23 deletions(-) diff --git a/apps/journal.json b/apps/journal.json index ec6540f3..4aa00c25 100644 --- a/apps/journal.json +++ b/apps/journal.json @@ -1,5 +1,5 @@ { "type": "tildefriends-app", "emoji": "📝", - "previous": "&2hdIDbBrAg63T2X1MzdGSF7yiqHvlnfF0PnInQLp0DA=.sha256" + "previous": "&b//KqE4Vx6kOSBRODK1p/8wjOLKZJ+CBB5IkaBt5YsM=.sha256" } diff --git a/apps/sneaker.json b/apps/sneaker.json index a22bb953..20ab7890 100644 --- a/apps/sneaker.json +++ b/apps/sneaker.json @@ -1,4 +1,5 @@ { "type": "tildefriends-app", - "emoji": "👟" + "emoji": "👟", + "previous": "&zhV2BKLLZ6aG3HsVyRTs9ESLxE2lb0e7TDE7PobnyNY=.sha256" } diff --git a/apps/ssb.json b/apps/ssb.json index d358695a..01b38ef9 100644 --- a/apps/ssb.json +++ b/apps/ssb.json @@ -1,5 +1,5 @@ { "type": "tildefriends-app", "emoji": "🐌", - "previous": "&DUxMMCJcuhm6S9jg/eKgEyWodkITu6Tg9g5I5wgLWFU=.sha256" + "previous": "&p3OKuQTktIRVTG1IT2PCPpkfkI/36G4JSTBAi5u24IU=.sha256" } diff --git a/apps/ssb/tf-app.js b/apps/ssb/tf-app.js index 244ebedf..5dcd39d1 100644 --- a/apps/ssb/tf-app.js +++ b/apps/ssb/tf-app.js @@ -158,7 +158,7 @@ class TfElement extends LitElement { async fetch_new_message(id) { let messages = await tfrpc.rpc.query( ` - SELECT messages.* + SELECT id, previous, author, sequence, timestamp, hash, json(content), signature FROM messages JOIN json_each(?) AS following ON messages.author = following.value WHERE messages.id = ? @@ -221,7 +221,7 @@ class TfElement extends LitElement { this.tags = await tfrpc.rpc.query( ` WITH - recent AS (SELECT id, content FROM messages + recent AS (SELECT id, json(content) AS content FROM messages WHERE messages.timestamp > ? AND json_extract(content, '$.type') = 'post' ORDER BY timestamp DESC LIMIT 1024), recent_channels AS (SELECT recent.id, '#' || json_extract(content, '$.channel') AS tag diff --git a/apps/ssb/tf-tab-news-feed.js b/apps/ssb/tf-tab-news-feed.js index bc7606cd..52a76650 100644 --- a/apps/ssb/tf-tab-news-feed.js +++ b/apps/ssb/tf-tab-news-feed.js @@ -33,12 +33,12 @@ class TfTabNewsFeedElement extends LitElement { if (this.hash.startsWith('#@')) { let r = await tfrpc.rpc.query( ` - WITH mine AS (SELECT messages.* + WITH mine AS (SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature FROM messages WHERE messages.author = ? ORDER BY sequence DESC LIMIT 20) - SELECT messages.* + SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM mine JOIN messages_refs ON mine.id = messages_refs.ref JOIN messages ON messages_refs.message = messages.id @@ -51,11 +51,11 @@ class TfTabNewsFeedElement extends LitElement { } else if (this.hash.startsWith('#%')) { return await tfrpc.rpc.query( ` - SELECT messages.* + SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature FROM messages WHERE id = ?1 UNION - SELECT messages.* + SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature FROM messages JOIN messages_refs ON messages.id = messages_refs.message WHERE messages_refs.ref = ?1 @@ -69,17 +69,17 @@ class TfTabNewsFeedElement extends LitElement { promises.push( tfrpc.rpc.query( ` - WITH news AS (SELECT messages.* + WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM messages JOIN json_each(?) AS following ON messages.author = following.value WHERE messages.timestamp > ? AND messages.timestamp < ? ORDER BY messages.timestamp DESC) - SELECT messages.* + SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM news JOIN messages_refs ON news.id = messages_refs.ref JOIN messages ON messages_refs.message = messages.id UNION - SELECT messages.* + SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM news JOIN messages_refs ON news.id = messages_refs.message JOIN messages ON messages_refs.ref = messages.id @@ -107,18 +107,18 @@ class TfTabNewsFeedElement extends LitElement { this.start_time = last_start_time - 24 * 60 * 60 * 1000; let more = await tfrpc.rpc.query( ` - WITH news AS (SELECT messages.* + WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM messages JOIN json_each(?) AS following ON messages.author = following.value WHERE messages.timestamp > ? AND messages.timestamp <= ? ORDER BY messages.timestamp DESC) - SELECT messages.* + SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM news JOIN messages_refs ON news.id = messages_refs.ref JOIN messages ON messages_refs.message = messages.id UNION - SELECT messages.* + SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature FROM news JOIN messages_refs ON news.id = messages_refs.message JOIN messages ON messages_refs.ref = messages.id diff --git a/apps/wiki.json b/apps/wiki.json index ba40464b..2762535f 100644 --- a/apps/wiki.json +++ b/apps/wiki.json @@ -1,5 +1,5 @@ { "type": "tildefriends-app", "emoji": "📝", - "previous": "&/wl8HE2jZShRXTYEVYRrK3pjHwi41Wbxl9HoSJaQP6Y=.sha256" + "previous": "&DnfuAUGzzalSh9NgZXnzDc9Ru5aM0omfRJ4h27jYw4k=.sha256" } diff --git a/src/ssb.db.c b/src/ssb.db.c index 36668899..3d874f4e 100644 --- a/src/ssb.db.c +++ b/src/ssb.db.c @@ -101,11 +101,28 @@ void tf_ssb_db_init(tf_ssb_t* ssb) " timestamp REAL," " previous TEXT," " hash TEXT," - " content TEXT," + " content BLOB," " signature TEXT," " sequence_before_author INTEGER," " UNIQUE(author, sequence)" ")"); + + if (_tf_ssb_db_has_rows(db, "SELECT name FROM pragma_table_info('messages') WHERE name = 'content' AND type == 'TEXT'")) + { + tf_printf("converting to JSONB\n"); + _tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ai_refs"); + _tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ad_refs"); + _tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ai"); + _tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ad"); + _tf_ssb_db_exec(db, "DROP TABLE IF EXISTS messages_fts"); + _tf_ssb_db_exec(db, "BEGIN TRANSACTION"); + _tf_ssb_db_exec(db, "ALTER TABLE messages ADD COLUMN contentb BLOB"); + _tf_ssb_db_exec(db, "UPDATE messages SET contentb = jsonb(content)"); + _tf_ssb_db_exec(db, "ALTER TABLE messages DROP COLUMN content"); + _tf_ssb_db_exec(db, "ALTER TABLE messages RENAME COLUMN contentb TO content"); + _tf_ssb_db_exec(db, "COMMIT TRANSACTION"); + } + _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_id_index ON messages (author, id)"); _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_sequence_index ON messages (author, sequence)"); _tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_timestamp_index ON messages (author, timestamp)"); @@ -287,7 +304,7 @@ static int64_t _tf_ssb_db_store_message_raw(tf_ssb_t* ssb, const char* id, const if (_tf_ssb_db_previous_message_exists(db, author, sequence, previous)) { - const char* query = "INSERT INTO messages (id, previous, author, sequence, timestamp, content, hash, signature, sequence_before_author) VALUES (?, ?, ?, ?, ?, ?, " + const char* query = "INSERT INTO messages (id, previous, author, sequence, timestamp, content, hash, signature, sequence_before_author) VALUES (?, ?, ?, ?, ?, jsonb(?), " "?, ?, ?) ON CONFLICT DO NOTHING"; sqlite3_stmt* statement; if (sqlite3_prepare(db, query, -1, &statement, NULL) == SQLITE_OK) @@ -557,7 +574,7 @@ bool tf_ssb_db_message_content_get(tf_ssb_t* ssb, const char* id, uint8_t** out_ bool result = false; sqlite3_stmt* statement; sqlite3* db = tf_ssb_acquire_db_reader(ssb); - const char* query = "SELECT content FROM messages WHERE id = ?"; + const char* query = "SELECT json(content) FROM messages WHERE id = ?"; if (sqlite3_prepare(db, query, -1, &statement, NULL) == SQLITE_OK) { if (sqlite3_bind_text(statement, 1, id, -1, NULL) == SQLITE_OK && sqlite3_step(statement) == SQLITE_ROW) @@ -764,7 +781,7 @@ bool tf_ssb_db_get_message_by_author_and_sequence( { bool found = false; sqlite3_stmt* statement; - const char* query = "SELECT id, timestamp, content FROM messages WHERE author = ?1 AND sequence = ?2"; + const char* query = "SELECT id, timestamp, json(content) FROM messages WHERE author = ?1 AND sequence = ?2"; sqlite3* db = tf_ssb_acquire_db_reader(ssb); if (sqlite3_prepare(db, query, -1, &statement, NULL) == SQLITE_OK) { @@ -1481,8 +1498,8 @@ JSValue tf_ssb_db_get_message_by_id(tf_ssb_t* ssb, const char* id, bool is_keys) JSContext* context = tf_ssb_get_context(ssb); sqlite3* db = tf_ssb_acquire_db_reader(ssb); sqlite3_stmt* statement; - if (sqlite3_prepare( - db, "SELECT previous, author, id, sequence, timestamp, hash, content, signature, sequence_before_author FROM messages WHERE id = ?", -1, &statement, NULL) == SQLITE_OK) + if (sqlite3_prepare(db, "SELECT previous, author, id, sequence, timestamp, hash, json(content), signature, sequence_before_author FROM messages WHERE id = ?", -1, &statement, + NULL) == SQLITE_OK) { if (sqlite3_bind_text(statement, 1, id, -1, NULL) == SQLITE_OK) { diff --git a/src/ssb.rpc.c b/src/ssb.rpc.c index 16fc1274..93fd56f0 100644 --- a/src/ssb.rpc.c +++ b/src/ssb.rpc.c @@ -678,7 +678,7 @@ static void _tf_ssb_connection_send_history_stream_work(tf_ssb_connection_t* con sqlite3_stmt* statement; const int k_max = 32; if (sqlite3_prepare(db, - "SELECT previous, author, id, sequence, timestamp, hash, content, signature, sequence_before_author FROM messages WHERE author = ?1 AND sequence > ?2 AND " + "SELECT previous, author, id, sequence, timestamp, hash, json(content), signature, sequence_before_author FROM messages WHERE author = ?1 AND sequence > ?2 AND " "sequence " "< ?3 ORDER BY sequence", -1, &statement, NULL) == SQLITE_OK)