Convert messages.content to JSONB. This is a very disruptive change.

apps/journal.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "📝",
-	"previous": "&2hdIDbBrAg63T2X1MzdGSF7yiqHvlnfF0PnInQLp0DA=.sha256"
+	"previous": "&b//KqE4Vx6kOSBRODK1p/8wjOLKZJ+CBB5IkaBt5YsM=.sha256"
 }

apps/sneaker.json

@@ -1,4 +1,5 @@
 {
 	"type": "tildefriends-app",
-	"emoji": "👟"
+	"emoji": "👟",
+	"previous": "&zhV2BKLLZ6aG3HsVyRTs9ESLxE2lb0e7TDE7PobnyNY=.sha256"
 }

apps/ssb.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🐌",
-	"previous": "&DUxMMCJcuhm6S9jg/eKgEyWodkITu6Tg9g5I5wgLWFU=.sha256"
+	"previous": "&p3OKuQTktIRVTG1IT2PCPpkfkI/36G4JSTBAi5u24IU=.sha256"
 }

apps/ssb/tf-app.js

@@ -158,7 +158,7 @@ class TfElement extends LitElement {
 	async fetch_new_message(id) {
 		let messages = await tfrpc.rpc.query(
 			`
-				SELECT messages.*
+				SELECT id, previous, author, sequence, timestamp, hash, json(content), signature
 				FROM messages
 				JOIN json_each(?) AS following ON messages.author = following.value
 				WHERE messages.id = ?
@@ -221,7 +221,7 @@ class TfElement extends LitElement {
 		this.tags = await tfrpc.rpc.query(
 			`
 			WITH
-				recent AS (SELECT id, content FROM messages
+				recent AS (SELECT id, json(content) AS content FROM messages
 					WHERE messages.timestamp > ? AND json_extract(content, '$.type') = 'post'
 					ORDER BY timestamp DESC LIMIT 1024),
 				recent_channels AS (SELECT recent.id, '#' || json_extract(content, '$.channel') AS tag

apps/ssb/tf-tab-news-feed.js

@@ -33,12 +33,12 @@ class TfTabNewsFeedElement extends LitElement {
 		if (this.hash.startsWith('#@')) {
 			let r = await tfrpc.rpc.query(
 				`
-					WITH mine AS (SELECT messages.*
+					WITH mine AS (SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 						FROM messages
 						WHERE messages.author = ?
 						ORDER BY sequence DESC
 						LIMIT 20)
-					SELECT messages.*
+					SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 						FROM mine
 						JOIN messages_refs ON mine.id = messages_refs.ref
 						JOIN messages ON messages_refs.message = messages.id
@@ -51,11 +51,11 @@ class TfTabNewsFeedElement extends LitElement {
 		} else if (this.hash.startsWith('#%')) {
 			return await tfrpc.rpc.query(
 				`
-					SELECT messages.*
+					SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 					FROM messages
 					WHERE id = ?1
 					UNION
-					SELECT messages.*
+					SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 					FROM messages JOIN messages_refs
 					ON messages.id = messages_refs.message
 					WHERE messages_refs.ref = ?1
@@ -69,17 +69,17 @@ class TfTabNewsFeedElement extends LitElement {
 				promises.push(
 					tfrpc.rpc.query(
 						`
-						WITH news AS (SELECT messages.*
+						WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 						FROM messages
 						JOIN json_each(?) AS following ON messages.author = following.value
 						WHERE messages.timestamp > ? AND messages.timestamp < ?
 						ORDER BY messages.timestamp DESC)
-						SELECT messages.*
+						SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 							FROM news
 							JOIN messages_refs ON news.id = messages_refs.ref
 							JOIN messages ON messages_refs.message = messages.id
 						UNION
-						SELECT messages.*
+						SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 							FROM news
 							JOIN messages_refs ON news.id = messages_refs.message
 							JOIN messages ON messages_refs.ref = messages.id
@@ -107,18 +107,18 @@ class TfTabNewsFeedElement extends LitElement {
 		this.start_time = last_start_time - 24 * 60 * 60 * 1000;
 		let more = await tfrpc.rpc.query(
 			`
-				WITH news AS (SELECT messages.*
+				WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 				FROM messages
 				JOIN json_each(?) AS following ON messages.author = following.value
 				WHERE messages.timestamp > ?
 				AND messages.timestamp <= ?
 				ORDER BY messages.timestamp DESC)
-				SELECT messages.*
+				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 					FROM news
 					JOIN messages_refs ON news.id = messages_refs.ref
 					JOIN messages ON messages_refs.message = messages.id
 				UNION
-				SELECT messages.*
+				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 					FROM news
 					JOIN messages_refs ON news.id = messages_refs.message
 					JOIN messages ON messages_refs.ref = messages.id

apps/wiki.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "📝",
-	"previous": "&/wl8HE2jZShRXTYEVYRrK3pjHwi41Wbxl9HoSJaQP6Y=.sha256"
+	"previous": "&DnfuAUGzzalSh9NgZXnzDc9Ru5aM0omfRJ4h27jYw4k=.sha256"
 }

src/ssb.db.c

@@ -101,11 +101,28 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
 		"  timestamp REAL,"
 		"  previous TEXT,"
 		"  hash TEXT,"
-		"  content TEXT,"
+		"  content BLOB,"
 		"  signature TEXT,"
 		"  sequence_before_author INTEGER,"
 		"  UNIQUE(author, sequence)"
 		")");
+
+	if (_tf_ssb_db_has_rows(db, "SELECT name FROM pragma_table_info('messages') WHERE name = 'content' AND type == 'TEXT'"))
+	{
+		tf_printf("converting to JSONB\n");
+		_tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ai_refs");
+		_tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ad_refs");
+		_tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ai");
+		_tf_ssb_db_exec(db, "DROP TRIGGER IF EXISTS messages_ad");
+		_tf_ssb_db_exec(db, "DROP TABLE IF EXISTS messages_fts");
+		_tf_ssb_db_exec(db, "BEGIN TRANSACTION");
+		_tf_ssb_db_exec(db, "ALTER TABLE messages ADD COLUMN contentb BLOB");
+		_tf_ssb_db_exec(db, "UPDATE messages SET contentb = jsonb(content)");
+		_tf_ssb_db_exec(db, "ALTER TABLE messages DROP COLUMN content");
+		_tf_ssb_db_exec(db, "ALTER TABLE messages RENAME COLUMN contentb TO content");
+		_tf_ssb_db_exec(db, "COMMIT TRANSACTION");
+	}
+
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_id_index ON messages (author, id)");
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_sequence_index ON messages (author, sequence)");
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_timestamp_index ON messages (author, timestamp)");
@@ -287,7 +304,7 @@ static int64_t _tf_ssb_db_store_message_raw(tf_ssb_t* ssb, const char* id, const
 
 	if (_tf_ssb_db_previous_message_exists(db, author, sequence, previous))
 	{
-		const char* query = "INSERT INTO messages (id, previous, author, sequence, timestamp, content, hash, signature, sequence_before_author) VALUES (?, ?, ?, ?, ?, ?, "
+		const char* query = "INSERT INTO messages (id, previous, author, sequence, timestamp, content, hash, signature, sequence_before_author) VALUES (?, ?, ?, ?, ?, jsonb(?), "
 							"?, ?, ?) ON CONFLICT DO NOTHING";
 		sqlite3_stmt* statement;
 		if (sqlite3_prepare(db, query, -1, &statement, NULL) == SQLITE_OK)
@@ -557,7 +574,7 @@ bool tf_ssb_db_message_content_get(tf_ssb_t* ssb, const char* id, uint8_t** out_
 	bool result = false;
 	sqlite3_stmt* statement;
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
-	const char* query = "SELECT content FROM messages WHERE id = ?";
+	const char* query = "SELECT json(content) FROM messages WHERE id = ?";
 	if (sqlite3_prepare(db, query, -1, &statement, NULL) == SQLITE_OK)
 	{
 		if (sqlite3_bind_text(statement, 1, id, -1, NULL) == SQLITE_OK && sqlite3_step(statement) == SQLITE_ROW)
@@ -764,7 +781,7 @@ bool tf_ssb_db_get_message_by_author_and_sequence(
 {
 	bool found = false;
 	sqlite3_stmt* statement;
-	const char* query = "SELECT id, timestamp, content FROM messages WHERE author = ?1 AND sequence = ?2";
+	const char* query = "SELECT id, timestamp, json(content) FROM messages WHERE author = ?1 AND sequence = ?2";
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
 	if (sqlite3_prepare(db, query, -1, &statement, NULL) == SQLITE_OK)
 	{
@@ -1481,8 +1498,8 @@ JSValue tf_ssb_db_get_message_by_id(tf_ssb_t* ssb, const char* id, bool is_keys)
 	JSContext* context = tf_ssb_get_context(ssb);
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
 	sqlite3_stmt* statement;
-	if (sqlite3_prepare(
+	if (sqlite3_prepare(db, "SELECT previous, author, id, sequence, timestamp, hash, json(content), signature, sequence_before_author FROM messages WHERE id = ?", -1, &statement,
-			db, "SELECT previous, author, id, sequence, timestamp, hash, content, signature, sequence_before_author FROM messages WHERE id = ?", -1, &statement, NULL) == SQLITE_OK)
+			NULL) == SQLITE_OK)
 	{
 		if (sqlite3_bind_text(statement, 1, id, -1, NULL) == SQLITE_OK)
 		{

src/ssb.rpc.c

@@ -678,7 +678,7 @@ static void _tf_ssb_connection_send_history_stream_work(tf_ssb_connection_t* con
 	sqlite3_stmt* statement;
 	const int k_max = 32;
 	if (sqlite3_prepare(db,
-			"SELECT previous, author, id, sequence, timestamp, hash, content, signature, sequence_before_author FROM messages WHERE author = ?1 AND sequence > ?2 AND "
+			"SELECT previous, author, id, sequence, timestamp, hash, json(content), signature, sequence_before_author FROM messages WHERE author = ?1 AND sequence > ?2 AND "
 			"sequence "
 			"< ?3 ORDER BY sequence",
 			-1, &statement, NULL) == SQLITE_OK)