Message IDs are apparently generated from the latin1 encoding of a message. Added a command to check/fix that.

git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@3833 ed5197a5-7fde-0310-b194-c3ffbd925b24

src/main.c

@@ -1,4 +1,6 @@
 #include "ssb.h"
+#include "ssb.db.h"
+#include "ssb.import.h"
 #include "ssb.import.h"
 #include "ssb.export.h"
 #include "task.h"
@@ -57,6 +59,7 @@ static int _tf_command_export(const char* file, int argc, char* argv[]);
 static int _tf_command_run(const char* file, int argc, char* argv[]);
 static int _tf_command_sandbox(const char* file, int argc, char* argv[]);
 static int _tf_command_post(const char* file, int argc, char* argv[]);
+static int _tf_command_check(const char* file, int argc, char* argv[]);
 static int _tf_command_usage(const char* file, int argc, char* argv[]);
 
 typedef struct _command_t {
@@ -72,6 +75,7 @@ const command_t k_commands[] = {
 	{ "import", _tf_command_import, "Import apps to SSB." },
 	{ "export", _tf_command_export, "Export apps from SSB." },
 	{ "test", _tf_command_test, "Test SSB." },
+	{ "check", _tf_command_check, "Validate messages in the SSB database." },
 };
 
 void shedPrivileges()
@@ -549,6 +553,46 @@ xopt_help:
 	return 1;
 }
 
+static int _tf_command_check(const char* file, int argc, char* argv[])
+{
+	typedef struct args_t {
+		bool help;
+	} args_t;
+
+	xoptOption options[] = {
+		{ "help", 'h', offsetof(args_t, help), NULL, XOPT_TYPE_BOOL, NULL, "Shows this help message." },
+		XOPT_NULLOPTION,
+	};
+
+	args_t args = { 0 };
+	const char** extras = NULL;
+	int extra_count = 0;
+	const char *err = NULL;
+	XOPT_PARSE(file, XOPT_CTX_KEEPFIRST | XOPT_CTX_STRICT, options, &args, argc, (const char**)argv, &extra_count, &extras, &err, stderr, "post [options]", "options:", NULL, 15);
+	if (extras)
+	{
+		free((void*)extras);
+	}
+	if (err)
+	{
+		fprintf(stderr, "Error: %s\n", err);
+		return 2;
+	}
+
+	sqlite3* db = NULL;
+	sqlite3_open("db.sqlite", &db);
+	bool result = tf_ssb_db_check(db);
+	sqlite3_close(db);
+	return result ? EXIT_SUCCESS : EXIT_FAILURE;
+
+xopt_help:
+	if (extras)
+	{
+		free((void*)extras);
+	}
+	return 1;
+}
+
 static int _tf_command_usage(const char* file, int argc, char* argv[])
 {
 	printf("Usage: %s command [command-options]\n", file);

src/ssb.c

@@ -477,26 +477,76 @@ void tf_ssb_connection_rpc_send(tf_ssb_connection_t* connection, uint8_t flags,
 	connection->ssb->rpc_out++;
 }
 
+static int _utf8_len(uint8_t ch)
+{
+	static const uint8_t k_length[] =
+	{
+		1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4
+	};
+	return k_length[(ch & 0xf0) >> 4];
+}
+
+static uint32_t _utf8_decode(uint32_t c)
+{
+	if (c > 0x7f)
+	{
+		uint32_t mask = (c <= 0x00efbfbf) ? 0x000f0000 : 0x003f0000;
+		c = ((c & 0x07000000) >> 6) |
+			((c & mask) >> 4) |
+			((c & 0x00003f00) >> 2) |
+			(c & 0x0000003f);
+	}
+	return c;
+}
+
+static const uint8_t* _utf8_to_cp(const uint8_t* ch, uint32_t* out_cp)
+{
+	int len = _utf8_len(*ch);
+	int actual_len = 0;
+	uint32_t encoding = 0;
+	for (int i = 0; i < len && ch[i]; i++, actual_len++)
+	{
+		encoding = (encoding << 8) | ch[i];
+	}
+	*out_cp = _utf8_decode(encoding);
+	return ch + actual_len;
+}
+
 void tf_ssb_calculate_message_id(JSContext* context, JSValue message, char* out_id, size_t out_id_size)
 {
 	JSValue idval = JS_JSONStringify(context, message, JS_NULL, JS_NewInt32(context, 2));
 	size_t len = 0;
 	const char* messagestr = JS_ToCStringLen(context, &len, idval);
+
+	char* latin1 = strdup(messagestr);
+	uint8_t* write_pos = (uint8_t*)latin1;
+	const uint8_t* p = (const uint8_t*)messagestr;
+	while (p && *p)
+	{
+		uint32_t cp = 0;
+		p = _utf8_to_cp(p, &cp);
+		*write_pos++ = (cp & 0xff);
+	}
+	size_t latin1_len = write_pos - (uint8_t*)latin1;
+	*write_pos++ = '\0';
+
 	uint8_t id[crypto_hash_sha256_BYTES];
-	crypto_hash_sha256(id, (uint8_t*)messagestr, len);
+	crypto_hash_sha256(id, (uint8_t*)latin1, latin1_len);
 
 	char id_base64[k_id_base64_len];
 	base64c_encode(id, sizeof(id), (uint8_t*)id_base64, sizeof(id_base64));
 
 	snprintf(out_id, out_id_size, "%%%s.sha256", id_base64);
 
+	free(latin1);
 	JS_FreeCString(context, messagestr);
 	JS_FreeValue(context, idval);
 }
 
-static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSValue val, char* out_signature, size_t out_signature_size)
+static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSValue val, char* out_id, size_t out_id_size, char* out_signature, size_t out_signature_size)
 {
 	bool verified = false;
+	tf_ssb_calculate_message_id(context, val, out_id, out_id_size);
 	JSValue signature = JS_GetPropertyStr(context, val, "signature");
 	const char* str = JS_ToCString(context, signature);
 	JSAtom sigatom = JS_NewAtom(context, "signature");
@@ -530,7 +580,7 @@ static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSVa
 			verified = r == 0;
 			if (!verified)
 			{
-				printf("crypto_sign_verify_detached fail (r=%d)\n", r);
+				//printf("crypto_sign_verify_detached fail (r=%d)\n", r);
 				if (false)
 				{
 					printf("val=[%.*s]\n", (int)strlen(sigstr), sigstr);
@@ -565,9 +615,9 @@ static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSVa
 	return verified;
 }
 
-bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author)
+bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_id, size_t out_id_size, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author)
 {
-	if (_tf_ssb_verify_and_strip_signature_internal(context, val, out_signature, out_signature_size))
+	if (_tf_ssb_verify_and_strip_signature_internal(context, val, out_id, out_id_size, out_signature, out_signature_size))
 	{
 		if (out_sequence_before_author)
 		{
@@ -585,7 +635,7 @@ bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* ou
 		JS_SetPropertyStr(context, reordered, "hash", JS_GetPropertyStr(context, val, "hash"));
 		JS_SetPropertyStr(context, reordered, "content", JS_GetPropertyStr(context, val, "content"));
 		JS_SetPropertyStr(context, reordered, "signature", JS_GetPropertyStr(context, val, "signature"));
-		bool result = _tf_ssb_verify_and_strip_signature_internal(context, reordered, out_signature, out_signature_size);
+		bool result = _tf_ssb_verify_and_strip_signature_internal(context, reordered, out_id, out_id_size, out_signature, out_signature_size);
 		JS_FreeValue(context, reordered);
 		if (result)
 		{
@@ -1220,8 +1270,7 @@ void tf_ssb_append_message(tf_ssb_t* ssb, JSValue message)
 	JS_FreeValue(context, jsonval);
 
 	char id[sodium_base64_ENCODED_LEN(crypto_hash_sha256_BYTES, sodium_base64_VARIANT_ORIGINAL) + 7 + 1];
-	tf_ssb_calculate_message_id(ssb->context, root, id, sizeof(id));
+	if (valid && tf_ssb_verify_and_strip_signature(ssb->context, root, id, sizeof(id), NULL, 0, NULL))
-	if (valid)
 	{
 		if (tf_ssb_db_store_message(ssb, ssb->context, id, root, signature_base64, false))
 		{
@@ -1232,8 +1281,7 @@ void tf_ssb_append_message(tf_ssb_t* ssb, JSValue message)
 			printf("message not stored.\n");
 		}
 	}
-
+	else
-	if (!tf_ssb_verify_and_strip_signature(ssb->context, root, NULL, 0, NULL))
 	{
 		printf("Failed to verify message signature.\n");
 	}

src/ssb.db.c

@@ -507,3 +507,101 @@ void tf_ssb_db_visit_query(tf_ssb_t* ssb, const char* query, const JSValue binds
 	}
 	sqlite3_set_authorizer(db, NULL, NULL);
 }
+
+static JSValue _tf_ssb_format_message(JSContext* context, const char* previous, const char* author, int64_t sequence, int64_t timestamp, const char* hash, const char* content, const char* signature, bool sequence_before_author)
+{
+	JSValue value = JS_NewObject(context);
+	JS_SetPropertyStr(context, value, "previous", previous ? JS_NewString(context, previous) : JS_NULL);
+	if (sequence_before_author)
+	{
+		JS_SetPropertyStr(context, value, "sequence", JS_NewInt64(context, sequence));
+		JS_SetPropertyStr(context, value, "author", JS_NewString(context, author));
+	}
+	else
+	{
+		JS_SetPropertyStr(context, value, "author", JS_NewString(context, author));
+		JS_SetPropertyStr(context, value, "sequence", JS_NewInt64(context, sequence));
+	}
+	JS_SetPropertyStr(context, value, "timestamp", JS_NewInt64(context, timestamp));
+	JS_SetPropertyStr(context, value, "hash", JS_NewString(context, hash));
+	JS_SetPropertyStr(context, value, "content", JS_ParseJSON(context, content, strlen(content), NULL));
+	JS_SetPropertyStr(context, value, "signature", JS_NewString(context, signature));
+	return value;
+}
+
+bool _tf_ssb_update_message_id(sqlite3* db, const char* old_id, const char* new_id)
+{
+	bool success = false;
+	sqlite3_stmt* statement = NULL;
+	if (sqlite3_prepare(db, "UPDATE messages SET id = ? WHERE id = ?", -1, &statement, NULL) == SQLITE_OK)
+	{
+		if (sqlite3_bind_text(statement, 1, new_id, -1, NULL) == SQLITE_OK &&
+			sqlite3_bind_text(statement, 2, old_id, -1, NULL) == SQLITE_OK)
+		{
+			success = sqlite3_step(statement) == SQLITE_DONE;
+		}
+		sqlite3_finalize(statement);
+	}
+	return success;
+}
+
+bool tf_ssb_db_check(sqlite3* db)
+{
+	JSRuntime* runtime = JS_NewRuntime();
+	JSContext* context = JS_NewContext(runtime);
+
+	sqlite3_stmt* statement = NULL;
+	if (sqlite3_prepare(db, "SELECT id, previous, author, sequence, timestamp, hash, content, signature, sequence_before_author FROM messages ORDER BY author, sequence", -1, &statement, NULL) == SQLITE_OK)
+	{
+		char previous_id[k_id_base64_len];
+		while (sqlite3_step(statement) == SQLITE_ROW)
+		{
+			const char* id = (const char*)sqlite3_column_text(statement, 0);
+			const char* previous = (const char*)sqlite3_column_text(statement, 1);
+			const char* author = (const char*)sqlite3_column_text(statement, 2);
+			int64_t sequence = sqlite3_column_int64(statement, 3);
+			int64_t timestamp = sqlite3_column_int64(statement, 4);
+			const char* hash = (const char*)sqlite3_column_text(statement, 5);
+			const char* content = (const char*)sqlite3_column_text(statement, 6);
+			const char* signature = (const char*)sqlite3_column_text(statement, 7);
+			bool sequence_before_author = sqlite3_column_int(statement, 8);
+			JSValue message = _tf_ssb_format_message(context, previous, author, sequence, timestamp, hash, content, signature, sequence_before_author);
+			char out_signature[512];
+			char actual_id[k_id_base64_len];
+			bool actual_sequence_before_author = false;
+			JSValue j = JS_JSONStringify(context, message, JS_NULL, JS_NewInt32(context, 2));
+			const char* jv = JS_ToCString(context, j);
+			if (tf_ssb_verify_and_strip_signature(context, message, actual_id, sizeof(actual_id), out_signature, sizeof(out_signature), &actual_sequence_before_author))
+			{
+				/*if (previous && strcmp(previous, previous_id))
+				{
+					printf("%s:%d previous was %s should be %s\n", id, (int)sequence, previous_id, previous);
+				}*/
+				if (strcmp(id, actual_id))
+				{
+					if (_tf_ssb_update_message_id(db, id, actual_id))
+					{
+						printf("updated %s to %s\n", id, actual_id);
+					}
+					else
+					{
+						printf("failed to update %s to %s\n", id, actual_id);
+					}
+				}
+			}
+			else
+			{
+				printf("unable to verify signature for %s\n", id);
+			}
+			JS_FreeCString(context, jv);
+			JS_FreeValue(context, j);
+			snprintf(previous_id, sizeof(previous_id), "%s", id);
+			JS_FreeValue(context, message);
+		}
+		sqlite3_finalize(statement);
+	}
+
+	JS_FreeContext(context);
+	JS_FreeRuntime(runtime);
+	return false;
+}

src/ssb.db.h

@@ -14,3 +14,6 @@ bool tf_ssb_db_blob_store(tf_ssb_t* ssb, const uint8_t* blob, size_t size, char*
 bool tf_ssb_db_get_message_by_author_and_sequence(tf_ssb_t* ssb, const char* author, int64_t sequence, char* out_message_id, size_t out_message_id_size, int64_t* out_timestamp, char** out_content);
 bool tf_ssb_db_get_latest_message_by_author(tf_ssb_t* ssb, const char* author, int64_t* out_sequence, char* out_message_id, size_t out_message_id_size);
 void tf_ssb_db_visit_query(tf_ssb_t* ssb, const char* query, const JSValue binds, void (*callback)(JSValue row, void* user_data), void* user_data);
+
+typedef struct sqlite3 sqlite3;
+bool tf_ssb_db_check(sqlite3* db);

src/ssb.h

@@ -89,7 +89,7 @@ void tf_ssb_send_close(tf_ssb_t* ssb);
 bool tf_ssb_id_str_to_bin(uint8_t* bin, const char* str);
 bool tf_ssb_id_bin_to_str(char* str, size_t str_size, const uint8_t* bin);
 
-bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author);
+bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_id, size_t out_id_size, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author);
 void tf_ssb_calculate_message_id(JSContext* context, JSValue message, char* out_id, size_t out_id_size);
 
 const char* tf_ssb_connection_get_host(tf_ssb_connection_t* connection);

src/ssb.js.c

@@ -224,9 +224,8 @@ static JSValue _tf_ssb_storeMessage(JSContext* context, JSValueConst this_val, i
 	tf_ssb_t* ssb = JS_GetOpaque(this_val, _tf_ssb_classId);
 	char signature[crypto_sign_BYTES + 128];
 	char id[crypto_hash_sha256_BYTES * 2 + 1];
-	tf_ssb_calculate_message_id(context, argv[0], id, sizeof(id));
 	bool sequence_before_author = false;
-	if (tf_ssb_verify_and_strip_signature(context, argv[0], signature, sizeof(signature), &sequence_before_author))
+	if (tf_ssb_verify_and_strip_signature(context, argv[0], id, sizeof(id), signature, sizeof(signature), &sequence_before_author))
 	{
 		if (tf_ssb_db_store_message(ssb, context, id, argv[0], signature, sequence_before_author))
 		{