From d4135f7133a6cff5a043762df1378e138c6b1b5b Mon Sep 17 00:00:00 2001 From: Cory McWilliams Date: Thu, 10 Feb 2022 03:58:33 +0000 Subject: [PATCH] Message IDs are apparently generated from the latin1 encoding of a message. Added a command to check/fix that. git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@3833 ed5197a5-7fde-0310-b194-c3ffbd925b24 --- src/main.c | 44 +++++++++++++++++++++++ src/ssb.c | 68 ++++++++++++++++++++++++++++++------ src/ssb.db.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/ssb.db.h | 3 ++ src/ssb.h | 2 +- src/ssb.js.c | 3 +- 6 files changed, 205 insertions(+), 13 deletions(-) diff --git a/src/main.c b/src/main.c index a65531ac..a5235d12 100644 --- a/src/main.c +++ b/src/main.c @@ -1,4 +1,6 @@ #include "ssb.h" +#include "ssb.db.h" +#include "ssb.import.h" #include "ssb.import.h" #include "ssb.export.h" #include "task.h" @@ -57,6 +59,7 @@ static int _tf_command_export(const char* file, int argc, char* argv[]); static int _tf_command_run(const char* file, int argc, char* argv[]); static int _tf_command_sandbox(const char* file, int argc, char* argv[]); static int _tf_command_post(const char* file, int argc, char* argv[]); +static int _tf_command_check(const char* file, int argc, char* argv[]); static int _tf_command_usage(const char* file, int argc, char* argv[]); typedef struct _command_t { @@ -72,6 +75,7 @@ const command_t k_commands[] = { { "import", _tf_command_import, "Import apps to SSB." }, { "export", _tf_command_export, "Export apps from SSB." }, { "test", _tf_command_test, "Test SSB." }, + { "check", _tf_command_check, "Validate messages in the SSB database." }, }; void shedPrivileges() @@ -549,6 +553,46 @@ xopt_help: return 1; } +static int _tf_command_check(const char* file, int argc, char* argv[]) +{ + typedef struct args_t { + bool help; + } args_t; + + xoptOption options[] = { + { "help", 'h', offsetof(args_t, help), NULL, XOPT_TYPE_BOOL, NULL, "Shows this help message." }, + XOPT_NULLOPTION, + }; + + args_t args = { 0 }; + const char** extras = NULL; + int extra_count = 0; + const char *err = NULL; + XOPT_PARSE(file, XOPT_CTX_KEEPFIRST | XOPT_CTX_STRICT, options, &args, argc, (const char**)argv, &extra_count, &extras, &err, stderr, "post [options]", "options:", NULL, 15); + if (extras) + { + free((void*)extras); + } + if (err) + { + fprintf(stderr, "Error: %s\n", err); + return 2; + } + + sqlite3* db = NULL; + sqlite3_open("db.sqlite", &db); + bool result = tf_ssb_db_check(db); + sqlite3_close(db); + return result ? EXIT_SUCCESS : EXIT_FAILURE; + +xopt_help: + if (extras) + { + free((void*)extras); + } + return 1; +} + static int _tf_command_usage(const char* file, int argc, char* argv[]) { printf("Usage: %s command [command-options]\n", file); diff --git a/src/ssb.c b/src/ssb.c index f65ea27f..d5d03565 100644 --- a/src/ssb.c +++ b/src/ssb.c @@ -477,26 +477,76 @@ void tf_ssb_connection_rpc_send(tf_ssb_connection_t* connection, uint8_t flags, connection->ssb->rpc_out++; } +static int _utf8_len(uint8_t ch) +{ + static const uint8_t k_length[] = + { + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 + }; + return k_length[(ch & 0xf0) >> 4]; +} + +static uint32_t _utf8_decode(uint32_t c) +{ + if (c > 0x7f) + { + uint32_t mask = (c <= 0x00efbfbf) ? 0x000f0000 : 0x003f0000; + c = ((c & 0x07000000) >> 6) | + ((c & mask) >> 4) | + ((c & 0x00003f00) >> 2) | + (c & 0x0000003f); + } + return c; +} + +static const uint8_t* _utf8_to_cp(const uint8_t* ch, uint32_t* out_cp) +{ + int len = _utf8_len(*ch); + int actual_len = 0; + uint32_t encoding = 0; + for (int i = 0; i < len && ch[i]; i++, actual_len++) + { + encoding = (encoding << 8) | ch[i]; + } + *out_cp = _utf8_decode(encoding); + return ch + actual_len; +} + void tf_ssb_calculate_message_id(JSContext* context, JSValue message, char* out_id, size_t out_id_size) { JSValue idval = JS_JSONStringify(context, message, JS_NULL, JS_NewInt32(context, 2)); size_t len = 0; const char* messagestr = JS_ToCStringLen(context, &len, idval); + + char* latin1 = strdup(messagestr); + uint8_t* write_pos = (uint8_t*)latin1; + const uint8_t* p = (const uint8_t*)messagestr; + while (p && *p) + { + uint32_t cp = 0; + p = _utf8_to_cp(p, &cp); + *write_pos++ = (cp & 0xff); + } + size_t latin1_len = write_pos - (uint8_t*)latin1; + *write_pos++ = '\0'; + uint8_t id[crypto_hash_sha256_BYTES]; - crypto_hash_sha256(id, (uint8_t*)messagestr, len); + crypto_hash_sha256(id, (uint8_t*)latin1, latin1_len); char id_base64[k_id_base64_len]; base64c_encode(id, sizeof(id), (uint8_t*)id_base64, sizeof(id_base64)); snprintf(out_id, out_id_size, "%%%s.sha256", id_base64); + free(latin1); JS_FreeCString(context, messagestr); JS_FreeValue(context, idval); } -static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSValue val, char* out_signature, size_t out_signature_size) +static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSValue val, char* out_id, size_t out_id_size, char* out_signature, size_t out_signature_size) { bool verified = false; + tf_ssb_calculate_message_id(context, val, out_id, out_id_size); JSValue signature = JS_GetPropertyStr(context, val, "signature"); const char* str = JS_ToCString(context, signature); JSAtom sigatom = JS_NewAtom(context, "signature"); @@ -530,7 +580,7 @@ static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSVa verified = r == 0; if (!verified) { - printf("crypto_sign_verify_detached fail (r=%d)\n", r); + //printf("crypto_sign_verify_detached fail (r=%d)\n", r); if (false) { printf("val=[%.*s]\n", (int)strlen(sigstr), sigstr); @@ -565,9 +615,9 @@ static bool _tf_ssb_verify_and_strip_signature_internal(JSContext* context, JSVa return verified; } -bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author) +bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_id, size_t out_id_size, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author) { - if (_tf_ssb_verify_and_strip_signature_internal(context, val, out_signature, out_signature_size)) + if (_tf_ssb_verify_and_strip_signature_internal(context, val, out_id, out_id_size, out_signature, out_signature_size)) { if (out_sequence_before_author) { @@ -585,7 +635,7 @@ bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* ou JS_SetPropertyStr(context, reordered, "hash", JS_GetPropertyStr(context, val, "hash")); JS_SetPropertyStr(context, reordered, "content", JS_GetPropertyStr(context, val, "content")); JS_SetPropertyStr(context, reordered, "signature", JS_GetPropertyStr(context, val, "signature")); - bool result = _tf_ssb_verify_and_strip_signature_internal(context, reordered, out_signature, out_signature_size); + bool result = _tf_ssb_verify_and_strip_signature_internal(context, reordered, out_id, out_id_size, out_signature, out_signature_size); JS_FreeValue(context, reordered); if (result) { @@ -1220,8 +1270,7 @@ void tf_ssb_append_message(tf_ssb_t* ssb, JSValue message) JS_FreeValue(context, jsonval); char id[sodium_base64_ENCODED_LEN(crypto_hash_sha256_BYTES, sodium_base64_VARIANT_ORIGINAL) + 7 + 1]; - tf_ssb_calculate_message_id(ssb->context, root, id, sizeof(id)); - if (valid) + if (valid && tf_ssb_verify_and_strip_signature(ssb->context, root, id, sizeof(id), NULL, 0, NULL)) { if (tf_ssb_db_store_message(ssb, ssb->context, id, root, signature_base64, false)) { @@ -1232,8 +1281,7 @@ void tf_ssb_append_message(tf_ssb_t* ssb, JSValue message) printf("message not stored.\n"); } } - - if (!tf_ssb_verify_and_strip_signature(ssb->context, root, NULL, 0, NULL)) + else { printf("Failed to verify message signature.\n"); } diff --git a/src/ssb.db.c b/src/ssb.db.c index 1257907f..2e58919c 100644 --- a/src/ssb.db.c +++ b/src/ssb.db.c @@ -507,3 +507,101 @@ void tf_ssb_db_visit_query(tf_ssb_t* ssb, const char* query, const JSValue binds } sqlite3_set_authorizer(db, NULL, NULL); } + +static JSValue _tf_ssb_format_message(JSContext* context, const char* previous, const char* author, int64_t sequence, int64_t timestamp, const char* hash, const char* content, const char* signature, bool sequence_before_author) +{ + JSValue value = JS_NewObject(context); + JS_SetPropertyStr(context, value, "previous", previous ? JS_NewString(context, previous) : JS_NULL); + if (sequence_before_author) + { + JS_SetPropertyStr(context, value, "sequence", JS_NewInt64(context, sequence)); + JS_SetPropertyStr(context, value, "author", JS_NewString(context, author)); + } + else + { + JS_SetPropertyStr(context, value, "author", JS_NewString(context, author)); + JS_SetPropertyStr(context, value, "sequence", JS_NewInt64(context, sequence)); + } + JS_SetPropertyStr(context, value, "timestamp", JS_NewInt64(context, timestamp)); + JS_SetPropertyStr(context, value, "hash", JS_NewString(context, hash)); + JS_SetPropertyStr(context, value, "content", JS_ParseJSON(context, content, strlen(content), NULL)); + JS_SetPropertyStr(context, value, "signature", JS_NewString(context, signature)); + return value; +} + +bool _tf_ssb_update_message_id(sqlite3* db, const char* old_id, const char* new_id) +{ + bool success = false; + sqlite3_stmt* statement = NULL; + if (sqlite3_prepare(db, "UPDATE messages SET id = ? WHERE id = ?", -1, &statement, NULL) == SQLITE_OK) + { + if (sqlite3_bind_text(statement, 1, new_id, -1, NULL) == SQLITE_OK && + sqlite3_bind_text(statement, 2, old_id, -1, NULL) == SQLITE_OK) + { + success = sqlite3_step(statement) == SQLITE_DONE; + } + sqlite3_finalize(statement); + } + return success; +} + +bool tf_ssb_db_check(sqlite3* db) +{ + JSRuntime* runtime = JS_NewRuntime(); + JSContext* context = JS_NewContext(runtime); + + sqlite3_stmt* statement = NULL; + if (sqlite3_prepare(db, "SELECT id, previous, author, sequence, timestamp, hash, content, signature, sequence_before_author FROM messages ORDER BY author, sequence", -1, &statement, NULL) == SQLITE_OK) + { + char previous_id[k_id_base64_len]; + while (sqlite3_step(statement) == SQLITE_ROW) + { + const char* id = (const char*)sqlite3_column_text(statement, 0); + const char* previous = (const char*)sqlite3_column_text(statement, 1); + const char* author = (const char*)sqlite3_column_text(statement, 2); + int64_t sequence = sqlite3_column_int64(statement, 3); + int64_t timestamp = sqlite3_column_int64(statement, 4); + const char* hash = (const char*)sqlite3_column_text(statement, 5); + const char* content = (const char*)sqlite3_column_text(statement, 6); + const char* signature = (const char*)sqlite3_column_text(statement, 7); + bool sequence_before_author = sqlite3_column_int(statement, 8); + JSValue message = _tf_ssb_format_message(context, previous, author, sequence, timestamp, hash, content, signature, sequence_before_author); + char out_signature[512]; + char actual_id[k_id_base64_len]; + bool actual_sequence_before_author = false; + JSValue j = JS_JSONStringify(context, message, JS_NULL, JS_NewInt32(context, 2)); + const char* jv = JS_ToCString(context, j); + if (tf_ssb_verify_and_strip_signature(context, message, actual_id, sizeof(actual_id), out_signature, sizeof(out_signature), &actual_sequence_before_author)) + { + /*if (previous && strcmp(previous, previous_id)) + { + printf("%s:%d previous was %s should be %s\n", id, (int)sequence, previous_id, previous); + }*/ + if (strcmp(id, actual_id)) + { + if (_tf_ssb_update_message_id(db, id, actual_id)) + { + printf("updated %s to %s\n", id, actual_id); + } + else + { + printf("failed to update %s to %s\n", id, actual_id); + } + } + } + else + { + printf("unable to verify signature for %s\n", id); + } + JS_FreeCString(context, jv); + JS_FreeValue(context, j); + snprintf(previous_id, sizeof(previous_id), "%s", id); + JS_FreeValue(context, message); + } + sqlite3_finalize(statement); + } + + JS_FreeContext(context); + JS_FreeRuntime(runtime); + return false; +} diff --git a/src/ssb.db.h b/src/ssb.db.h index ec03d1d7..1e6f3145 100644 --- a/src/ssb.db.h +++ b/src/ssb.db.h @@ -14,3 +14,6 @@ bool tf_ssb_db_blob_store(tf_ssb_t* ssb, const uint8_t* blob, size_t size, char* bool tf_ssb_db_get_message_by_author_and_sequence(tf_ssb_t* ssb, const char* author, int64_t sequence, char* out_message_id, size_t out_message_id_size, int64_t* out_timestamp, char** out_content); bool tf_ssb_db_get_latest_message_by_author(tf_ssb_t* ssb, const char* author, int64_t* out_sequence, char* out_message_id, size_t out_message_id_size); void tf_ssb_db_visit_query(tf_ssb_t* ssb, const char* query, const JSValue binds, void (*callback)(JSValue row, void* user_data), void* user_data); + +typedef struct sqlite3 sqlite3; +bool tf_ssb_db_check(sqlite3* db); diff --git a/src/ssb.h b/src/ssb.h index e60b0bc6..86530a12 100644 --- a/src/ssb.h +++ b/src/ssb.h @@ -89,7 +89,7 @@ void tf_ssb_send_close(tf_ssb_t* ssb); bool tf_ssb_id_str_to_bin(uint8_t* bin, const char* str); bool tf_ssb_id_bin_to_str(char* str, size_t str_size, const uint8_t* bin); -bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author); +bool tf_ssb_verify_and_strip_signature(JSContext* context, JSValue val, char* out_id, size_t out_id_size, char* out_signature, size_t out_signature_size, bool* out_sequence_before_author); void tf_ssb_calculate_message_id(JSContext* context, JSValue message, char* out_id, size_t out_id_size); const char* tf_ssb_connection_get_host(tf_ssb_connection_t* connection); diff --git a/src/ssb.js.c b/src/ssb.js.c index 55d05d35..d6f40f69 100644 --- a/src/ssb.js.c +++ b/src/ssb.js.c @@ -224,9 +224,8 @@ static JSValue _tf_ssb_storeMessage(JSContext* context, JSValueConst this_val, i tf_ssb_t* ssb = JS_GetOpaque(this_val, _tf_ssb_classId); char signature[crypto_sign_BYTES + 128]; char id[crypto_hash_sha256_BYTES * 2 + 1]; - tf_ssb_calculate_message_id(context, argv[0], id, sizeof(id)); bool sequence_before_author = false; - if (tf_ssb_verify_and_strip_signature(context, argv[0], signature, sizeof(signature), &sequence_before_author)) + if (tf_ssb_verify_and_strip_signature(context, argv[0], id, sizeof(id), signature, sizeof(signature), &sequence_before_author)) { if (tf_ssb_db_store_message(ssb, context, id, argv[0], signature, sequence_before_author)) {