Allow running read-only sqlite queries from libuv worker threads. Needs so much more testing.

git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@4172 ed5197a5-7fde-0310-b194-c3ffbd925b24
This commit is contained in:
Cory McWilliams 2023-02-08 01:29:44 +00:00
parent ac60be14a5
commit b35d74ce36
7 changed files with 332 additions and 93 deletions

View File

@ -208,12 +208,7 @@ static int _tf_command_import(const char* file, int argc, char* argv[])
return 2;
}
sqlite3* db = NULL;
if (args.db_path)
{
sqlite3_open(args.db_path, &db);
}
tf_ssb_t* ssb = tf_ssb_create(NULL, NULL, db);
tf_ssb_t* ssb = tf_ssb_create(NULL, NULL, args.db_path);
if (extra_count)
{
for (int i = 0; i < extra_count; i++)
@ -228,10 +223,6 @@ static int _tf_command_import(const char* file, int argc, char* argv[])
tf_ssb_import(ssb, args.user, "apps");
}
tf_ssb_destroy(ssb);
if (db)
{
sqlite3_close(db);
}
if (extras)
{
@ -277,12 +268,7 @@ static int _tf_command_export(const char* file, int argc, char* argv[])
}
return 2;
}
sqlite3* db = NULL;
if (args.db_path)
{
sqlite3_open(args.db_path, &db);
}
tf_ssb_t* ssb = tf_ssb_create(NULL, NULL, db);
tf_ssb_t* ssb = tf_ssb_create(NULL, NULL, args.db_path);
if (extra_count)
{
for (int i = 0; i < extra_count; i++)
@ -312,10 +298,6 @@ static int _tf_command_export(const char* file, int argc, char* argv[])
}
}
tf_ssb_destroy(ssb);
if (db)
{
sqlite3_close(db);
}
if (extras)
{

View File

@ -171,8 +171,12 @@ typedef struct _tf_ssb_t
tf_trace_t* trace;
const char* db_path;
sqlite3* db;
bool owns_db;
uv_mutex_t db_readers_lock;
sqlite3** db_readers;
int db_readers_count;
uv_loop_t own_loop;
uv_loop_t* loop;
@ -2064,7 +2068,7 @@ void tf_ssb_get_stats(tf_ssb_t* ssb, tf_ssb_stats_t* out_stats)
ssb->rpc_out = 0;
}
tf_ssb_t* tf_ssb_create(uv_loop_t* loop, JSContext* context, sqlite3* db)
tf_ssb_t* tf_ssb_create(uv_loop_t* loop, JSContext* context, const char* db_path)
{
tf_ssb_t* ssb = tf_malloc(sizeof(tf_ssb_t));
memset(ssb, 0, sizeof(*ssb));
@ -2082,6 +2086,8 @@ tf_ssb_t* tf_ssb_create(uv_loop_t* loop, JSContext* context, sqlite3* db)
ssb->context = JS_NewContext(ssb->runtime);
}
uv_mutex_init(&ssb->db_readers_lock);
JS_NewClassID(&_connection_class_id);
JSClassDef def =
{
@ -2090,15 +2096,8 @@ tf_ssb_t* tf_ssb_create(uv_loop_t* loop, JSContext* context, sqlite3* db)
};
JS_NewClass(JS_GetRuntime(ssb->context), _connection_class_id, &def);
if (db)
{
ssb->db = db;
}
else
{
sqlite3_open("db.sqlite", &ssb->db);
ssb->owns_db = true;
}
ssb->db_path = tf_strdup(db_path);
sqlite3_open(db_path, &ssb->db);
tf_ssb_db_init(ssb);
if (loop)
@ -2131,6 +2130,30 @@ sqlite3* tf_ssb_get_db(tf_ssb_t* ssb)
return ssb->db;
}
sqlite3* tf_ssb_acquire_db_reader(tf_ssb_t* ssb)
{
sqlite3* db = NULL;
uv_mutex_lock(&ssb->db_readers_lock);
if (ssb->db_readers_count)
{
db = ssb->db_readers[--ssb->db_readers_count];
}
else
{
sqlite3_open_v2(ssb->db_path, &db, SQLITE_OPEN_READONLY, NULL);
}
uv_mutex_unlock(&ssb->db_readers_lock);
return db;
}
void tf_ssb_release_db_reader(tf_ssb_t* ssb, sqlite3* db)
{
uv_mutex_lock(&ssb->db_readers_lock);
ssb->db_readers = tf_resize_vec(ssb->db_readers, sizeof(sqlite3*) * (ssb->db_readers_count + 1));
ssb->db_readers[ssb->db_readers_count++] = db;
uv_mutex_unlock(&ssb->db_readers_lock);
}
uv_loop_t* tf_ssb_get_loop(tf_ssb_t* ssb)
{
return ssb->loop;
@ -2289,10 +2312,7 @@ void tf_ssb_destroy(tf_ssb_t* ssb)
JS_FreeContext(ssb->context);
JS_FreeRuntime(ssb->runtime);
}
if (ssb->owns_db)
{
sqlite3_close(ssb->db);
}
sqlite3_close(ssb->db);
while (ssb->broadcasts)
{
tf_ssb_broadcast_t* broadcast = ssb->broadcasts;
@ -2307,6 +2327,12 @@ void tf_ssb_destroy(tf_ssb_t* ssb)
tf_free(ssb->debug_close[i].messages[j]);
}
}
for (int i = 0; i < ssb->db_readers_count; i++)
{
sqlite3_close(ssb->db_readers[i]);
}
uv_mutex_destroy(&ssb->db_readers_lock);
tf_free((void*)ssb->db_path);
tf_free(ssb);
}

View File

@ -52,11 +52,16 @@ static bool _tf_ssb_db_has_rows(sqlite3* db, const char* query)
return found;
}
static void _tf_ssb_db_init_internal(sqlite3* db)
{
_tf_ssb_db_exec(db, "PRAGMA journal_mode = WAL");
_tf_ssb_db_exec(db, "PRAGMA synchronous = NORMAL");
}
void tf_ssb_db_init(tf_ssb_t* ssb)
{
sqlite3* db = tf_ssb_get_db(ssb);
_tf_ssb_db_exec(db, "PRAGMA journal_mode = WAL");
_tf_ssb_db_exec(db, "PRAGMA synchronous = NORMAL");
_tf_ssb_db_init_internal(db);
_tf_ssb_db_exec(db,
"CREATE TABLE IF NOT EXISTS messages ("
" author TEXT,"

View File

@ -65,10 +65,12 @@ typedef struct _tf_ssb_blob_wants_t
int wants_sent;
} tf_ssb_blob_wants_t;
tf_ssb_t* tf_ssb_create(uv_loop_t* loop, JSContext* context, sqlite3* db);
tf_ssb_t* tf_ssb_create(uv_loop_t* loop, JSContext* context, const char* db_path);
void tf_ssb_destroy(tf_ssb_t* ssb);
sqlite3* tf_ssb_get_db(tf_ssb_t* ssb);
sqlite3* tf_ssb_acquire_db_reader(tf_ssb_t* ssb);
void tf_ssb_release_db_reader(tf_ssb_t* ssb, sqlite3* db);
uv_loop_t* tf_ssb_get_loop(tf_ssb_t* ssb);
void tf_ssb_generate_keys(tf_ssb_t* ssb);

View File

@ -4,15 +4,17 @@
#include "mem.h"
#include "ssb.db.h"
#include "ssb.h"
#include "task.h"
#include "util.js.h"
#include <base64c.h>
#include <sodium/crypto_hash_sha256.h>
#include <sodium/crypto_sign.h>
#include <string.h>
#include <sqlite3.h>
#include <uv.h>
#include <inttypes.h>
#include "quickjs-libc.h"
static JSClassID _tf_ssb_classId;
@ -332,6 +334,265 @@ static JSValue _tf_ssb_sqlStream(JSContext* context, JSValueConst this_val, int
return result;
}
typedef struct _sql_work_t
{
uv_work_t request;
tf_ssb_t* ssb;
const char* query;
uint8_t* binds;
size_t binds_count;
uint8_t* rows;
size_t rows_count;
JSValue callback;
JSValue promise[2];
int result;
} sql_work_t;
static void _tf_ssb_sql_append(uint8_t** rows, size_t* rows_count, const void* data, size_t size)
{
*rows = tf_resize_vec(*rows, *rows_count + size);
memcpy(*rows + *rows_count, data, size);
*rows_count += size;
}
static void _tf_ssb_sqlAsync_work(uv_work_t* work)
{
sql_work_t* sql_work = work->data;
sqlite3* db = tf_ssb_acquire_db_reader(sql_work->ssb);
sqlite3_stmt* statement = NULL;
if (sqlite3_prepare(db, sql_work->query, -1, &statement, NULL) == SQLITE_OK)
{
const uint8_t* p = sql_work->binds;
int column = 0;
while (p < sql_work->binds + sql_work->binds_count)
{
switch (*p++)
{
case SQLITE_INTEGER:
{
int64_t value = 0;
memcpy(&value, p, sizeof(value));
sqlite3_bind_int64(statement, column + 1, value);
p += sizeof(value);
}
break;
case SQLITE_TEXT:
{
size_t length = 0;
memcpy(&length, p, sizeof(length));
p += sizeof(length);
sqlite3_bind_text(statement, column + 1, (const char*)p, length, NULL);
}
break;
case SQLITE_NULL:
sqlite3_bind_null(statement, column + 1);
break;
default:
abort();
}
column++;
}
int r = SQLITE_OK;
while ((r = sqlite3_step(statement)) == SQLITE_ROW)
{
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &(uint8_t[]) { 'r' }, 1);
for (int i = 0; i < sqlite3_column_count(statement); i++)
{
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &(uint8_t[]) { 'c' }, 1);
const char* name = sqlite3_column_name(statement, i);
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, name, strlen(name) + 1);
uint8_t type = sqlite3_column_type(statement, i);
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &type, sizeof(type));
switch (type)
{
case SQLITE_INTEGER:
{
int64_t value = sqlite3_column_int64(statement, i);
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &value, sizeof(value));
}
break;
case SQLITE_FLOAT:
{
double value = sqlite3_column_double(statement, i);
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &value, sizeof(value));
}
break;
case SQLITE_TEXT:
{
size_t bytes = sqlite3_column_bytes(statement, i);
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &bytes, sizeof(bytes));
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, sqlite3_column_text(statement, i), bytes);
}
break;
case SQLITE_BLOB:
{
size_t bytes = sqlite3_column_bytes(statement, i);
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &bytes, sizeof(bytes));
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, sqlite3_column_blob(statement, i), bytes);
}
break;
case SQLITE_NULL:
break;
default:
abort();
}
}
}
_tf_ssb_sql_append(&sql_work->rows, &sql_work->rows_count, &(uint8_t[]) { 0 }, 1);
sqlite3_finalize(statement);
}
else
{
printf("prepare failed\n");
}
tf_ssb_release_db_reader(sql_work->ssb, db);
}
static void _tf_ssb_sqlAsync_after_work(uv_work_t* work, int status)
{
sql_work_t* sql_work = work->data;
JSContext* context = tf_ssb_get_context(sql_work->ssb);
uint8_t* p = sql_work->rows;
while (p < sql_work->rows + sql_work->rows_count)
{
if (*p++ == 'r')
{
JSValue row = JS_NewObject(context);
while (*p == 'c')
{
p++;
const char* column_name = (const char*)p;
size_t length = strlen((char*)p);
p += length + 1;
switch (*p++)
{
case SQLITE_INTEGER:
{
int64_t value = 0;
memcpy(&value, p, sizeof(value));
JS_SetPropertyStr(context, row, column_name, JS_NewInt64(context, value));
p += sizeof(value);
}
break;
case SQLITE_FLOAT:
{
double value = 0.0;
memcpy(&value, p, sizeof(value));
JS_SetPropertyStr(context, row, column_name, JS_NewFloat64(context, value));
p += sizeof(value);
}
break;
case SQLITE_TEXT:
case SQLITE_BLOB:
{
size_t length = 0;
memcpy(&length, p, sizeof(length));
p += sizeof(length);
JS_SetPropertyStr(context, row, column_name, JS_NewStringLen(context, (const char*)p, length));
p += length;
}
break;
case SQLITE_NULL:
JS_SetPropertyStr(context, row, column_name, JS_NULL);
break;
}
}
JSValue response = JS_Call(context, sql_work->callback, JS_UNDEFINED, 1, &row);
tf_util_report_error(context, response);
JS_FreeValue(context, row);
}
else
{
break;
}
}
tf_free(sql_work->binds);
tf_free(sql_work->rows);
JSValue result = JS_Call(context, sql_work->promise[0], JS_UNDEFINED, 0, NULL);
tf_util_report_error(context, result);
JS_FreeValue(context, sql_work->promise[0]);
JS_FreeValue(context, sql_work->promise[1]);
tf_free(sql_work);
}
static JSValue _tf_ssb_sqlAsync(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
{
tf_ssb_t* ssb = JS_GetOpaque(this_val, _tf_ssb_classId);
JSValue result = JS_UNDEFINED;
if (ssb)
{
const char* query = JS_ToCString(context, argv[0]);
sql_work_t* work = tf_malloc(sizeof(sql_work_t));
*work = (sql_work_t)
{
.request =
{
.data = work,
},
.ssb = ssb,
.callback = JS_DupValue(context, argv[2]),
.query = query,
};
result = JS_NewPromiseCapability(context, work->promise);
int32_t length = tf_util_get_length(context, argv[1]);
for (int i = 0; i < length; i++)
{
JSValue value = JS_GetPropertyUint32(context, argv[1], i);
if (JS_IsNumber(value))
{
uint8_t type = SQLITE_INTEGER;
int64_t number = 0;
JS_ToInt64(context, &number, value);
_tf_ssb_sql_append(&work->binds, &work->binds_count, &type, sizeof(type));
_tf_ssb_sql_append(&work->binds, &work->binds_count, &number, sizeof(number));
}
else if (JS_IsBool(value))
{
uint8_t type = SQLITE_INTEGER;
int64_t number = JS_ToBool(context, value) ? 1 : 0;
_tf_ssb_sql_append(&work->binds, &work->binds_count, &type, sizeof(type));
_tf_ssb_sql_append(&work->binds, &work->binds_count, &number, sizeof(number));
}
else if (JS_IsNumber(value))
{
uint8_t type = SQLITE_NULL;
_tf_ssb_sql_append(&work->binds, &work->binds_count, &type, sizeof(type));
}
else
{
uint8_t type = SQLITE_TEXT;
size_t length = 0;
const char* string = JS_ToCStringLen(context, &length, value);
if (!string)
{
string = "";
}
_tf_ssb_sql_append(&work->binds, &work->binds_count, &type, sizeof(type));
_tf_ssb_sql_append(&work->binds, &work->binds_count, string, length);
JS_FreeCString(context, string);
}
}
int r = uv_queue_work(tf_ssb_get_loop(ssb), &work->request, _tf_ssb_sqlAsync_work, _tf_ssb_sqlAsync_after_work);
if (r)
{
JSValue error = JS_ThrowInternalError(context, "uv_queue_work failed: %s", uv_strerror(r));
JSValue result = JS_Call(context, work->promise[1], JS_UNDEFINED, 1, &error);
tf_util_report_error(context, result);
JS_FreeValue(context, work->promise[0]);
JS_FreeValue(context, work->promise[1]);
JS_FreeValue(context, error);
tf_free(work->binds);
tf_free(work);
}
}
return result;
}
static JSValue _tf_ssb_storeMessage(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
{
tf_ssb_t* ssb = JS_GetOpaque(this_val, _tf_ssb_classId);
@ -855,6 +1116,7 @@ void tf_ssb_register(JSContext* context, tf_ssb_t* ssb)
JS_SetPropertyStr(context, object, "closeConnection", JS_NewCFunction(context, _tf_ssb_closeConnection, "closeConnection", 1));
JS_SetPropertyStr(context, object, "forgetStoredConnection", JS_NewCFunction(context, _tf_ssb_forgetStoredConnection, "forgetStoredConnection", 1));
JS_SetPropertyStr(context, object, "sqlStream", JS_NewCFunction(context, _tf_ssb_sqlStream, "sqlStream", 3));
JS_SetPropertyStr(context, object, "sqlAsync", JS_NewCFunction(context, _tf_ssb_sqlAsync, "sqlAsync", 3));
JS_SetPropertyStr(context, object, "storeMessage", JS_NewCFunction(context, _tf_ssb_storeMessage, "storeMessage", 1));
JS_SetPropertyStr(context, object, "getBroadcasts", JS_NewCFunction(context, _tf_ssb_getBroadcasts, "getBroadcasts", 0));
JS_SetPropertyStr(context, object, "connect", JS_NewCFunction(context, _tf_ssb_connect, "connect", 1));

View File

@ -127,21 +127,13 @@ static void _ssb_test_idle(uv_idle_t* idle)
void tf_ssb_test_ssb(const tf_test_options_t* options)
{
printf("Testing SSB.\n");
sqlite3* db0 = NULL;
sqlite3* db1 = NULL;
int r = sqlite3_open(":memory:", &db0);
(void)r;
assert(r == SQLITE_OK);
r = sqlite3_open(":memory:", &db1);
assert(r == SQLITE_OK);
uv_loop_t loop = { 0 };
uv_loop_init(&loop);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, db0);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_register(tf_ssb_get_context(ssb0), ssb0);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, db1);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_register(tf_ssb_get_context(ssb1), ssb1);
uv_idle_t idle0 = { .data = ssb0 };
@ -267,9 +259,6 @@ void tf_ssb_test_ssb(const tf_test_options_t* options)
tf_ssb_destroy(ssb1);
uv_loop_close(&loop);
sqlite3_close(db0);
sqlite3_close(db1);
}
static void _broadcasts_visit(const char* host, const struct sockaddr_in* addr, tf_ssb_connection_t* tunnel, const uint8_t* pub, void* user_data)
@ -302,26 +291,15 @@ static void _broadcasts_changed(tf_ssb_t* ssb, void* user_data)
void tf_ssb_test_rooms(const tf_test_options_t* options)
{
printf("Testing Rooms.\n");
sqlite3* db0 = NULL;
sqlite3* db1 = NULL;
sqlite3* db2 = NULL;
int r = sqlite3_open(":memory:", &db0);
(void)r;
assert(r == SQLITE_OK);
r = sqlite3_open(":memory:", &db1);
assert(r == SQLITE_OK);
r = sqlite3_open(":memory:", &db2);
assert(r == SQLITE_OK);
uv_loop_t loop = { 0 };
uv_loop_init(&loop);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, db0);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_register(tf_ssb_get_context(ssb0), ssb0);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, db1);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_register(tf_ssb_get_context(ssb1), ssb1);
tf_ssb_t* ssb2 = tf_ssb_create(&loop, NULL, db2);
tf_ssb_t* ssb2 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_register(tf_ssb_get_context(ssb2), ssb2);
uv_idle_t idle0 = { .data = ssb0 };
@ -462,30 +440,22 @@ void tf_ssb_test_rooms(const tf_test_options_t* options)
uv_run(&loop, UV_RUN_DEFAULT);
uv_loop_close(&loop);
sqlite3_close(db0);
sqlite3_close(db1);
sqlite3_close(db2);
}
void tf_ssb_test_following(const tf_test_options_t* options)
{
printf("Testing following.\n");
sqlite3* db0 = NULL;
int r = sqlite3_open(":memory:", &db0);
(void)r;
assert(r == SQLITE_OK);
uv_loop_t loop = { 0 };
uv_loop_init(&loop);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, db0);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_generate_keys(ssb0);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, db0);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_generate_keys(ssb1);
tf_ssb_t* ssb2 = tf_ssb_create(&loop, NULL, db0);
tf_ssb_t* ssb2 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_generate_keys(ssb2);
char id0[k_id_base64_len] = { 0 };
@ -553,22 +523,16 @@ void tf_ssb_test_following(const tf_test_options_t* options)
tf_ssb_destroy(ssb2);
uv_loop_close(&loop);
sqlite3_close(db0);
}
void tf_ssb_test_bench(const tf_test_options_t* options)
{
printf("Testing following.\n");
sqlite3* db0 = NULL;
int r = sqlite3_open(":memory:", &db0);
(void)r;
assert(r == SQLITE_OK);
uv_loop_t loop = { 0 };
uv_loop_init(&loop);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, db0);
tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_generate_keys(ssb0);
char id0[k_id_base64_len] = { 0 };
@ -585,10 +549,7 @@ void tf_ssb_test_bench(const tf_test_options_t* options)
clock_gettime(CLOCK_REALTIME, &end_time);
printf("insert = %f seconds\n", (float)(end_time.tv_sec - start_time.tv_sec) + (end_time.tv_nsec - start_time.tv_nsec) / 1e9f);
sqlite3* db1 = NULL;
sqlite3_open(":memory:", &db1);
assert(r == SQLITE_OK);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, db1);
tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, ":memory:");
tf_ssb_generate_keys(ssb1);
uint8_t id0bin[k_id_bin_len];
tf_ssb_id_str_to_bin(id0bin, id0);
@ -629,7 +590,4 @@ void tf_ssb_test_bench(const tf_test_options_t* options)
tf_ssb_destroy(ssb0);
uv_loop_close(&loop);
sqlite3_close(db0);
sqlite3_close(db1);
}

View File

@ -1562,7 +1562,11 @@ void tf_task_activate(tf_task_t* task)
task->_trace = tf_trace_create();
if (task->_trusted)
{
sqlite3_open(*task->_db_path ? task->_db_path : "db.sqlite", &task->_db);
if (!*task->_db_path)
{
snprintf(task->_db_path, sizeof(task->_db_path), "db.sqlite");
}
sqlite3_open(task->_db_path, &task->_db);
JS_SetPropertyStr(context, global, "Task", tf_taskstub_register(context));
JS_SetPropertyStr(context, global, "Socket", tf_socket_register(context));
@ -1570,7 +1574,7 @@ void tf_task_activate(tf_task_t* task)
tf_file_register(context);
tf_database_register(context, task->_db);
task->_ssb = tf_ssb_create(&task->_loop, task->_context, task->_db);
task->_ssb = tf_ssb_create(&task->_loop, task->_context, task->_db_path);
tf_ssb_set_trace(task->_ssb, task->_trace);
tf_ssb_register(context, task->_ssb);