tasiaiso/tildefriends
1
0
Fork 0
forked from cory/tildefriends
Code Issues Pull Requests Releases Wiki Activity

Refactored import and export. No user on disk.

Browse Source 
git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@4164 ed5197a5-7fde-0310-b194-c3ffbd925b24
This commit is contained in:
Cory McWilliams
2023-02-02 02:09:05 +00:00
parent ec52e62908
commit e04d137af5
12 changed files with 288 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main.c
View File

@ -257,12 +257,12 @@ static int _tf_command_export(const char* file, int argc, char* argv[])
xoptOption options[] = {
{ "db-path", 'd', offsetof(args_t, db_path), NULL, XOPT_TYPE_STRING, NULL, "Sqlite database path (default: db.sqlite)." },
{ "user", 'u', offsetof(args_t, user), NULL, XOPT_TYPE_STRING, NULL, "User into whose apps will be exported (default: \"cory\")." },
{ "user", 'u', offsetof(args_t, user), NULL, XOPT_TYPE_STRING, NULL, "User into whose apps will be exported (default: \"core\")." },
{ "help", 'h', offsetof(args_t, help), NULL, XOPT_TYPE_BOOL, NULL, "Shows this help message." },
XOPT_NULLOPTION,
};
args_t args = { .user = "cory" };
args_t args = { .user = "core" };
const char** extras = NULL;
int extra_count = 0;
const char *err = NULL;

84
src/ssb.export.c
View File

@ -9,7 +9,7 @@
#include <sqlite3.h>
#include <uv.h>
static void _write_file(const char* path, void* blob, size_t size)
static void _write_file(const char* path, const void* blob, size_t size)
{
FILE* file = fopen(path, "wb");
if (file)
@ -35,6 +35,45 @@ static void _make_dir(const char* path)
}
}
typedef struct _tf_export_t
{
tf_ssb_t* ssb;
const char* parent;
JSValue files;
uv_fs_t req;
bool done;
} tf_export_t;
static void _tf_ssb_export_scandir(uv_fs_t* req)
{
tf_export_t* export = req->data;
JSContext* context = tf_ssb_get_context(export->ssb);
uv_dirent_t ent;
while (uv_fs_scandir_next(req, &ent) == 0)
{
if (ent.type == UV_DIRENT_FILE)
{
JSValue found = JS_GetPropertyStr(context, export->files, ent.name);
if (JS_IsUndefined(found))
{
size_t len = strlen(export->parent) + strlen(ent.name) + 2;
char* path = tf_malloc(len);
snprintf(path, len, "%s/%s", export->parent, ent.name);
uv_fs_t req = { 0 };
int r = uv_fs_unlink(tf_ssb_get_loop(export->ssb), &req, path, NULL);
if (r)
{
printf("Failed to unlink %s: %s.", path, uv_strerror(r));
}
uv_fs_req_cleanup(&req);
tf_free(path);
}
JS_FreeValue(context, found);
}
}
export->done = true;
}
void tf_ssb_export(tf_ssb_t* ssb, const char* key)
{
char user[256] = { 0 };
@ -80,12 +119,9 @@ void tf_ssb_export(tf_ssb_t* ssb, const char* key)
}
char file_path[1024];
_make_dir("apps/");
snprintf(file_path, sizeof(file_path), "apps/%s", user);
snprintf(file_path, sizeof(file_path), "apps/%s", path);
_make_dir(file_path);
snprintf(file_path, sizeof(file_path), "apps/%s/%s", user, path);
_make_dir(file_path);
snprintf(file_path, sizeof(file_path), "apps/%s/%s.json", user, path);
_write_file(file_path, blob, size);
snprintf(file_path, sizeof(file_path), "apps/%s.json", path);
JSContext* context = tf_ssb_get_context(ssb);
JSValue app = JS_ParseJSON(context, (const char*)blob, size, NULL);
tf_free(blob);
@ -108,7 +144,7 @@ void tf_ssb_export(tf_ssb_t* ssb, const char* key)
size_t file_size = 0;
if (tf_ssb_db_blob_get(ssb, blob_id, &file_blob, &file_size))
{
snprintf(file_path, sizeof(file_path), "apps/%s/%s/%s", user, path, file_name);
snprintf(file_path, sizeof(file_path), "apps/%s/%s", path, file_name);
_write_file(file_path, file_blob, file_size);
tf_free(file_blob);
}
@ -129,6 +165,40 @@ void tf_ssb_export(tf_ssb_t* ssb, const char* key)
}
js_free(context, ptab);
JSAtom files_atom = JS_NewAtom(context, "files");
JS_DeleteProperty(context, app, files_atom, 0);
JS_FreeAtom(context, files_atom);
JSValue json = JS_JSONStringify(context, app, JS_NULL, JS_NewInt32(context, 2));
size_t length = 0;
const char* string = JS_ToCStringLen(context, &length, json);
snprintf(file_path, sizeof(file_path), "apps/%s.json", path);
_write_file(file_path, string, length);
JS_FreeCString(context, string);
JS_FreeValue(context, json);
snprintf(file_path, sizeof(file_path), "apps//%s", path);
tf_export_t export =
{
.parent = file_path,
.ssb = ssb,
.files = files,
.req =
{
.data = &export,
},
};
int r = uv_fs_scandir(tf_ssb_get_loop(ssb), &export.req, file_path, 0, _tf_ssb_export_scandir);
if (r)
{
printf("Failed to scan directory %s: %s.", file_path, uv_strerror(r));
}
while (!export.done)
{
uv_run(tf_ssb_get_loop(ssb), UV_RUN_ONCE);
}
uv_fs_req_cleanup(&export.req);
JS_FreeValue(context, files);
JS_FreeValue(context, app);
}

266
src/ssb.import.c
View File

@ -12,25 +12,6 @@
#include <sqlite3.h>
#include <uv.h>
typedef struct _tf_import_file_t {
uv_fs_t req;
uv_file file;
tf_ssb_t* ssb;
const char* user;
const char* parent;
const char* name;
char data[k_ssb_blob_bytes_max];
int* work_left;
} tf_import_file_t;
static void _tf_ssb_import_file_close(uv_fs_t* req)
{
tf_import_file_t* file = req->data;
(*file->work_left)--;
uv_fs_req_cleanup(req);
tf_free(req->data);
}
static void _tf_ssb_import_add_app(tf_ssb_t* ssb, const char* user, const char* app)
{
sqlite3_stmt* statement;
@ -80,6 +61,7 @@ static void _tf_ssb_import_add_app(tf_ssb_t* ssb, const char* user, const char*
JS_FreeCString(context, last_added);
last_added = read_string;
}
JS_FreeCString(context, last_added);
JSValue json = JS_JSONStringify(context, out_apps, JS_NULL, JS_NULL);
const char* text = JS_ToCString(context, json);
@ -99,6 +81,7 @@ static void _tf_ssb_import_add_app(tf_ssb_t* ssb, const char* user, const char*
JS_FreeValue(context, apps);
}
/*
static void _tf_ssb_import_file_read(uv_fs_t* req)
{
tf_import_file_t* file = req->data;
@ -106,32 +89,49 @@ static void _tf_ssb_import_file_read(uv_fs_t* req)
if (req->result >= 0)
{
bool is_new = false;
if (tf_ssb_db_blob_store(file->ssb, (const uint8_t*)file->data, req->result, id, sizeof(id), &is_new))
bool is_app_json = false;
if (strcasecmp(file->name + strlen(file->name) - strlen(".json"), ".json") == 0)
{
if (is_new)
JSContext* context = tf_ssb_get_context(file->ssb);
JSValue object = JS_ParseJSON(context, file->data, req->result, NULL);
JSValue type = JS_GetPropertyStr(context, object, "type");
if (!JS_IsUndefined(type))
{
printf("Stored %s/%s as %s.\n", file->parent, file->name, id);
}
if (strcasecmp(file->name + strlen(file->name) - strlen(".json"), ".json") == 0)
{
sqlite3_stmt* statement;
if (sqlite3_prepare(tf_ssb_get_db(file->ssb), "INSERT OR REPLACE INTO properties (id, key, value) VALUES ($1, 'path:' || $2, $3)", -1, &statement, NULL) == SQLITE_OK)
const char* type_string = JS_ToCString(context, type);
if (type_string && strcmp(type_string, "tf-application") == 0)
{
((char*)file->name)[strlen(file->name) - strlen(".json")] = '\0';
if (sqlite3_bind_text(statement, 1, file->user, -1, NULL) == SQLITE_OK &&
sqlite3_bind_text(statement, 2, file->name, -1, NULL) == SQLITE_OK &&
sqlite3_bind_text(statement, 3, id, -1, NULL) == SQLITE_OK &&
sqlite3_step(statement) == SQLITE_DONE)
is_app_json = true;
sqlite3_stmt* statement;
if (sqlite3_prepare(tf_ssb_get_db(file->ssb), "INSERT OR REPLACE INTO properties (id, key, value) VALUES ($1, 'path:' || $2, $3)", -1, &statement, NULL) == SQLITE_OK)
{
if (sqlite3_changes(tf_ssb_get_db(file->ssb)))
((char*)file->name)[strlen(file->name) - strlen(".json")] = '\0';
if (sqlite3_bind_text(statement, 1, file->user, -1, NULL) == SQLITE_OK &&
sqlite3_bind_text(statement, 2, file->name, -1, NULL) == SQLITE_OK &&
sqlite3_bind_text(statement, 3, id, -1, NULL) == SQLITE_OK &&
sqlite3_step(statement) == SQLITE_DONE)
{
printf("Registered %s path:%s as %s.\n", file->user, file->name, id);
if (sqlite3_changes(tf_ssb_get_db(file->ssb)))
{
printf("Registered %s path:%s as %s.\n", file->user, file->name, id);
}
_tf_ssb_import_add_app(file->ssb, file->user, file->name);
}
_tf_ssb_import_add_app(file->ssb, file->user, file->name);
sqlite3_finalize(statement);
}
sqlite3_finalize(statement);
}
JS_FreeCString(context, type_string);
}
JS_FreeValue(context, object);
}
if (!is_app_json &&
tf_ssb_db_blob_store(file->ssb, (const uint8_t*)file->data, req->result, id, sizeof(id), &is_new) &&
is_new)
{
printf("Stored %s/%s as %s.\n", file->parent, file->name, id);
}
}
uv_fs_req_cleanup(req);
@ -146,74 +146,178 @@ static void _tf_ssb_import_file_open(uv_fs_t* req)
uv_fs_read(tf_ssb_get_loop(file->ssb), req, file->file, &(uv_buf_t) { .base = file->data, .len = k_ssb_blob_bytes_max }, 1, 0, _tf_ssb_import_file_read);
}
typedef struct _tf_import_t {
static void _tf_ssb_import_register_app(tf_ssb_t* ssb, const char* use, const char* app, const char* id)
{
sqlite3_stmt* statement;
if (sqlite3_prepare(tf_ssb_get_db(file->ssb), "INSERT OR REPLACE INTO properties (id, key, value) VALUES ($1, 'path:' || $2, $3)", -1, &statement, NULL) == SQLITE_OK)
{
((char*)file->name)[strlen(file->name) - strlen(".json")] = '\0';
if (sqlite3_bind_text(statement, 1, file->user, -1, NULL) == SQLITE_OK &&
sqlite3_bind_text(statement, 2, file->name, -1, NULL) == SQLITE_OK &&
sqlite3_bind_text(statement, 3, id, -1, NULL) == SQLITE_OK &&
sqlite3_step(statement) == SQLITE_DONE)
{
if (sqlite3_changes(tf_ssb_get_db(file->ssb)))
{
printf("Registered %s path:%s as %s.\n", file->user, file->name, id);
}
_tf_ssb_import_add_app(file->ssb, file->user, file->name);
}
sqlite3_finalize(statement);
}
}
*/
typedef struct _tf_import_t
{
tf_ssb_t* ssb;
const char* user;
const char* parent;
uv_fs_t req;
int work_left;
} tf_import_t;
static void _tf_ssb_import_scandir(uv_fs_t* req)
static char* _tf_ssb_import_read_file(uv_loop_t* loop, const char* path, size_t* out_size)
{
tf_import_t* import = req->data;
uv_dirent_t ent;
while (uv_fs_scandir_next(req, &ent) == 0)
char* data = NULL;
uv_fs_t req = { 0 };
int handle = uv_fs_open(loop, &req, path, 0, 0, NULL);
if (handle >= 0)
{
size_t len = strlen(import->parent) + strlen(ent.name) + 2;
char* path = tf_malloc(len);
snprintf(path, len, "%s/%s", import->parent, ent.name);
if (ent.type == UV_DIRENT_DIR)
uv_fs_t read_req = { 0 };
data = tf_malloc(k_ssb_blob_bytes_max);
int r = uv_fs_read(loop, &read_req, handle, &(uv_buf_t) { .base = data, .len = k_ssb_blob_bytes_max }, 1, 0, NULL);
if (r >= 0 && r < k_ssb_blob_bytes_max)
{
tf_ssb_import(import->ssb, import->user, path);
data[r] = '\0';
*out_size = r;
}
else
{
size_t size = sizeof(tf_import_file_t) + strlen(import->parent) +1 + strlen(ent.name) + 1;
tf_import_file_t* file = tf_malloc(size);
memset(file, 0, size);
file->ssb = import->ssb;
file->user = import->user;
file->parent = (void*)(file + 1);
file->name = file->parent + strlen(import->parent) + 1;
file->req.data = file;
file->work_left = &import->work_left;
memcpy((char*)file->parent, import->parent, strlen(import->parent) + 1);
memcpy((char*)file->name, ent.name, strlen(ent.name) + 1);
printf("Failed to read %s: %s.\n", path, uv_strerror(r));
}
uv_fs_req_cleanup(&read_req);
import->work_left++;
int r = uv_fs_open(tf_ssb_get_loop(import->ssb), &file->req, path, 0, 0, _tf_ssb_import_file_open);
if (r < 0)
uv_fs_t close_req = { 0 };
r = uv_fs_close(loop, &close_req, handle, NULL);
if (r)
{
printf("Failed to close %s: %s.\n", path, uv_strerror(r));
}
}
else
{
printf("Failed to open %s: %s.\n", path, uv_strerror(handle));
}
uv_fs_req_cleanup(&req);
return data;
}
static void _tf_ssb_import_recursive_add_files(tf_ssb_t* ssb, uv_loop_t* loop, JSContext* context, JSValue files, const char* root, const char* path)
{
uv_fs_t req = { 0 };
int r = uv_fs_scandir(loop, &req, path, 0, NULL);
if (r >= 0)
{
uv_dirent_t ent;
while (uv_fs_scandir_next(&req, &ent) == 0)
{
if (ent.type == UV_DIRENT_FILE)
{
printf("Failed to open %s: %s.\n", path, uv_strerror(r));
tf_free(file);
import->work_left--;
size_t len = strlen(path) + strlen(ent.name) + 2;
char* full_path = tf_malloc(len);
snprintf(full_path, len, "%s/%s", path, ent.name);
size_t size = 0;
char* blob = _tf_ssb_import_read_file(loop, full_path, &size);
char id[k_id_base64_len] = { 0 };
bool is_new = false;
if (tf_ssb_db_blob_store(ssb, (const uint8_t*)blob, size, id, sizeof(id), &is_new) && is_new)
{
printf("Stored %s as %s.\n", full_path, id);
}
JS_SetPropertyStr(context, files, full_path + strlen(root) + 1, JS_NewString(context, id));
tf_free(blob);
tf_free(full_path);
}
}
tf_free(path);
}
import->work_left--;
else
{
printf("Failed to scan directory %s: %s.", path, uv_strerror(r));
}
uv_fs_req_cleanup(&req);
}
static void _tf_ssb_import_app_json(tf_ssb_t* ssb, uv_loop_t* loop, JSContext* context, const char* user, const char* path)
{
uv_fs_t req = { 0 };
int r = uv_fs_open(loop, &req, path, 0, 0, NULL);
if (r >= 0)
{
size_t size = 0;
char* file = _tf_ssb_import_read_file(loop, path, &size);
if (file)
{
JSValue app = JS_ParseJSON(context, file, size, NULL);
if (!tf_util_report_error(context, app))
{
char* dir = tf_strdup(path);
dir[strlen(dir) - strlen(".json")] = '\0';
JSValue files = JS_NewObject(context);
_tf_ssb_import_recursive_add_files(ssb, loop, context, files, dir, dir);
JS_SetPropertyStr(context, app, "files", files);
JSValue json = JS_JSONStringify(context, app, JS_NULL, JS_NULL);
size_t size = 0;
const char* blob = JS_ToCStringLen(context, &size, json);
char id[k_id_base64_len] = { 0 };
if (tf_ssb_db_blob_store(ssb, (const uint8_t*)blob, size, id, sizeof(id), NULL))
{
const char* app = dir;
char* slash = strrchr(dir, '/');
app = slash ? slash + 1 : app;
_tf_ssb_import_add_app(ssb, user, app);
}
JS_FreeCString(context, blob);
JS_FreeValue(context, json);
tf_free(dir);
}
JS_FreeValue(context, app);
tf_free(file);
}
}
else
{
printf("Failed to open %s: %s.\n", path, uv_strerror(r));
}
uv_fs_req_cleanup(&req);
}
void tf_ssb_import(tf_ssb_t* ssb, const char* user, const char* path)
{
tf_import_t import = {
.ssb = ssb,
.user = user,
.parent = path,
.work_left = 1,
};
import.req.data = &import;
uv_fs_t req = { 0 };
sqlite3_busy_timeout(tf_ssb_get_db(ssb), 10000);
int r = uv_fs_scandir(tf_ssb_get_loop(ssb), &import.req, path, 0, _tf_ssb_import_scandir);
if (r)
int r = uv_fs_scandir(tf_ssb_get_loop(ssb), &req, path, 0, NULL);
if (r >= 0)
{
uv_dirent_t ent;
while (uv_fs_scandir_next(&req, &ent) == 0)
{
size_t len = strlen(path) + strlen(ent.name) + 2;
char* full_path = tf_malloc(len);
snprintf(full_path, len, "%s/%s", path, ent.name);
if (strcasecmp(ent.name + strlen(ent.name) - strlen(".json"), ".json") == 0)
{
_tf_ssb_import_app_json(ssb, tf_ssb_get_loop(ssb), tf_ssb_get_context(ssb), user, full_path);
}
tf_free(full_path);
}
}
else
{
printf("Failed to scan directory %s: %s.", path, uv_strerror(r));
}
while (import.work_left > 0)
{
uv_run(tf_ssb_get_loop(ssb), UV_RUN_ONCE);
}
uv_fs_req_cleanup(&import.req);
uv_fs_req_cleanup(&req);
}