Brute force memory tracking.

git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@4186 ed5197a5-7fde-0310-b194-c3ffbd925b24
This commit is contained in:
Cory McWilliams 2023-02-18 21:00:39 +00:00
parent 53cb80ebf7
commit 88c7d91858
8 changed files with 347 additions and 66 deletions

View File

@ -838,6 +838,10 @@ loadSettings().then(function() {
let data = JSON.stringify(getDebug(), null, 2);
response.writeHead(200, {"Content-Type": "application/json; charset=utf-8", "Content-Length": data.length.toString()});
return response.end(data);
} else if (match = /^\/mem$/.exec(request.uri)) {
let data = JSON.stringify(getAllocations(), null, 2);
response.writeHead(200, {"Content-Type": "application/json; charset=utf-8", "Content-Length": data.length.toString()});
return response.end(data);
} else if (request.uri == "/robots.txt") {
return blobHandler(request, response, null, request.uri);
} else if ((match = /^\/.well-known\/(.*)/.exec(request.uri)) && request.uri.indexOf("..") == -1) {

View File

@ -674,6 +674,16 @@ static void _backtrace_error(void* data, const char* message, int errnum)
int main(int argc, char* argv[])
{
bool tracking = true;
for (int i = 1; i < argc; i++)
{
if (strcmp(argv[i], "sandbox") == 0)
{
tracking = false;
}
}
tf_mem_startup(tracking);
g_backtrace_state = backtrace_create_state(
argv[0],
0,
@ -700,6 +710,7 @@ int main(int argc, char* argv[])
}
#endif
int result = 0;
if (argc >= 2)
{
for (int i = 0; i < (int)_countof(k_commands); i++)
@ -707,11 +718,17 @@ int main(int argc, char* argv[])
const command_t* command = &k_commands[i];
if (strcmp(argv[1], command->name) == 0)
{
return command->callback(argv[0], argc - 2, argv + 2);
result = command->callback(argv[0], argc - 2, argv + 2);
goto done;
}
}
return _tf_command_usage(argv[0], argc, argv);
result = _tf_command_usage(argv[0], argc, argv);
}
return _tf_command_run(argv[0], argc - 1, argv + 1);
else
{
result = _tf_command_run(argv[0], argc - 1, argv + 1);
}
done:
tf_mem_shutdown();
return result;
}

211
src/mem.c
View File

@ -1,27 +1,213 @@
#include "mem.h"
#include "util.js.h"
#include <uv.h>
#include <openssl/crypto.h>
#include <quickjs.h>
#include <sqlite3.h>
#include <uv.h>
#include <string.h>
#include <stdbool.h>
static uv_mutex_t s_tracking_mutex;
static bool s_mem_tracking;
static void** s_mem_tracked;
static int s_mem_tracked_count;
static int s_mem_tracked_capacity;
static int64_t s_tf_malloc_size;
static int64_t s_uv_malloc_size;
static int64_t s_tls_malloc_size;
static int64_t s_js_malloc_size;
static int64_t s_sqlite_malloc_size;
extern uint32_t fnv32a(const void* buffer, int length, uint32_t start);
void tf_mem_startup(bool tracking)
{
s_mem_tracking = tracking;
uv_mutex_init(&s_tracking_mutex);
}
void tf_mem_shutdown()
{
s_mem_tracking = false;
free(s_mem_tracked);
s_mem_tracked = NULL;
s_mem_tracked_capacity = 0;
uv_mutex_destroy(&s_tracking_mutex);
}
static void _tf_mem_add_tracked_allocation(void* ptr)
{
if (s_mem_tracking)
{
uv_mutex_lock(&s_tracking_mutex);
if (s_mem_tracked_count + 1 >= s_mem_tracked_capacity)
{
s_mem_tracked_capacity = s_mem_tracked_capacity ? (s_mem_tracked_capacity * 2) : 256;
s_mem_tracked = realloc(s_mem_tracked, sizeof(void*) * s_mem_tracked_capacity);
}
s_mem_tracked[s_mem_tracked_count++] = ptr;
uv_mutex_unlock(&s_tracking_mutex);
}
}
static void _tf_mem_remove_tracked_allocation(void* ptr)
{
if (s_mem_tracking)
{
uv_mutex_lock(&s_tracking_mutex);
for (int i = 0; i < s_mem_tracked_count; i++)
{
if (s_mem_tracked[i] == ptr)
{
s_mem_tracked[i] = s_mem_tracked[--s_mem_tracked_count];
break;
}
}
uv_mutex_unlock(&s_tracking_mutex);
}
}
void tf_mem_walk_allocations(void (*callback)(void* ptr, size_t size, int frames_count, void* const* frames, void* user_data), void* user_data)
{
uv_mutex_lock(&s_tracking_mutex);
for (int i = 0; i < s_mem_tracked_count; i++)
{
size_t size = 0;
int frames_count = 0;
void* frames[32];
memcpy(&size, s_mem_tracked[i], sizeof(size));
if (s_mem_tracking)
{
memcpy(&frames_count, (void*)((intptr_t)s_mem_tracked[i] + sizeof(size_t) + size), sizeof(frames_count));
if (frames_count)
{
memcpy(frames, (void*)((intptr_t)s_mem_tracked[i] + sizeof(size_t) + size + sizeof(frames_count)), sizeof(void*) * frames_count);
}
}
callback(
(void*)((intptr_t)s_mem_tracked[i] + sizeof(size_t)),
size,
frames_count,
frames_count ? frames : NULL,
user_data);
}
uv_mutex_unlock(&s_tracking_mutex);
}
typedef struct _summary_t
{
tf_mem_allocation_t* allocations;
int count;
int capacity;
} summary_t;
static int _tf_mem_hash_stack_compare(const void* a, const void* b)
{
const tf_mem_allocation_t* aa = a;
const tf_mem_allocation_t* ab = b;
if (aa->stack_hash != ab->stack_hash)
{
return aa->stack_hash < ab->stack_hash ? -1 : 1;
}
if (aa->frames_count != ab->frames_count)
{
return aa->frames_count < ab->frames_count ? -1 : 1;
}
return memcmp(aa->frames, ab->frames, sizeof(void*) * aa->frames_count);
}
static int _tf_mem_size_compare(const void* a, const void* b)
{
const tf_mem_allocation_t* aa = a;
const tf_mem_allocation_t* ab = b;
if (aa->size > ab->size)
{
return -1;
}
else if (ab->size > aa->size)
{
return 1;
}
return 0;
}
static void _tf_mem_summarize(void* ptr, size_t size, int frames_count, void* const* frames, void* user_data)
{
summary_t* summary = user_data;
tf_mem_allocation_t allocation =
{
.stack_hash = fnv32a(frames, sizeof(void*) * frames_count, 0),
.count = 1,
.size = size,
.frames_count = frames_count,
};
memcpy(allocation.frames, frames, sizeof(void*) * frames_count);
int index = tf_util_insert_index(&allocation, summary->allocations, summary->count, sizeof(tf_mem_allocation_t), _tf_mem_hash_stack_compare);
if (index < summary->count &&
allocation.stack_hash == summary->allocations[index].stack_hash &&
allocation.frames_count == summary->allocations[index].frames_count &&
memcmp(frames, summary->allocations[index].frames, sizeof(void*) * frames_count) == 0)
{
summary->allocations[index].count++;
summary->allocations[index].size += size;
}
else
{
if (summary->count + 1 >= summary->capacity)
{
summary->capacity = summary->capacity ? summary->capacity * 2 : 256;
summary->allocations = realloc(summary->allocations, sizeof(tf_mem_allocation_t) * summary->capacity);
}
if (index < summary->count)
{
memmove(summary->allocations + index + 1, summary->allocations + index, sizeof(tf_mem_allocation_t) * (summary->count - index));
}
summary->allocations[index] = allocation;
summary->count++;
}
}
tf_mem_allocation_t* tf_mem_summarize_allocations(int* out_count)
{
summary_t summary = { 0 };
tf_mem_walk_allocations(_tf_mem_summarize, &summary);
qsort(summary.allocations, summary.count, sizeof(tf_mem_allocation_t), _tf_mem_size_compare);
*out_count = summary.count;
tf_mem_allocation_t* result = tf_malloc(sizeof(tf_mem_allocation_t) * summary.count);
memcpy(result, summary.allocations, sizeof(tf_mem_allocation_t) * summary.count);
free(summary.allocations);
return result;
}
static void* _tf_alloc(int64_t* total, size_t size)
{
void* ptr = malloc(size + sizeof(size_t));
size_t overhead = sizeof(size_t);
void* buffer[32];
int count = 0;
if (s_mem_tracking)
{
count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
overhead += sizeof(count) + sizeof(void*) * count;
}
void* ptr = malloc(size + overhead);
if (ptr)
{
__atomic_add_fetch(total, size, __ATOMIC_RELAXED);
memcpy(ptr, &size, sizeof(size_t));
if (count)
{
memcpy((void*)((intptr_t)ptr + sizeof(size_t) + size), &count, sizeof(count));
memcpy((void*)((intptr_t)ptr + sizeof(size_t) + size + sizeof(count)), buffer, sizeof(void*) * count);
}
_tf_mem_add_tracked_allocation(ptr);
return (void*)((intptr_t)ptr + sizeof(size_t));
}
else
@ -32,6 +218,15 @@ static void* _tf_alloc(int64_t* total, size_t size)
static void* _tf_realloc(int64_t* total, void* ptr, size_t size)
{
void* buffer[32];
int count = 0;
size_t overhead = sizeof(size_t);
if (s_mem_tracking)
{
count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
overhead += sizeof(count) + sizeof(void*) * count;
}
void* old_ptr = ptr ? (void*)((intptr_t)ptr - sizeof(size_t)) : NULL;
size_t old_size = 0;
if (old_ptr)
@ -41,16 +236,27 @@ static void* _tf_realloc(int64_t* total, void* ptr, size_t size)
void* new_ptr = NULL;
if (old_ptr && !size)
{
_tf_mem_remove_tracked_allocation(old_ptr);
free(old_ptr);
}
else
{
new_ptr = realloc(old_ptr, size + sizeof(size_t));
if (old_ptr)
{
_tf_mem_remove_tracked_allocation(old_ptr);
}
new_ptr = realloc(old_ptr, size + overhead);
}
if (new_ptr)
{
__atomic_add_fetch(total, (int64_t)size - (int64_t)old_size, __ATOMIC_RELAXED);
memcpy(new_ptr, &size, sizeof(size_t));
if (count)
{
memcpy((void*)((intptr_t)new_ptr + sizeof(size_t) + size), &count, sizeof(count));
memcpy((void*)((intptr_t)new_ptr + sizeof(size_t) + size + sizeof(count)), buffer, sizeof(void*) * count);
}
_tf_mem_add_tracked_allocation(new_ptr);
return (void*)((intptr_t)new_ptr + sizeof(size_t));
}
else
@ -68,6 +274,7 @@ static void _tf_free(int64_t* total, void* ptr)
size_t size = 0;
memcpy(&size, old_ptr, sizeof(size_t));
__atomic_sub_fetch(total, size, __ATOMIC_RELAXED);
_tf_mem_remove_tracked_allocation(old_ptr);
free(old_ptr);
}
}

View File

@ -1,9 +1,14 @@
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
typedef struct JSMallocFunctions JSMallocFunctions;
void tf_mem_startup(bool tracking);
void tf_mem_shutdown();
void tf_mem_replace_uv_allocator();
size_t tf_mem_get_uv_malloc_size();
@ -24,3 +29,16 @@ void* tf_resize_vec(void* ptr, size_t size);
void tf_get_js_malloc_functions(JSMallocFunctions* out);
size_t tf_mem_get_js_malloc_size();
void tf_mem_walk_allocations(void (*callback)(void* ptr, size_t size, int frames_count, void* const* frames, void* user_data), void* user_data);
typedef struct _tf_mem_allocation_t
{
uint32_t stack_hash;
int count;
size_t size;
void* frames[32];
int frames_count;
} tf_mem_allocation_t;
tf_mem_allocation_t* tf_mem_summarize_allocations(int* out_count);

View File

@ -24,8 +24,6 @@
#include <time.h>
#include <uv.h>
#include <backtrace.h>
#ifndef _WIN32
#ifndef __ANDROID__
#include <execinfo.h>
@ -735,63 +733,17 @@ void tf_ssb_connection_rpc_send_json(tf_ssb_connection_t* connection, uint8_t fl
JS_FreeValue(context, json);
}
static int _tf_ssb_backtrace_callback(void* data, uintptr_t pc, const char* filename, int line_number, const char* function)
{
char** stack = data;
char line[256];
int length = snprintf(line, sizeof(line), "%p %s:%d %s\n", (void*)pc, filename, line_number, function);
int current = *stack ? strlen(*stack) : 0;
*stack = tf_resize_vec(*stack, current + length + 1);
memcpy(*stack + current, line, length + 1);
return 0;
}
static void _tf_ssb_backtrace_error(void* data, const char* message, int error)
{
char** stack = data;
int length = strlen(message);
if (message)
{
int current = *stack ? strlen(*stack) : 0;
*stack = tf_resize_vec(*stack, current + length + 1);
memcpy(*stack + current, message, length + 1);
}
}
static char* _tf_ssb_backtrace_string()
{
extern struct backtrace_state* g_backtrace_state;
int count = 0;
void* buffer[32];
char* string = NULL;
#ifdef _WIN32
count = CaptureStackBackTrace(0, sizeof(buffer) / sizeof(*buffer), buffer, NULL);
#elif !defined(__ANDROID__)
count = backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
#endif
for (int i = 0; i < count; i++)
{
backtrace_pcinfo(
g_backtrace_state,
(uintptr_t)buffer[i],
_tf_ssb_backtrace_callback,
_tf_ssb_backtrace_error,
&string);
}
return string;
}
void tf_ssb_connection_rpc_send_error(tf_ssb_connection_t* connection, uint8_t flags, int32_t request_number, const char* error)
{
JSContext* context = connection->ssb->context;
JSValue message = JS_NewObject(context);
char* stack = _tf_ssb_backtrace_string();
const char* stack = tf_util_backtrace_string();
JS_SetPropertyStr(context, message, "name", JS_NewString(context, "Error"));
JS_SetPropertyStr(context, message, "stack", JS_NewString(context, stack));
JS_SetPropertyStr(context, message, "message", JS_NewString(context, error));
tf_ssb_connection_rpc_send_json(connection, ((flags & k_ssb_rpc_flag_stream) ? (k_ssb_rpc_flag_stream | k_ssb_rpc_flag_end_error) : 0), request_number, message, NULL, NULL, NULL);
JS_FreeValue(context, message);
tf_free(stack);
tf_free((void*)stack);
}
void tf_ssb_connection_rpc_send_error_method_not_allowed(tf_ssb_connection_t* connection, uint8_t flags, int32_t request_number)

View File

@ -29,9 +29,6 @@
#include <backtrace.h>
#ifndef _WIN32
#ifndef __ANDROID__
#include <execinfo.h>
#endif
#include <unistd.h>
#endif
@ -827,6 +824,29 @@ static JSValue _tf_task_getDebug(JSContext* context, JSValueConst this_val, int
return result;
}
static JSValue _tf_task_getAllocations(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
{
JSValue result = JS_NewObject(context);
int count = 0;
tf_mem_allocation_t* allocation_info = tf_mem_summarize_allocations(&count);
JSValue allocations = JS_NewArray(context);
JS_SetPropertyStr(context, result, "allocations", allocations);
for (int i = 0; i < count; i++)
{
JSValue allocation = JS_NewObject(context);
JS_SetPropertyStr(context, allocation, "size", JS_NewInt64(context, allocation_info[i].size));
JS_SetPropertyStr(context, allocation, "count", JS_NewInt32(context, allocation_info[i].count));
const char* stack = tf_util_backtrace_to_string(allocation_info[i].frames, allocation_info[i].frames_count);
JS_SetPropertyStr(context, allocation, "stack", JS_NewString(context, stack));
tf_free((void*)stack);
JS_SetPropertyUint32(context, allocations, i, allocation);
}
return result;
}
static JSValue _tf_task_disconnectionsDebug(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
{
tf_task_t* task = JS_GetContextOpaque(context);
@ -1152,15 +1172,9 @@ JSValue tf_task_allocate_promise(tf_task_t* task, promiseid_t* out_promise)
size_t length = 0;
const char* stack = JS_ToCStringLen(task->_context, &length, stack_value);
uint32_t stack_hash = fnv32a((const void*)stack, (int)length, 0);
int count = 0;
void* buffer[32];
#ifdef _WIN32
count = CaptureStackBackTrace(0, sizeof(buffer) / sizeof(*buffer), buffer, NULL);
int count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
stack_hash = fnv32a((const void*)buffer, sizeof(void*) * count, stack_hash);
#elif !defined(__ANDROID__)
count = backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
stack_hash = fnv32a((const void*)buffer, sizeof(void*) * count, stack_hash);
#endif
_add_promise_stack(task, stack_hash, stack, buffer, count);
JS_FreeCString(task->_context, stack);
JS_FreeValue(task->_context, stack_value);
@ -1588,6 +1602,7 @@ void tf_task_activate(tf_task_t* task)
JS_SetPropertyStr(context, global, "trace", JS_NewCFunction(context, _tf_task_trace, "trace", 1));
JS_SetPropertyStr(context, global, "getStats", JS_NewCFunction(context, _tf_task_getStats, "getStats", 0));
JS_SetPropertyStr(context, global, "getDebug", JS_NewCFunction(context, _tf_task_getDebug, "getDebug", 0));
JS_SetPropertyStr(context, global, "getAllocations", JS_NewCFunction(context, _tf_task_getAllocations, "getAllocations", 0));
JS_SetPropertyStr(context, global, "disconnectionsDebug", JS_NewCFunction(context, _tf_task_disconnectionsDebug, "disconnectionsDebug", 0));
}
else

View File

@ -4,6 +4,7 @@
#include "task.h"
#include "trace.h"
#include <backtrace.h>
#include <openssl/crypto.h>
#include <openssl/sha.h>
#include <picohttpparser.h>
@ -13,6 +14,12 @@
#include <string.h>
#ifndef _WIN32
#ifndef __ANDROID__
#include <execinfo.h>
#endif
#endif
static JSValue _util_utf8_encode(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
{
size_t length = 0;
@ -417,3 +424,60 @@ size_t tf_base64_decode(const char* source, size_t source_length, uint8_t* out,
size_t actual_length = 0;
return sodium_base642bin(out, out_length, source, source_length, NULL, &actual_length, NULL, sodium_base64_VARIANT_ORIGINAL) == 0 ? actual_length : 0;
}
static int _tf_util_backtrace_callback(void* data, uintptr_t pc, const char* filename, int line_number, const char* function)
{
char** stack = data;
char line[256];
int length = snprintf(line, sizeof(line), "%p %s:%d %s\n", (void*)pc, filename, line_number, function);
int current = *stack ? strlen(*stack) : 0;
*stack = tf_resize_vec(*stack, current + length + 1);
memcpy(*stack + current, line, length + 1);
return 0;
}
static void _tf_util_backtrace_error(void* data, const char* message, int error)
{
char** stack = data;
int length = strlen(message);
if (message)
{
int current = *stack ? strlen(*stack) : 0;
*stack = tf_resize_vec(*stack, current + length + 1);
memcpy(*stack + current, message, length + 1);
}
}
const char* tf_util_backtrace_to_string(void* const* buffer, int count)
{
extern struct backtrace_state* g_backtrace_state;
char* string = NULL;
for (int i = 0; i < count; i++)
{
backtrace_pcinfo(
g_backtrace_state,
(uintptr_t)buffer[i],
_tf_util_backtrace_callback,
_tf_util_backtrace_error,
&string);
}
return string;
}
const char* tf_util_backtrace_string()
{
void* buffer[32];
int count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
return tf_util_backtrace_to_string(buffer, count);
}
int tf_util_backtrace(void** buffer, int count)
{
#ifdef _WIN32
return CaptureStackBackTrace(0, count, buffer, NULL);
#elif !defined(__ANDROID__)
return backtrace(buffer, count);
#else
return 0;
#endif
}

View File

@ -15,3 +15,7 @@ JSValue tf_util_new_uint8_array(JSContext* context, const uint8_t* data, size_t
size_t tf_base64_encode(const uint8_t* source, size_t source_length, char* out, size_t out_length);
size_t tf_base64_decode(const char* source, size_t source_length, uint8_t* out, size_t out_length);
int tf_util_backtrace(void** buffer, int count);
const char* tf_util_backtrace_to_string(void* const* buffer, int count);
const char* tf_util_backtrace_string();