tildefriends/src/mem.c

#include "mem.h"

#include "util.js.h"

#include "quickjs.h"
#include "sqlite3.h"
#include "uv.h"

#include <openssl/crypto.h>

#include <stdbool.h>
#include <string.h>

typedef struct _tf_mem_node_t tf_mem_node_t;

static uv_mutex_t s_tracking_mutex;
static bool s_mem_tracking;
static tf_mem_node_t* s_mem_tracked;
static int64_t s_tf_malloc_size;
static int64_t s_uv_malloc_size;
static int64_t s_tls_malloc_size;
static int64_t s_js_malloc_size;
static int64_t s_sqlite_malloc_size;

extern uint32_t fnv32a(const void* buffer, int length, uint32_t start);

static size_t _tf_mem_round_up(size_t size)
{
	return (size + 7) & ~7;
}

void tf_mem_startup(bool tracking)
{
	s_mem_tracking = tracking;
	uv_mutex_init(&s_tracking_mutex);
}

void tf_mem_shutdown()
{
	s_mem_tracking = false;
	s_mem_tracked = NULL;
	uv_mutex_destroy(&s_tracking_mutex);
}

typedef struct _tf_mem_node_t
{
	void* ptr;
	tf_mem_node_t* next;
	tf_mem_node_t* previous;
	int frames_count;
	void* frames[];
} tf_mem_node_t;

static void _tf_mem_add_tracked_allocation(tf_mem_node_t* node)
{
	if (s_mem_tracking)
	{
		uv_mutex_lock(&s_tracking_mutex);
		if (s_mem_tracked)
		{
			node->next = s_mem_tracked;
			node->previous = s_mem_tracked->previous;
			s_mem_tracked->previous->next = node;
			s_mem_tracked->previous = node;
		}
		else
		{
			s_mem_tracked = node;
			node->next = node->previous = node;
		}
		uv_mutex_unlock(&s_tracking_mutex);
	}
}

static void _tf_mem_remove_tracked_allocation(tf_mem_node_t* node)
{
	if (s_mem_tracking)
	{
		uv_mutex_lock(&s_tracking_mutex);
		tf_mem_node_t* previous = node->previous;
		tf_mem_node_t* next = node->next;
		next->previous = previous;
		previous->next = next;
		node->next = NULL;
		node->previous = NULL;
		if (node == s_mem_tracked)
		{
			s_mem_tracked = next != node ? next : NULL;
		}
		uv_mutex_unlock(&s_tracking_mutex);
	}
}

void tf_mem_walk_allocations(void (*callback)(void* ptr, size_t size, int frames_count, void* const* frames, void* user_data), void* user_data)
{
	uv_mutex_lock(&s_tracking_mutex);
	for (tf_mem_node_t* node = s_mem_tracked ? s_mem_tracked->next : NULL; node; node = node->next)
	{
		size_t size = 0;
		void* frames[32];
		memcpy(&size, node->ptr, sizeof(size));
		if (s_mem_tracking)
		{
			if (node->frames_count)
			{
				memcpy(frames, node->frames, sizeof(void*) * node->frames_count);
			}
		}
		callback((void*)((intptr_t)node->ptr + sizeof(size_t)), size, node->frames_count, node->frames_count ? frames : NULL, user_data);
		if (node == s_mem_tracked)
		{
			break;
		}
	}
	uv_mutex_unlock(&s_tracking_mutex);
}

typedef struct _summary_t
{
	tf_mem_allocation_t* allocations;
	int count;
	int capacity;
} summary_t;

static int _tf_mem_hash_stack_compare(const void* a, const void* b)
{
	const tf_mem_allocation_t* aa = a;
	const tf_mem_allocation_t* ab = b;
	if (aa->stack_hash != ab->stack_hash)
	{
		return aa->stack_hash < ab->stack_hash ? -1 : 1;
	}
	if (aa->frames_count != ab->frames_count)
	{
		return aa->frames_count < ab->frames_count ? -1 : 1;
	}
	return memcmp(aa->frames, ab->frames, sizeof(void*) * aa->frames_count);
}

static int _tf_mem_size_compare(const void* a, const void* b)
{
	const tf_mem_allocation_t* aa = a;
	const tf_mem_allocation_t* ab = b;
	if (aa->size > ab->size)
	{
		return -1;
	}
	else if (ab->size > aa->size)
	{
		return 1;
	}
	return 0;
}

static void _tf_mem_summarize(void* ptr, size_t size, int frames_count, void* const* frames, void* user_data)
{
	summary_t* summary = user_data;
	tf_mem_allocation_t allocation = {
		.stack_hash = fnv32a(frames, sizeof(void*) * frames_count, 0),
		.count = 1,
		.size = size,
		.frames_count = frames_count,
	};
	memcpy(allocation.frames, frames, sizeof(void*) * frames_count);

	int index = tf_util_insert_index(&allocation, summary->allocations, summary->count, sizeof(tf_mem_allocation_t), _tf_mem_hash_stack_compare);
	if (index < summary->count && allocation.stack_hash == summary->allocations[index].stack_hash && allocation.frames_count == summary->allocations[index].frames_count &&
		memcmp(frames, summary->allocations[index].frames, sizeof(void*) * frames_count) == 0)
	{
		summary->allocations[index].count++;
		summary->allocations[index].size += size;
	}
	else
	{
		if (summary->count + 1 >= summary->capacity)
		{
			summary->capacity = summary->capacity ? summary->capacity * 2 : 256;
			summary->allocations = realloc(summary->allocations, sizeof(tf_mem_allocation_t) * summary->capacity);
		}
		if (index < summary->count)
		{
			memmove(summary->allocations + index + 1, summary->allocations + index, sizeof(tf_mem_allocation_t) * (summary->count - index));
		}
		summary->allocations[index] = allocation;
		summary->count++;
	}
}

tf_mem_allocation_t* tf_mem_summarize_allocations(int* out_count)
{
	summary_t summary = { 0 };
	tf_mem_walk_allocations(_tf_mem_summarize, &summary);
	if (summary.count)
	{
		qsort(summary.allocations, summary.count, sizeof(tf_mem_allocation_t), _tf_mem_size_compare);
	}
	*out_count = summary.count;
	tf_mem_allocation_t* result = tf_malloc(sizeof(tf_mem_allocation_t) * summary.count);
	if (result && summary.count)
	{
		memcpy(result, summary.allocations, sizeof(tf_mem_allocation_t) * summary.count);
	}
	free(summary.allocations);
	return result;
}

#if defined(__GNUC__) && !defined(__APPLE__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wanalyzer-malloc-leak"
#endif
static void* _tf_alloc(int64_t* total, size_t size)
{
	size_t overhead = sizeof(size_t);
	void* buffer[32];
	int count = 0;
	if (s_mem_tracking)
	{
		count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
		overhead += sizeof(tf_mem_node_t) + sizeof(void*) * count;
	}

	size_t rounded_up_size = _tf_mem_round_up(size);
	void* ptr = malloc(rounded_up_size + overhead);
	if (ptr)
	{
		__atomic_add_fetch(total, size, __ATOMIC_RELAXED);
		memcpy(ptr, &size, sizeof(size_t));
		if (s_mem_tracking)
		{
			tf_mem_node_t* node = (tf_mem_node_t*)((intptr_t)ptr + sizeof(size_t) + rounded_up_size);
			memcpy(node, &(tf_mem_node_t) { .ptr = ptr, .frames_count = count }, sizeof(tf_mem_node_t));
			if (count)
			{
				memcpy(node->frames, buffer, sizeof(void*) * count);
			}
			_tf_mem_add_tracked_allocation(node);
		}
		return (void*)((intptr_t)ptr + sizeof(size_t));
	}
	else
	{
		return NULL;
	}
}

static void* _tf_realloc(int64_t* total, void* ptr, size_t size)
{
	if (!ptr && !size)
	{
		return NULL;
	}

	void* buffer[32];
	int count = 0;
	size_t overhead = sizeof(size_t);
	if (s_mem_tracking)
	{
		count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
		overhead += sizeof(tf_mem_node_t) + sizeof(void*) * count;
	}

	void* old_ptr = ptr ? (void*)((intptr_t)ptr - sizeof(size_t)) : NULL;
	size_t old_size = 0;
	if (old_ptr)
	{
		memcpy(&old_size, old_ptr, sizeof(size_t));
	}
	void* new_ptr = NULL;
	tf_mem_node_t* node = (void*)((intptr_t)ptr + _tf_mem_round_up(old_size));
	size_t rounded_up_size = _tf_mem_round_up(size);
	if (old_ptr && !size)
	{
		_tf_mem_remove_tracked_allocation(node);
		free(old_ptr);
	}
	else
	{
		if (old_ptr)
		{
			_tf_mem_remove_tracked_allocation(node);
		}
		new_ptr = realloc(old_ptr, rounded_up_size + overhead);
	}
	if (new_ptr)
	{
		__atomic_add_fetch(total, (int64_t)size - (int64_t)old_size, __ATOMIC_RELAXED);
		memcpy(new_ptr, &size, sizeof(size_t));
		if (s_mem_tracking)
		{
			tf_mem_node_t* node = (tf_mem_node_t*)((intptr_t)new_ptr + sizeof(size_t) + rounded_up_size);
			memcpy(node, &(tf_mem_node_t) { .ptr = new_ptr, .frames_count = count }, sizeof(tf_mem_node_t));
			if (count)
			{
				memcpy(node->frames, buffer, sizeof(void*) * count);
			}
			_tf_mem_add_tracked_allocation(node);
		}
		return (void*)((intptr_t)new_ptr + sizeof(size_t));
	}
	else
	{
		__atomic_sub_fetch(total, old_size, __ATOMIC_RELAXED);
		return NULL;
	}
}
#if defined(__GNUC__) && !defined(__APPLE__)
#pragma GCC diagnostic pop
#endif

static void _tf_free(int64_t* total, void* ptr)
{
	if (ptr)
	{
		void* old_ptr = (void*)((intptr_t)ptr - sizeof(size_t));
		size_t size = 0;
		memcpy(&size, old_ptr, sizeof(size_t));
		tf_mem_node_t* node = (void*)((intptr_t)ptr + _tf_mem_round_up(size));
		__atomic_sub_fetch(total, size, __ATOMIC_RELAXED);
		_tf_mem_remove_tracked_allocation(node);
		free(old_ptr);
	}
}

static void* _tf_uv_alloc(size_t size)
{
	return _tf_alloc(&s_uv_malloc_size, size);
}

static void* _tf_uv_realloc(void* ptr, size_t size)
{
	return _tf_realloc(&s_uv_malloc_size, ptr, size);
}

#if defined(__GNUC__) && !defined(__APPLE__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wanalyzer-malloc-leak"
#endif
static void* _tf_uv_calloc(size_t nmemb, size_t size)
{
	size_t total_size = nmemb * size;
	size_t rounded_up_size = _tf_mem_round_up(total_size);
	size_t overhead = sizeof(size_t);
	void* buffer[32];
	int count = 0;
	if (s_mem_tracking)
	{
		count = tf_util_backtrace(buffer, sizeof(buffer) / sizeof(*buffer));
		overhead += sizeof(tf_mem_node_t) + sizeof(void*) * count;
	}
	void* ptr = calloc(1, rounded_up_size + overhead);
	if (ptr)
	{
		__atomic_add_fetch(&s_uv_malloc_size, total_size, __ATOMIC_RELAXED);
		memcpy(ptr, &total_size, sizeof(size_t));
		if (s_mem_tracking)
		{
			tf_mem_node_t* node = (tf_mem_node_t*)((intptr_t)ptr + sizeof(size_t) + rounded_up_size);
			memcpy(node, &(tf_mem_node_t) { .ptr = ptr, .frames_count = count }, sizeof(tf_mem_node_t));
			if (count)
			{
				memcpy(node->frames, buffer, sizeof(void*) * count);
			}
			_tf_mem_add_tracked_allocation(node);
		}
		return (void*)((intptr_t)ptr + sizeof(size_t));
	}
	else
	{
		return NULL;
	}
}
#if defined(__GNUC__) && !defined(__APPLE__)
#pragma GCC diagnostic pop
#endif

static void _tf_uv_free(void* ptr)
{
	_tf_free(&s_uv_malloc_size, ptr);
}

void tf_mem_replace_uv_allocator()
{
	uv_replace_allocator(_tf_uv_alloc, _tf_uv_realloc, _tf_uv_calloc, _tf_uv_free);
}

size_t tf_mem_get_uv_malloc_size()
{
	return s_uv_malloc_size;
}

#if defined(__OpenBSD__)
static void* _tf_tls_alloc(size_t size)
#else
static void* _tf_tls_alloc(size_t size, const char* file, int line)
#endif
{
	return _tf_alloc(&s_tls_malloc_size, size);
}

#if defined(__OpenBSD__)
static void* _tf_tls_realloc(void* ptr, size_t size)
#else
static void* _tf_tls_realloc(void* ptr, size_t size, const char* file, int line)
#endif
{
	return _tf_realloc(&s_tls_malloc_size, ptr, size);
}

#if defined(__OpenBSD__)
static void _tf_tls_free(void* ptr)
#else
static void _tf_tls_free(void* ptr, const char* file, int line)
#endif
{
	_tf_free(&s_tls_malloc_size, ptr);
}

void tf_mem_replace_tls_allocator()
{
	CRYPTO_set_mem_functions(_tf_tls_alloc, _tf_tls_realloc, _tf_tls_free);
}

size_t tf_mem_get_tls_malloc_size()
{
	return s_tls_malloc_size;
}

void* tf_malloc(size_t size)
{
	return _tf_alloc(&s_tf_malloc_size, size);
}

void* tf_realloc(void* ptr, size_t size)
{
	return _tf_realloc(&s_tf_malloc_size, ptr, size);
}

void tf_free(void* ptr)
{
	_tf_free(&s_tf_malloc_size, ptr);
}

char* tf_strdup(const char* string)
{
	if (!string)
	{
		return NULL;
	}
	size_t len = strlen(string);
	char* buffer = tf_malloc(len + 1);
	if (buffer)
	{
		memcpy(buffer, string, len + 1);
	}
	return buffer;
}

void* tf_resize_vec(void* ptr, size_t size)
{
	void* alloc_ptr = ptr ? (void*)((intptr_t)ptr - sizeof(size_t)) : NULL;
	size_t alloc_size = 0;
	if (alloc_ptr)
	{
		memcpy(&alloc_size, alloc_ptr, sizeof(size_t));
	}

	if ((alloc_size >= 16 * size + sizeof(size_t)) || !size)
	{
		/* If we've dropped significantly in size or are freeing, resize down. */
		return tf_realloc(ptr, size);
	}
	else if (alloc_size >= size + sizeof(size_t))
	{
		/* Otherwise, if we're big enough, stay the same size. */
		return ptr;
	}
	else
	{
		/* If we need to grow, overallocate 2x to give room to continue growing. */
		return tf_realloc(ptr, size * 2);
	}
}

size_t tf_mem_get_tf_malloc_size()
{
	return s_tf_malloc_size;
}

static void* _tf_js_malloc(JSMallocState* state, size_t size)
{
	int64_t delta = 0;
	void* ptr = _tf_alloc(&delta, size);
	if (ptr)
	{
		__atomic_add_fetch(&s_js_malloc_size, delta, __ATOMIC_RELAXED);
		state->malloc_count++;
		state->malloc_size += delta;
	}
	return ptr;
}

static void _tf_js_free(JSMallocState* state, void* ptr)
{
	if (ptr)
	{
		int64_t delta = 0;
		_tf_free(&delta, ptr);
		__atomic_add_fetch(&s_js_malloc_size, delta, __ATOMIC_RELAXED);
		state->malloc_count--;
		state->malloc_size += delta;
	}
}

static void* _tf_js_realloc(JSMallocState* state, void* ptr, size_t size)
{
	int64_t delta = 0;
	void* result = _tf_realloc(&delta, ptr, size);
	__atomic_add_fetch(&s_js_malloc_size, delta, __ATOMIC_RELAXED);
	state->malloc_count += (ptr ? -1 : 0) + (result ? 1 : 0);
	state->malloc_size += delta;
	return result;
}

static size_t _tf_js_malloc_usable_size(const void* ptr)
{
	void* old_ptr = ptr ? (void*)((intptr_t)ptr - sizeof(size_t)) : NULL;
	size_t old_size = 0;
	if (old_ptr)
	{
		memcpy(&old_size, old_ptr, sizeof(size_t));
	}
	return old_size;
}

void tf_get_js_malloc_functions(JSMallocFunctions* out)
{
	*out = (JSMallocFunctions) {
		.js_malloc = _tf_js_malloc,
		.js_free = _tf_js_free,
		.js_realloc = _tf_js_realloc,
		.js_malloc_usable_size = _tf_js_malloc_usable_size,
	};
}

size_t tf_mem_get_js_malloc_size()
{
	return s_js_malloc_size;
}

static void* _tf_sqlite_malloc(int size)
{
	return _tf_alloc(&s_sqlite_malloc_size, size);
}

static void _tf_sqlite_free(void* ptr)
{
	_tf_free(&s_sqlite_malloc_size, ptr);
}

static void* _tf_sqlite_realloc(void* ptr, int size)
{
	return _tf_realloc(&s_sqlite_malloc_size, ptr, size);
}

static int _tf_sqlite_size(void* ptr)
{
	void* old_ptr = ptr ? (void*)((intptr_t)ptr - sizeof(size_t)) : NULL;
	size_t old_size = 0;
	if (old_ptr)
	{
		memcpy(&old_size, old_ptr, sizeof(size_t));
	}
	return (int)old_size;
}

static int _tf_sqlite_roundup(int size)
{
	return (size + 7) & ~7;
}

static int _tf_sqlite_init(void* user_data)
{
	return SQLITE_OK;
}

static void _tf_sqlite_shutdown(void* user_data)
{
}

void tf_mem_replace_sqlite_allocator()
{
	sqlite3_mem_methods methods = {
		.xMalloc = _tf_sqlite_malloc,
		.xFree = _tf_sqlite_free,
		.xRealloc = _tf_sqlite_realloc,
		.xSize = _tf_sqlite_size,
		.xRoundup = _tf_sqlite_roundup,
		.xInit = _tf_sqlite_init,
		.xShutdown = _tf_sqlite_shutdown,
	};
	sqlite3_config(SQLITE_CONFIG_MALLOC, &methods);
}

size_t tf_mem_get_sqlite_malloc_size()
{
	return s_sqlite_malloc_size;
}