Dqn/dqn_memory.cpp

571 lines
21 KiB
C++

// NOTE: [$ALLO] Dqn_Allocator =====================================================================
DQN_API void *Dqn_Allocator_Alloc(Dqn_Allocator allocator, size_t size, uint8_t align, Dqn_ZeroMem zero_mem)
{
void *result = NULL;
if (allocator.alloc) {
result = allocator.alloc(size, align, zero_mem, allocator.user_context);
} else {
result = DQN_ALLOC(size);
}
return result;
}
DQN_API void Dqn_Allocator_Dealloc(Dqn_Allocator allocator, void *ptr, size_t size)
{
if (allocator.dealloc) {
allocator.dealloc(ptr, size, allocator.user_context);
} else {
DQN_DEALLOC(ptr, size);
}
}
// NOTE: [$VMEM] Dqn_VMem ==========================================================================
DQN_FILE_SCOPE uint32_t Dqn_VMem_ConvertPageToOSFlags_(uint32_t protect)
{
DQN_ASSERT((protect & ~(Dqn_VMemPage_ReadWrite | Dqn_VMemPage_Guard)) == 0);
DQN_ASSERT(protect != 0);
uint32_t result = 0;
#if defined(DQN_OS_WIN32)
if (protect & Dqn_VMemPage_NoAccess) {
result = PAGE_NOACCESS;
} else {
if (protect & Dqn_VMemPage_ReadWrite) {
result = PAGE_READWRITE;
} else if (protect & Dqn_VMemPage_Read) {
result = PAGE_READONLY;
} else if (protect & Dqn_VMemPage_Write) {
Dqn_Log_WarningF("Windows does not support write-only pages, granting read+write access");
result = PAGE_READWRITE;
}
}
if (protect & Dqn_VMemPage_Guard)
result |= PAGE_GUARD;
DQN_ASSERTF(result != PAGE_GUARD, "Page guard is a modifier, you must also specify a page permission like read or/and write");
#else
if (protect & (Dqn_VMemPage_NoAccess | Dqn_VMemPage_Guard)) {
result = PROT_NONE;
} else {
if (protect & Dqn_VMemPage_Read)
result = PROT_READ;
if (protect & Dqn_VMemPage_Write)
result = PROT_WRITE;
}
#endif
return result;
}
DQN_API void *Dqn_VMem_Reserve(Dqn_usize size, Dqn_VMemCommit commit, uint32_t page_flags)
{
unsigned long os_page_flags = Dqn_VMem_ConvertPageToOSFlags_(page_flags);
#if defined(DQN_OS_WIN32)
unsigned long flags = MEM_RESERVE | (commit == Dqn_VMemCommit_Yes ? MEM_COMMIT : 0);
void *result = VirtualAlloc(nullptr, size, flags, os_page_flags);
#elif defined(DQN_OS_UNIX)
if (commit == Dqn_VMemCommit_Yes)
os_page_flags |= (PROT_READ | PROT_WRITE);
void *result = mmap(nullptr, size, os_page_flags, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (result == MAP_FAILED)
result = nullptr;
#else
#error "Missing implementation for Dqn_VMem_Reserve"
#endif
Dqn_Debug_TrackAlloc(result, size, (page_flags & Dqn_VMemPage_AllocRecordLeakPermitted));
return result;
}
DQN_API bool Dqn_VMem_Commit(void *ptr, Dqn_usize size, uint32_t page_flags)
{
bool result = false;
if (!ptr || size == 0)
return false;
unsigned long os_page_flags = Dqn_VMem_ConvertPageToOSFlags_(page_flags);
#if defined(DQN_OS_WIN32)
result = VirtualAlloc(ptr, size, MEM_COMMIT, os_page_flags) != nullptr;
#elif defined(DQN_OS_UNIX)
result = mprotect(ptr, size, os_page_flags) == 0;
#else
#error "Missing implementation for Dqn_VMem_Commit"
#endif
return result;
}
DQN_API void Dqn_VMem_Decommit(void *ptr, Dqn_usize size)
{
#if defined(DQN_OS_WIN32)
// NOTE: This is a decommit call, which is explicitly saying to free the
// pages but not the VADs, you would use VMem_Release to release everything.
DQN_MSVC_WARNING_PUSH
DQN_MSVC_WARNING_DISABLE(6250) // Calling 'VirtualFree' without the MEM_RELEASE flag might free memory but not address descriptors (VADs). This causes address space leaks.
VirtualFree(ptr, size, MEM_DECOMMIT);
DQN_MSVC_WARNING_POP
#elif defined(DQN_OS_UNIX)
mprotect(ptr, size, PROT_NONE);
madvise(ptr, size, MADV_FREE);
#else
#error "Missing implementation for Dqn_VMem_Decommit"
#endif
}
DQN_API void Dqn_VMem_Release(void *ptr, Dqn_usize size)
{
#if defined(DQN_OS_WIN32)
(void)size;
VirtualFree(ptr, 0, MEM_RELEASE);
#elif defined(DQN_OS_UNIX)
munmap(ptr, size);
#else
#error "Missing implementation for Dqn_VMem_Release"
#endif
Dqn_Debug_TrackDealloc(ptr);
}
DQN_API int Dqn_VMem_Protect(void *ptr, Dqn_usize size, uint32_t page_flags)
{
if (!ptr || size == 0)
return 0;
static Dqn_String8 const ALIGNMENT_ERROR_MSG =
DQN_STRING8("Page protection requires pointers to be page aligned because we "
"can only guard memory at a multiple of the page boundary.");
DQN_ASSERTF(Dqn_IsPowerOfTwoAligned(DQN_CAST(uintptr_t)ptr, g_dqn_library->os_page_size), "%s", ALIGNMENT_ERROR_MSG.data);
DQN_ASSERTF(Dqn_IsPowerOfTwoAligned(size, g_dqn_library->os_page_size), "%s", ALIGNMENT_ERROR_MSG.data);
unsigned long os_page_flags = Dqn_VMem_ConvertPageToOSFlags_(page_flags);
#if defined(DQN_OS_WIN32)
unsigned long prev_flags = 0;
int result = VirtualProtect(ptr, size, os_page_flags, &prev_flags);
(void)prev_flags;
if (result == 0) {
#if defined(DQN_NO_WIN)
DQN_ASSERTF(result, "VirtualProtect failed");
#else
Dqn_ThreadScratch scratch = Dqn_Thread_GetScratch(nullptr);
Dqn_WinError error = Dqn_Win_LastError(scratch.arena);
DQN_ASSERTF(result, "VirtualProtect failed (%d): %.*s", error.code, DQN_STRING_FMT(error.msg));
#endif
}
#else
int result = mprotect(result->memory, result->size, os_page_flags);
DQN_ASSERTF(result == 0, "mprotect failed (%d)", errno);
#endif
return result;
}
// NOTE: [$MEMB] Dqn_MemBlock ======================================================================
DQN_API Dqn_MemBlockSizeRequiredResult Dqn_MemBlock_SizeRequired(Dqn_MemBlock const *block, Dqn_usize size, uint8_t alignment, uint32_t flags)
{
DQN_ASSERT(alignment > 0 && Dqn_IsPowerOfTwo(alignment));
Dqn_MemBlockSizeRequiredResult result = {};
result.alloc_size = size;
Dqn_MemBlockFlag block_flags = DQN_CAST(Dqn_MemBlockFlag)((block ? block->flags : 0) | flags);
uint8_t ptr_alignment = alignment;
if (DQN_ASAN_POISON) {
// NOTE: Guard a page after with poisoned memory. The first allocation
// is always guarded with poison-ed memory to prevent read/writes behind
// the block of memory.
if ((block_flags & Dqn_MemBlockFlag_AllocsAreContiguous) == 0) {
result.alloc_size = Dqn_AlignUpPowerOfTwo(size + DQN_ASAN_POISON_GUARD_SIZE, DQN_ASAN_POISON_ALIGNMENT);
}
ptr_alignment = DQN_MAX(alignment, DQN_ASAN_POISON_ALIGNMENT);
}
if (block) {
uintptr_t address = DQN_CAST(uintptr_t)block->data + block->used;
uintptr_t next_address = Dqn_AlignUpPowerOfTwo(address, ptr_alignment);
result.data_offset = next_address - DQN_CAST(uintptr_t)block->data;
Dqn_usize new_used = result.data_offset + result.alloc_size;
result.block_size = new_used - block->used;
} else {
result.block_size = result.alloc_size + (ptr_alignment - 1);
}
return result;
}
Dqn_usize Dqn_MemBlock_MetadataSize()
{
Dqn_usize init_poison_page = DQN_ASAN_POISON ? DQN_ASAN_POISON_GUARD_SIZE : 0;
Dqn_usize poison_alignment = DQN_ASAN_POISON ? DQN_ASAN_POISON_ALIGNMENT : 0;
Dqn_usize result = Dqn_AlignUpPowerOfTwo(sizeof(Dqn_MemBlock), poison_alignment) + init_poison_page;
return result;
}
DQN_API Dqn_MemBlock *Dqn_MemBlock_Init(Dqn_usize reserve, Dqn_usize commit, uint32_t flags)
{
DQN_ASSERTF(g_dqn_library->os_page_size, "Library needs to be initialised by calling Dqn_Library_Init()");
DQN_ASSERTF(Dqn_IsPowerOfTwo(g_dqn_library->os_page_size), "Invalid page size");
DQN_ASSERTF((flags & ~Dqn_MemBlockFlag_All) == 0, "Invalid flag combination, must adhere to Dqn_MemBlockFlags");
if (reserve == 0)
return nullptr;
Dqn_usize metadata_size = Dqn_MemBlock_MetadataSize();
Dqn_usize reserve_aligned = Dqn_AlignUpPowerOfTwo(metadata_size + reserve, g_dqn_library->os_page_size);
Dqn_usize commit_aligned = Dqn_AlignUpPowerOfTwo(metadata_size + commit, g_dqn_library->os_page_size);
commit_aligned = DQN_MIN(commit_aligned, reserve_aligned);
// NOTE: Avoid 1 syscall by committing on reserve if amounts are equal
Dqn_VMemCommit commit_on_reserve = commit_aligned == reserve_aligned ? Dqn_VMemCommit_Yes : Dqn_VMemCommit_No;
auto *result = DQN_CAST(Dqn_MemBlock *)Dqn_VMem_Reserve(reserve_aligned, commit_on_reserve, Dqn_VMemPage_ReadWrite);
if (result) {
// NOTE: Commit pages if we did not commit on the initial range.
if (!commit_on_reserve)
Dqn_VMem_Commit(result, commit_aligned, Dqn_VMemPage_ReadWrite);
result->data = DQN_CAST(uint8_t *)result + metadata_size;
result->size = reserve_aligned - metadata_size;
result->commit = commit_aligned - metadata_size;
result->flags = DQN_CAST(uint8_t)flags;
if (DQN_ASAN_POISON) { // NOTE: Poison (guard page + entire block), we unpoison as we allocate
DQN_ASSERT(Dqn_IsPowerOfTwoAligned(result->data, DQN_ASAN_POISON_ALIGNMENT));
DQN_ASSERT(Dqn_IsPowerOfTwoAligned(result->size, DQN_ASAN_POISON_ALIGNMENT));
void *poison_ptr = DQN_CAST(void *)Dqn_AlignUpPowerOfTwo(DQN_CAST(char *)result + sizeof(Dqn_MemBlock), DQN_ASAN_POISON_ALIGNMENT);
Dqn_usize bytes_to_poison = DQN_ASAN_POISON_GUARD_SIZE + result->size;
Dqn_ASAN_PoisonMemoryRegion(poison_ptr, bytes_to_poison);
}
}
return result;
}
DQN_API void *Dqn_MemBlock_Alloc(Dqn_MemBlock *block, Dqn_usize size, uint8_t alignment, Dqn_ZeroMem zero_mem)
{
DQN_ASSERT(zero_mem == Dqn_ZeroMem_Yes || zero_mem == Dqn_ZeroMem_No);
void *result = nullptr;
if (!block)
return result;
Dqn_MemBlockSizeRequiredResult size_required = Dqn_MemBlock_SizeRequired(block, size, alignment, Dqn_MemBlockFlag_Nil);
Dqn_usize new_used = size_required.data_offset + size_required.alloc_size;
if (new_used > block->size)
return result;
result = DQN_CAST(char *)block->data + size_required.data_offset;
block->used = new_used;
DQN_ASSERT(Dqn_IsPowerOfTwoAligned(result, alignment));
if (DQN_ASAN_POISON)
Dqn_ASAN_UnpoisonMemoryRegion(result, size);
if (zero_mem == Dqn_ZeroMem_Yes) {
Dqn_usize reused_bytes = DQN_MIN(block->commit - size_required.data_offset, size);
DQN_MEMSET(result, DQN_MEMSET_BYTE, reused_bytes);
}
if (block->commit < block->used) {
Dqn_usize commit_size = Dqn_AlignUpPowerOfTwo(block->used - block->commit, g_dqn_library->os_page_size);
void *commit_ptr = (void *)Dqn_AlignUpPowerOfTwo((char *)block->data + block->commit, g_dqn_library->os_page_size);
block->commit += commit_size;
Dqn_VMem_Commit(commit_ptr, commit_size, Dqn_VMemPage_ReadWrite);
DQN_ASSERT(block->commit <= block->size);
}
return result;
}
DQN_API void Dqn_MemBlock_Free(Dqn_MemBlock *block)
{
if (!block)
return;
Dqn_usize release_size = block->size + Dqn_MemBlock_MetadataSize();
if (DQN_ASAN_POISON)
Dqn_ASAN_UnpoisonMemoryRegion(block, release_size);
Dqn_VMem_Release(block, release_size);
}
DQN_API void Dqn_MemBlock_Pop(Dqn_MemBlock *block, Dqn_usize size)
{
if (!block)
return;
Dqn_usize size_adjusted = DQN_MIN(size, block->used);
Dqn_usize to = block->used - size_adjusted;
Dqn_MemBlock_PopTo(block, to);
}
DQN_API void Dqn_MemBlock_PopTo(Dqn_MemBlock *block, Dqn_usize to)
{
if (!block || to >= block->used)
return;
if (DQN_ASAN_POISON) {
void *poison_ptr = DQN_CAST(char *)block->data + to;
void *end_ptr = DQN_CAST(void *)Dqn_AlignUpPowerOfTwo((DQN_CAST(uintptr_t)block->data + block->used), DQN_ASAN_POISON_ALIGNMENT);
uintptr_t bytes_to_poison = DQN_CAST(uintptr_t)end_ptr - DQN_CAST(uintptr_t)poison_ptr;
Dqn_ASAN_PoisonMemoryRegion(poison_ptr, bytes_to_poison);
}
block->used = to;
}
// NOTE: [$AREN] Dqn_Arena =========================================================================
DQN_FILE_SCOPE void *Dqn_Arena_AllocatorAlloc(size_t size, uint8_t align, Dqn_ZeroMem zero_mem, void *user_context)
{
void *result = NULL;
if (!user_context)
return result;
Dqn_Arena *arena = DQN_CAST(Dqn_Arena *)user_context;
result = Dqn_Arena_Alloc(arena, size, align, zero_mem);
return result;
}
DQN_FILE_SCOPE void Dqn_Arena_AllocatorDealloc(void *, size_t, void *)
{
// NOTE: No-op, arenas batch allocate and batch deallocate. Call free on the
// underlying arena, since we can't free individual pointers.
}
DQN_API Dqn_Allocator Dqn_Arena_Allocator(Dqn_Arena *arena)
{
Dqn_Allocator result = {};
if (arena) {
result.user_context = arena;
result.alloc = Dqn_Arena_AllocatorAlloc;
result.dealloc = Dqn_Arena_AllocatorDealloc;
}
return result;
}
DQN_API Dqn_MemBlock *Dqn_Arena_Grow(Dqn_Arena *arena, Dqn_usize reserve, Dqn_usize commit, uint8_t flags)
{
if (!arena)
return nullptr;
uint8_t mem_block_flags = flags;
if (arena->allocs_are_allowed_to_leak)
mem_block_flags |= Dqn_MemBlockFlag_AllocRecordLeakPermitted;
Dqn_MemBlock *result = Dqn_MemBlock_Init(reserve, commit, mem_block_flags);
if (result) {
if (!arena->head)
arena->head = result;
if (arena->tail)
arena->tail->next = result;
if (!arena->curr)
arena->curr = result;
result->prev = arena->tail;
arena->tail = result;
arena->blocks += 1;
}
return result;
}
DQN_API void *Dqn_Arena_Alloc(Dqn_Arena *arena, Dqn_usize size, uint8_t align, Dqn_ZeroMem zero_mem)
{
DQN_ASSERT(Dqn_IsPowerOfTwo(align));
void *result = nullptr;
if (!arena || size == 0 || align == 0)
return result;
for (;;) {
while (arena->curr && (arena->curr->flags & Dqn_MemBlockFlag_ArenaPrivate))
arena->curr = arena->curr->next;
if (!arena->curr) {
Dqn_MemBlockSizeRequiredResult size_required = Dqn_MemBlock_SizeRequired(nullptr, size, align, Dqn_MemBlockFlag_Nil);
Dqn_usize block_size = size_required.block_size;
if (!Dqn_Arena_Grow(arena, block_size, block_size, Dqn_MemBlockFlag_Nil))
return result;
DQN_ASSERT(arena->curr);
}
result = Dqn_MemBlock_Alloc(arena->curr, size, align, zero_mem);
if (result)
break;
arena->curr = arena->curr->next;
}
return result;
}
DQN_API void *Dqn_Arena_Copy(Dqn_Arena *arena, void *src, Dqn_usize size, uint8_t align)
{
void *result = Dqn_Arena_Alloc(arena, size, align, Dqn_ZeroMem_No);
DQN_MEMCPY(result, src, size);
return result;
}
DQN_API void *Dqn_Arena_CopyZ(Dqn_Arena *arena, void *src, Dqn_usize size, uint8_t align)
{
void *result = Dqn_Arena_Alloc(arena, size + 1, align, Dqn_ZeroMem_No);
DQN_MEMCPY(result, src, size);
(DQN_CAST(char *)result)[size] = 0;
return result;
}
DQN_API void Dqn_Arena_Free(Dqn_Arena *arena)
{
if (!arena)
return;
for (Dqn_MemBlock *block = arena->head; block; ) {
Dqn_MemBlock *next = block->next;
Dqn_MemBlock_Free(block);
block = next;
}
arena->curr = arena->head = arena->tail = nullptr;
arena->blocks = 0;
}
DQN_API Dqn_ArenaTempMemory Dqn_Arena_BeginTempMemory(Dqn_Arena *arena)
{
Dqn_ArenaTempMemory result = {};
if (arena) {
result.arena = arena;
result.head = arena->head;
result.curr = arena->curr;
result.tail = arena->tail;
result.curr_used = (arena->curr) ? arena->curr->used : 0;
result.blocks = arena->blocks;
}
return result;
}
DQN_API void Dqn_Arena_EndTempMemory(Dqn_ArenaTempMemory temp_memory, bool cancel)
{
if (cancel || !temp_memory.arena)
return;
// NOTE: The arena has been freed or invalidly manipulated (e.g. freed)
// since the temp memory started as the head cannot change once it is
// captured in the temp memory.
Dqn_Arena *arena = temp_memory.arena;
if (arena->head != temp_memory.head)
return;
// NOTE: Revert the current block to the temp_memory's current block
arena->blocks = temp_memory.blocks;
arena->head = temp_memory.head;
arena->curr = temp_memory.curr;
Dqn_MemBlock_PopTo(arena->curr, temp_memory.curr_used);
// NOTE: Free the tail blocks until we reach the temp_memory's tail block
while (arena->tail != temp_memory.tail) {
Dqn_MemBlock *tail = arena->tail;
arena->tail = tail->prev;
Dqn_MemBlock_Free(tail);
}
// NOTE: Chop the restored tail link
if (arena->tail)
arena->tail->next = nullptr;
// NOTE: Reset the usage of all the blocks between the tail and current block's
for (Dqn_MemBlock *block = arena->tail; block && (block != arena->curr); block = block->prev)
Dqn_MemBlock_PopTo(block, 0);
}
Dqn_ArenaTempMemoryScope::Dqn_ArenaTempMemoryScope(Dqn_Arena *arena)
{
temp_memory = Dqn_Arena_BeginTempMemory(arena);
}
Dqn_ArenaTempMemoryScope::~Dqn_ArenaTempMemoryScope()
{
Dqn_Arena_EndTempMemory(temp_memory, cancel);
}
DQN_API Dqn_ArenaStatString Dqn_Arena_StatString(Dqn_ArenaStat const *stat)
{
// NOTE: We use a non-standard format string that is only usable via
// stb sprintf that GCC warns about as an error. This pragma mutes that.
DQN_GCC_WARNING_PUSH
DQN_GCC_WARNING_DISABLE("-Wformat-invalid-specifier")
DQN_GCC_WARNING_DISABLE("-Wformat-extra-args")
Dqn_ArenaStatString result = {};
int size16 = STB_SPRINTF_DECORATE(snprintf)(result.data, DQN_ARRAY_ICOUNT(result.data),
"ArenaStat{"
"used/hwm=%_$$zd/%_$$zd, "
"cap/hwm=%_$$zd/%_$$zd, "
"wasted/hwm=%_$$zd/%_$$zd, "
"blocks/hwm=%_$$u/%_$$u, "
"syscalls=%'zd"
"}",
stat->used, stat->used_hwm,
stat->capacity, stat->capacity_hwm,
stat->wasted, stat->wasted_hwm,
stat->blocks, stat->blocks_hwm,
stat->syscalls);
result.size = Dqn_Safe_SaturateCastIntToU16(size16);
DQN_GCC_WARNING_POP
return result;
}
// NOTE: [$ACAT] Dqn_ArenaCatalog ==================================================================
DQN_API void Dqn_ArenaCatalog_Init(Dqn_ArenaCatalog *catalog, Dqn_Arena *arena)
{
catalog->arena = arena;
catalog->sentinel.next = &catalog->sentinel;
catalog->sentinel.prev = &catalog->sentinel;
}
DQN_API void Dqn_ArenaCatalog_Add(Dqn_ArenaCatalog *catalog, Dqn_Arena *arena)
{
// NOTE: We could use an atomic for appending to the sentinel but it is such
// a rare operation to append to the catalog that we don't bother.
Dqn_TicketMutex_Begin(&catalog->ticket_mutex);
// NOTE: Create item in the catalog
Dqn_ArenaCatalogItem *result = Dqn_Arena_New(catalog->arena, Dqn_ArenaCatalogItem, Dqn_ZeroMem_Yes);
result->arena = arena;
// NOTE: Add to the catalog (linked list)
Dqn_ArenaCatalogItem *sentinel = &catalog->sentinel;
result->next = sentinel;
result->prev = sentinel->prev;
result->next->prev = result;
result->prev->next = result;
Dqn_TicketMutex_End(&catalog->ticket_mutex);
Dqn_Atomic_AddU32(&catalog->arena_count, 1);
}
DQN_API Dqn_Arena *Dqn_ArenaCatalog_Alloc(Dqn_ArenaCatalog *catalog, Dqn_usize byte_size, Dqn_usize commit)
{
Dqn_TicketMutex_Begin(&catalog->ticket_mutex);
Dqn_Arena *result = Dqn_Arena_New(catalog->arena, Dqn_Arena, Dqn_ZeroMem_Yes);
Dqn_TicketMutex_End(&catalog->ticket_mutex);
Dqn_Arena_Grow(result, byte_size, commit, 0 /*flags*/);
Dqn_ArenaCatalog_Add(catalog, result);
return result;
}
DQN_API Dqn_Arena *Dqn_ArenaCatalog_AllocFV(Dqn_ArenaCatalog *catalog, Dqn_usize byte_size, Dqn_usize commit, char const *fmt, va_list args)
{
Dqn_Arena *result = Dqn_ArenaCatalog_Alloc(catalog, byte_size, commit);
result->label = Dqn_String8_InitFV(Dqn_Arena_Allocator(result), fmt, args);
return result;
}
DQN_API Dqn_Arena *Dqn_ArenaCatalog_AllocF(Dqn_ArenaCatalog *catalog, Dqn_usize byte_size, Dqn_usize commit, char const *fmt, ...)
{
va_list args;
va_start(args, fmt);
Dqn_Arena *result = Dqn_ArenaCatalog_AllocFV(catalog, byte_size, commit, fmt, args);
va_end(args);
return result;
}