/* * Copyright 2020 Axel Davy * SPDX-License-Identifier: MIT */ /* * Memory util function to allocate RAM backing for textures. * DEFAULT textures are stored on GPU * MANAGED textures have a RAM backing and upload the content to a GPU texture for use * SYSTEMMEM textures are stored in RAM and are meant to be uploaded to DEFAULT textures. * Basically SYSTEMMEM + DEFAULT enables to do manually what MANAGED does automatically. * * Once the GPU texture is created, the RAM backing of MANAGED textures can be used in * two occasions: * . Recreating the GPU texture (for example lod change, or GPU memory eviction) * . Reading the texture content (some games do that to fill higher res versions of the texture) * * When a lot of textures are used, the amount of addressing space (virtual memory) taken by MANAGED * and SYSTEMMEM textures can be significant and cause virtual memory exhaustion for 32 bits programs. * * One way to reduce the virtual memory taken is to ignore lod and delete the RAM backing of * MANAGED textures once it is uploaded. If the texture is read, or evicted from GPU memory, the RAM * backing would be recreated (Note that mapping the GPU memory is not acceptable as RAM memory is supposed * to have smaller (fixed) stride constraints). * * Instead the approach taken here is to keep the RAM backing alive, but free its addressing space. * In other words virtual memory usage is reduced, but the RAM usage of the app is the same. * To do so, we use the memfd feature of the linux kernel. It enables to allocate a file * stored in RAM and visible only to the app. We can map/unmap portions of the file as we need. * When a portion is mapped, it takes virtual memory space. When it is not, it doesn't. * The file is stored in RAM, and thus the access speed is the same as normal RAM. Using such * file to allocate data enables to use more than 4GB RAM on 32 bits. * * This approach adds some overhead: when accessing mapped content the first time, pages are allocated * by the system. This has a lot of overhead (several times the time to memset the area). * Releasing these pages (when unmapping) has overhead too, though significantly less. * * This overhead however is much less significant than the overhead of downloading the GPU content. * In addition, we reduce significantly the overhead spent in Gallium nine for new allocations by * using the fact new contents of the file are zero-allocated. By not calling memset in Gallium nine, * the overhead of page allocation happens client side, thus outside the d3d mutex. This should give * a performance boost for multithreaded applications. As malloc also has this overhead (at least for * large enough allocations which use mmap internally), allocating ends up faster than with the standard * allocation path. * By far the overhead induced by page allocation/deallocation is the biggest overhead involved in this * code. It is reduced significantly with huge pages, but it is too complex to configure for the user * to use it (and it has some memory management downsides too). The memset trick enables to move most of * the overhead outside Nine anyway. * * To prevent useless unmappings quickly followed by mapping again, we do not unmap right away allocations * that are not locked for access anymore. Indeed it is likely the allocation will be accessed several times * in a row, for example first to fill it, then to upload it. * We keep everything mapped until we reach a threshold of memory allocated. Then we use hints to prioritize * which regions to unmap first. Thus virtual memory usage is only reduced when the threshold is reached. * * Multiple memfd files are used, each of 100MB. Thus memory usage (but not virtual memory usage) increases * by amounts of 100MB. When not on x86 32 bits, we do use the standard malloc. * * Finally, for ease of use, we do not implement packing of allocation inside page-aligned regions. * One allocation is given one page-aligned region inside a memfd file. * Allocations smaller than a page (4KB on x86) go through malloc. * As texture sizes are usually multiples of powers of two, allocations above the page size are typically * multiples of the page size, thus space is not wasted in practice. * */ #include #include #include #include #include #include #include #include #include #include #include #include "util/list.h" #include "util/u_memory.h" #include "util/slab.h" #include "nine_debug.h" #include "nine_memory_helper.h" #include "nine_state.h" #define DIVUP(a,b) (((a)+(b)-1)/(b)) /* Required alignment for allocations */ #define NINE_ALLOCATION_ALIGNMENT 32 #define DBG_CHANNEL (DBG_BASETEXTURE|DBG_SURFACE|DBG_VOLUME|DBG_TEXTURE|DBG_CUBETEXTURE) /* Use memfd only for 32 bits. Check for memfd_create support */ #if DETECT_ARCH_X86 && defined(HAVE_MEMFD_CREATE) #define NINE_ENABLE_MEMFD #endif #ifdef NINE_ENABLE_MEMFD struct nine_memfd_file_region { unsigned offset; unsigned size; void *map; /* pointer to the mapped content of the file. Can be NULL */ int num_locks; /* Total number of locks blocking the munmap */ int num_weak_unlocks; /* Number of users which weakly block the munmap */ bool zero_filled; struct list_head list; }; struct nine_memfd_file { int fd; int filesize; /* Size of the file */ struct list_head free_regions; /* This list is sorted by the offset, and consecutive regions are merged */ struct list_head unmapped_allocated_regions; /* This list and the following ones are not sorted */ struct list_head locked_mapped_allocated_regions; struct list_head weak_unlocked_mapped_allocated_regions; struct list_head unlocked_mapped_allocated_regions; }; /* The allocation is stored inside a memfd */ #define NINE_MEMFD_ALLOC 1 /* The allocation is part of another allocation, which is stored inside a memfd */ #define NINE_MEMFD_SUBALLOC 2 /* The allocation was allocated with malloc and will have to be freed */ #define NINE_MALLOC_ALLOC 3 /* The pointer doesn't need memory management */ #define NINE_EXTERNAL_ALLOC 4 struct nine_memfd_allocation { struct nine_memfd_file *file; /* File in which the data is allocated */ struct nine_memfd_file_region *region; /* Corresponding file memory region. Max 1 allocation per region */ }; /* 'Suballocations' are used to represent subregions of an allocation. * For example a given layer of a texture. These are not allocations, * but can be accessed separately. To correctly handle accessing them, * we encapsulate them into this structure. */ struct nine_memfd_suballocation { struct nine_memfd_allocation *parent; /* Parent allocation */ int relative_offset; /* Offset relative to the parent */ }; /* A standard allocation with malloc */ struct nine_malloc_allocation { void *buf; unsigned allocation_size; }; /* A pointer with no need of memory management. * For example a pointer passed by the application, * or a 'suballocation' inside a malloc-ed allocation. */ struct nine_external_allocation { void *buf; }; /* Encapsulates all allocations */ struct nine_allocation { unsigned allocation_type; /* Type of allocation */ union { struct nine_memfd_allocation memfd; struct nine_memfd_suballocation submemfd; struct nine_malloc_allocation malloc; struct nine_external_allocation external; } memory; struct list_head list_free; /* for pending frees */ /* The fields below are only used for memfd/submemfd allocations */ struct list_head list_release; /* for pending releases */ /* Handling of the CSMT thread: * API calls are singled thread (global mutex protection). * However we multithreading internally (CSMT worker thread). * To handle this thread, we map/lock the allocation in the * main thread and increase pending_counter. When the worker thread * is done with the scheduled function, the pending_counter is decreased. * If pending_counter is 0, locks_on_counter can be subtracted from * active_locks (in the main thread). */ unsigned locks_on_counter; unsigned *pending_counter; /* Hint from the last unlock indicating the data might be locked again soon */ bool weak_unlock; }; struct nine_allocator { struct NineDevice9 *device; int page_size; /* Page size */ int num_fd_max; /* Max number of memfd files */ int min_file_size; /* Minimum memfd file size */ /* Tracking of all allocations */ long long total_allocations; /* Amount of memory allocated */ long long total_locked_memory; /* TODO */ /* Amount of memory blocked by a lock */ long long total_virtual_memory; /* Current virtual memory used by our allocations */ long long total_virtual_memory_limit; /* Target maximum virtual memory used. Above that, tries to unmap memfd files whenever possible. */ int num_fd; /* Number of memfd files */ /* TODO release unused memfd files */ struct slab_mempool allocation_pool; struct slab_mempool region_pool; struct nine_memfd_file *memfd_pool; /* Table (of size num_fd) of memfd files */ struct list_head pending_releases; /* List of allocations with unlocks depending on pending_counter */ /* TODO: Elements seem removed only on flush. Destruction ? */ pthread_mutex_t mutex_pending_frees; struct list_head pending_frees; }; #if MESA_DEBUG static void debug_dump_memfd_state(struct nine_memfd_file *memfd_file, bool details) { struct nine_memfd_file_region *region; DBG("fd: %d, filesize: %d\n", memfd_file->fd, memfd_file->filesize); if (!details) return; LIST_FOR_EACH_ENTRY(region, &memfd_file->free_regions, list) { DBG("FREE block: offset %d, size %d, map=%p, locks=%d, weak=%d, z=%d\n", region->offset, region->size, region->map, region->num_locks, region->num_weak_unlocks, (int)region->zero_filled); } LIST_FOR_EACH_ENTRY(region, &memfd_file->unmapped_allocated_regions, list) { DBG("UNMAPPED ALLOCATED block: offset %d, size %d, map=%p, locks=%d, weak=%d, z=%d\n", region->offset, region->size, region->map, region->num_locks, region->num_weak_unlocks, (int)region->zero_filled); } LIST_FOR_EACH_ENTRY(region, &memfd_file->locked_mapped_allocated_regions, list) { DBG("LOCKED MAPPED ALLOCATED block: offset %d, size %d, map=%p, locks=%d, weak=%d, z=%d\n", region->offset, region->size, region->map, region->num_locks, region->num_weak_unlocks, (int)region->zero_filled); } LIST_FOR_EACH_ENTRY(region, &memfd_file->unlocked_mapped_allocated_regions, list) { DBG("UNLOCKED MAPPED ALLOCATED block: offset %d, size %d, map=%p, locks=%d, weak=%d, z=%d\n", region->offset, region->size, region->map, region->num_locks, region->num_weak_unlocks, (int)region->zero_filled); } LIST_FOR_EACH_ENTRY(region, &memfd_file->weak_unlocked_mapped_allocated_regions, list) { DBG("WEAK UNLOCKED MAPPED ALLOCATED block: offset %d, size %d, map=%p, locks=%d, weak=%d, z=%d\n", region->offset, region->size, region->map, region->num_locks, region->num_weak_unlocks, (int)region->zero_filled); } } static void debug_dump_allocation_state(struct nine_allocation *allocation) { switch(allocation->allocation_type) { case NINE_MEMFD_ALLOC: DBG("Allocation is stored in this memfd file:\n"); debug_dump_memfd_state(allocation->memory.memfd.file, true); DBG("Allocation is offset: %d, size: %d\n", allocation->memory.memfd.region->offset, allocation->memory.memfd.region->size); break; case NINE_MEMFD_SUBALLOC: DBG("Allocation is suballocation at relative offset %d of this allocation:\n", allocation->memory.submemfd.relative_offset); DBG("Parent allocation is stored in this memfd file:\n"); debug_dump_memfd_state(allocation->memory.submemfd.parent->file, false); DBG("Parent allocation is offset: %d, size: %d\n", allocation->memory.submemfd.parent->region->offset, allocation->memory.submemfd.parent->region->size); break; case NINE_MALLOC_ALLOC: DBG("Allocation is a standard malloc\n"); break; case NINE_EXTERNAL_ALLOC: DBG("Allocation is a suballocation of a standard malloc or an external allocation\n"); break; default: assert(false); } } #else static void debug_dump_memfd_state(struct nine_memfd_file *memfd_file, bool details) { (void)memfd_file; (void)details; } static void debug_dump_allocation_state(struct nine_allocation *allocation) { (void)allocation; } #endif static void debug_dump_allocator_state(struct nine_allocator *allocator) { DBG("SURFACE ALLOCATOR STATUS:\n"); DBG("Total allocated: %lld\n", allocator->total_allocations); DBG("Total virtual memory locked: %lld\n", allocator->total_locked_memory); DBG("Virtual memory used: %lld / %lld\n", allocator->total_virtual_memory, allocator->total_virtual_memory_limit); DBG("Num memfd files: %d / %d\n", allocator->num_fd, allocator->num_fd_max); } /* Retrieve file used for the storage of the content of this allocation. * NULL if not using memfd */ static struct nine_memfd_file * nine_get_memfd_file_backing(struct nine_allocation *allocation) { if (allocation->allocation_type > NINE_MEMFD_SUBALLOC) return NULL; if (allocation->allocation_type == NINE_MEMFD_ALLOC) return allocation->memory.memfd.file; return allocation->memory.submemfd.parent->file; } /* Retrieve region used for the storage of the content of this allocation. * NULL if not using memfd */ static struct nine_memfd_file_region * nine_get_memfd_region_backing(struct nine_allocation *allocation) { if (allocation->allocation_type > NINE_MEMFD_SUBALLOC) return NULL; if (allocation->allocation_type == NINE_MEMFD_ALLOC) return allocation->memory.memfd.region; return allocation->memory.submemfd.parent->region; } static void move_region(struct list_head *tail, struct nine_memfd_file_region *region) { /* Remove from previous list (if any) */ list_delinit(®ion->list); /* Insert in new list (last) */ list_addtail(®ion->list, tail); } #if 0 static void move_region_ordered(struct list_head *tail, struct nine_memfd_file_region *region) { struct nine_memfd_file_region *cur_region; struct list_head *insertion_point = tail; /* Remove from previous list (if any) */ list_delinit(®ion->list); LIST_FOR_EACH_ENTRY(cur_region, tail, list) { if (cur_region->offset > region->offset) break; insertion_point = &cur_region->list; } /* Insert just before cur_region */ list_add(®ion->list, insertion_point); } #endif static void move_region_ordered_merge(struct nine_allocator *allocator, struct list_head *tail, struct nine_memfd_file_region *region) { struct nine_memfd_file_region *p, *cur_region = NULL, *prev_region = NULL; /* Remove from previous list (if any) */ list_delinit(®ion->list); LIST_FOR_EACH_ENTRY(p, tail, list) { cur_region = p; if (cur_region->offset > region->offset) break; prev_region = cur_region; } /* Insert after prev_region and before cur_region. Try to merge */ if (prev_region && ((prev_region->offset + prev_region->size) == region->offset)) { if (cur_region && (cur_region->offset == (region->offset + region->size))) { /* Merge all three regions */ prev_region->size += region->size + cur_region->size; prev_region->zero_filled = prev_region->zero_filled && region->zero_filled && cur_region->zero_filled; list_del(&cur_region->list); slab_free_st(&allocator->region_pool, region); slab_free_st(&allocator->region_pool, cur_region); } else { prev_region->size += region->size; prev_region->zero_filled = prev_region->zero_filled && region->zero_filled; slab_free_st(&allocator->region_pool, region); } } else if (cur_region && (cur_region->offset == (region->offset + region->size))) { cur_region->offset = region->offset; cur_region->size += region->size; cur_region->zero_filled = region->zero_filled && cur_region->zero_filled; slab_free_st(&allocator->region_pool, region); } else { list_add(®ion->list, prev_region ? &prev_region->list : tail); } } static struct nine_memfd_file_region *allocate_region(struct nine_allocator *allocator, unsigned offset, unsigned size) { struct nine_memfd_file_region *region = slab_alloc_st(&allocator->allocation_pool); if (!region) return NULL; region->offset = offset; region->size = size; region->num_locks = 0; region->num_weak_unlocks = 0; region->map = NULL; region->zero_filled = false; list_inithead(®ion->list); return region; } /* Go through memfd allocated files, and try to use unused memory for the requested allocation. * Returns whether it succeeded */ static bool insert_new_allocation(struct nine_allocator *allocator, struct nine_allocation *new_allocation, unsigned allocation_size) { int memfd_index; struct nine_memfd_file *memfd_file, *best_memfd_file; struct nine_memfd_file_region *region, *best_region, *new_region; /* Find the smallest - but bigger than the requested size - unused memory * region inside the memfd files. */ int min_blocksize = INT_MAX; for (memfd_index = 0; memfd_index < allocator->num_fd; memfd_index++) { memfd_file = (void*)allocator->memfd_pool + memfd_index*sizeof(struct nine_memfd_file); LIST_FOR_EACH_ENTRY(region, &memfd_file->free_regions, list) { if (region->size <= min_blocksize && region->size >= allocation_size) { min_blocksize = region->size; best_region = region; best_memfd_file = memfd_file; } } if (min_blocksize == allocation_size) break; } /* The allocation doesn't fit in any memfd file */ if (min_blocksize == INT_MAX) return false; /* Target region found */ /* Move from free to unmapped allocated */ best_region->size = DIVUP(allocation_size, allocator->page_size) * allocator->page_size; assert(min_blocksize >= best_region->size); move_region(&best_memfd_file->unmapped_allocated_regions, best_region); new_allocation->memory.memfd.region = best_region; new_allocation->memory.memfd.file = best_memfd_file; /* If the original region is bigger than needed, add new region with remaining space */ min_blocksize -= best_region->size; if (min_blocksize > 0) { new_region = allocate_region(allocator, best_region->offset + best_region->size, min_blocksize); new_region->zero_filled = best_region->zero_filled; move_region_ordered_merge(allocator, &best_memfd_file->free_regions, new_region); } allocator->total_allocations += best_region->size; return true; } /* Go through allocations with unlocks waiting on pending_counter being 0. * If 0 is indeed reached, update the allocation status */ static void nine_flush_pending_releases(struct nine_allocator *allocator) { struct nine_allocation *allocation, *ptr; LIST_FOR_EACH_ENTRY_SAFE(allocation, ptr, &allocator->pending_releases, list_release) { assert(allocation->locks_on_counter > 0); /* If pending_releases reached 0, remove from the list and update the status */ if (*allocation->pending_counter == 0) { struct nine_memfd_file *memfd_file = nine_get_memfd_file_backing(allocation); struct nine_memfd_file_region *region = nine_get_memfd_region_backing(allocation); region->num_locks -= allocation->locks_on_counter; allocation->locks_on_counter = 0; list_delinit(&allocation->list_release); if (region->num_locks == 0) { /* Move to the correct list */ if (region->num_weak_unlocks) move_region(&memfd_file->weak_unlocked_mapped_allocated_regions, region); else move_region(&memfd_file->unlocked_mapped_allocated_regions, region); allocator->total_locked_memory -= region->size; } } } } static void nine_free_internal(struct nine_allocator *allocator, struct nine_allocation *allocation); static void nine_flush_pending_frees(struct nine_allocator *allocator) { struct nine_allocation *allocation, *ptr; pthread_mutex_lock(&allocator->mutex_pending_frees); /* The order of release matters as suballocations are supposed to be released first */ LIST_FOR_EACH_ENTRY_SAFE(allocation, ptr, &allocator->pending_frees, list_free) { /* Set the allocation in an unlocked state, and then free it */ if (allocation->allocation_type == NINE_MEMFD_ALLOC || allocation->allocation_type == NINE_MEMFD_SUBALLOC) { struct nine_memfd_file *memfd_file = nine_get_memfd_file_backing(allocation); struct nine_memfd_file_region *region = nine_get_memfd_region_backing(allocation); if (region->num_locks != 0) { region->num_locks = 0; allocator->total_locked_memory -= region->size; /* Useless, but to keep consistency */ move_region(&memfd_file->unlocked_mapped_allocated_regions, region); } region->num_weak_unlocks = 0; allocation->weak_unlock = false; allocation->locks_on_counter = 0; list_delinit(&allocation->list_release); } list_delinit(&allocation->list_free); nine_free_internal(allocator, allocation); } pthread_mutex_unlock(&allocator->mutex_pending_frees); } /* Try to unmap the memfd_index-th file if not already unmapped. * If even_if_weak is False, will not unmap if there are weak unlocks */ static void nine_memfd_unmap_region(struct nine_allocator *allocator, struct nine_memfd_file *memfd_file, struct nine_memfd_file_region *region) { DBG("Unmapping memfd mapped region at %d: size: %d, map=%p, locks=%d, weak=%d\n", region->offset, region->size, region->map, region->num_locks, region->num_weak_unlocks); assert(region->map != NULL); if (munmap(region->map, region->size) != 0) fprintf(stderr, "Error on unmapping, errno=%d\n", (int)errno); region->map = NULL; /* Move from one of the mapped region list to the unmapped one */ move_region(&memfd_file->unmapped_allocated_regions, region); allocator->total_virtual_memory -= region->size; } /* Unallocate a region of a memfd file */ static void remove_allocation(struct nine_allocator *allocator, struct nine_memfd_file *memfd_file, struct nine_memfd_file_region *region) { assert(region->num_locks == 0); region->num_weak_unlocks = 0; /* Move from mapped region to unmapped region */ if (region->map) { if (likely(!region->zero_filled)) { /* As the region is mapped, it is likely the pages are allocated. * Do the memset now for when we allocate again. It is much faster now, * as the pages are allocated. */ DBG("memset on data=%p, size %d\n", region->map, region->size); memset(region->map, 0, region->size); region->zero_filled = true; } nine_memfd_unmap_region(allocator, memfd_file, region); } /* Move from unmapped region to free region */ allocator->total_allocations -= region->size; move_region_ordered_merge(allocator, &memfd_file->free_regions, region); } /* Try to unmap the regions of the memfd_index-th file if not already unmapped. * If even_if_weak is False, will not unmap if there are weak unlocks */ static void nine_memfd_try_unmap_file(struct nine_allocator *allocator, int memfd_index, bool weak) { struct nine_memfd_file *memfd_file = (void*)allocator->memfd_pool + memfd_index*sizeof(struct nine_memfd_file); struct nine_memfd_file_region *region, *ptr; DBG("memfd file at %d: fd: %d, filesize: %d\n", memfd_index, memfd_file->fd, memfd_file->filesize); debug_dump_memfd_state(memfd_file, true); LIST_FOR_EACH_ENTRY_SAFE(region, ptr, weak ? &memfd_file->weak_unlocked_mapped_allocated_regions : &memfd_file->unlocked_mapped_allocated_regions, list) { nine_memfd_unmap_region(allocator, memfd_file, region); } } /* Unmap files until we are below the virtual memory target limit. * If unmap_everything_possible is set, ignore the limit and unmap * all that can be unmapped. */ static void nine_memfd_files_unmap(struct nine_allocator *allocator, bool unmap_everything_possible) { long long memory_limit = unmap_everything_possible ? 0 : allocator->total_virtual_memory_limit; int i; /* We are below the limit. Do nothing */ if (memory_limit >= allocator->total_virtual_memory) return; /* Update allocations with pending releases */ nine_flush_pending_releases(allocator); DBG("Trying to unmap files with no weak unlock (%lld / %lld)\n", allocator->total_virtual_memory, memory_limit); /* Try to release everything with no weak releases. * Those have data not needed for a long time (and * possibly ever). */ for (i = 0; i < allocator->num_fd; i++) { nine_memfd_try_unmap_file(allocator, i, false); if (memory_limit >= allocator->total_virtual_memory) { return;} } DBG("Trying to unmap files even with weak unlocks (%lld / %lld)\n", allocator->total_virtual_memory, memory_limit); /* This wasn't enough. Also release files with weak releases */ for (i = 0; i < allocator->num_fd; i++) { nine_memfd_try_unmap_file(allocator, i, true); /* Stop if the target is reached */ if (memory_limit >= allocator->total_virtual_memory) { return;} } if (!unmap_everything_possible) return; /* If there are some pending uploads, execute them, * and retry. */ if (list_is_empty(&allocator->pending_releases)) { return;} nine_csmt_process(allocator->device); nine_flush_pending_releases(allocator); DBG("Retrying after flushing (%lld / %lld)\n", allocator->total_virtual_memory, memory_limit); for (i = 0; i < allocator->num_fd; i++) { nine_memfd_try_unmap_file(allocator, i, false); nine_memfd_try_unmap_file(allocator, i, true); } /* We have done all we could */ } /* Map a given memfd file */ static bool nine_memfd_region_map(struct nine_allocator *allocator, struct nine_memfd_file *memfd_file, struct nine_memfd_file_region *region) { if (region->map != NULL) return true; debug_dump_memfd_state(memfd_file, true); nine_memfd_files_unmap(allocator, false); void *buf = mmap(NULL, region->size, PROT_READ | PROT_WRITE, MAP_SHARED, memfd_file->fd, region->offset); if (buf == MAP_FAILED && errno == ENOMEM) { DBG("Failed to mmap a memfd file - trying to unmap other files\n"); nine_memfd_files_unmap(allocator, true); buf = mmap(NULL, region->size, PROT_READ | PROT_WRITE, MAP_SHARED, memfd_file->fd, region->offset); } if (buf == MAP_FAILED) { DBG("Failed to mmap a memfd file, errno=%d\n", (int)errno); return false; } region->map = buf; /* no need to move to an unlocked mapped regions list, the caller will handle the list */ allocator->total_virtual_memory += region->size; assert((uintptr_t)buf % NINE_ALLOCATION_ALIGNMENT == 0); /* mmap should be page_size aligned, so it should be fine */ return true; } /* Allocate with memfd some memory. Returns True if successful. */ static bool nine_memfd_allocator(struct nine_allocator *allocator, struct nine_allocation *new_allocation, unsigned allocation_size) { struct nine_memfd_file *memfd_file; struct nine_memfd_file_region *region; allocation_size = DIVUP(allocation_size, allocator->page_size) * allocator->page_size; new_allocation->allocation_type = NINE_MEMFD_ALLOC; new_allocation->locks_on_counter = 0; new_allocation->pending_counter = NULL; new_allocation->weak_unlock = false; list_inithead(&new_allocation->list_free); list_inithead(&new_allocation->list_release); /* Try to find free space in a file already allocated */ if (insert_new_allocation(allocator, new_allocation, allocation_size)) return true; /* No - allocate new memfd file */ if (allocator->num_fd == allocator->num_fd_max) return false; /* Too many memfd files */ allocator->num_fd++; memfd_file = (void*)allocator->memfd_pool + (allocator->num_fd-1)*sizeof(struct nine_memfd_file); /* If the allocation size is above the memfd file default size, use a bigger size */ memfd_file->filesize = MAX2(allocation_size, allocator->min_file_size); memfd_file->fd = memfd_create("gallium_nine_ram", 0); if (memfd_file->fd == -1) { DBG("Failed to created a memfd file, errno=%d\n", (int)errno); allocator->num_fd--; return false; } if (ftruncate(memfd_file->fd, memfd_file->filesize) != 0) { DBG("Failed to resize a memfd file, errno=%d\n", (int)errno); close(memfd_file->fd); allocator->num_fd--; return false; } list_inithead(&memfd_file->free_regions); list_inithead(&memfd_file->unmapped_allocated_regions); list_inithead(&memfd_file->locked_mapped_allocated_regions); list_inithead(&memfd_file->unlocked_mapped_allocated_regions); list_inithead(&memfd_file->weak_unlocked_mapped_allocated_regions); /* Initialize the memfd file with empty region and the allocation */ region = allocate_region(allocator, 0, allocation_size); region->zero_filled = true; /* ftruncate does zero-fill the new data */ list_add(®ion->list, &memfd_file->unmapped_allocated_regions); new_allocation->memory.memfd.file = memfd_file; new_allocation->memory.memfd.region = region; allocator->total_allocations += allocation_size; if (allocation_size == memfd_file->filesize) return true; /* Add empty region */ region = allocate_region(allocator, allocation_size, memfd_file->filesize - allocation_size); region->zero_filled = true; /* ftruncate does zero-fill the new data */ list_add(®ion->list, &memfd_file->free_regions); return true; } /* Allocate memory */ struct nine_allocation * nine_allocate(struct nine_allocator *allocator, unsigned size) { struct nine_allocation *new_allocation = slab_alloc_st(&allocator->allocation_pool); debug_dump_allocator_state(allocator); if (!new_allocation) return NULL; nine_flush_pending_frees(allocator); /* Restrict to >= page_size to prevent having too much fragmentation, as the size of * allocations is rounded to the next page_size multiple. */ if (size >= allocator->page_size && allocator->total_virtual_memory_limit >= 0 && nine_memfd_allocator(allocator, new_allocation, size)) { struct nine_memfd_file_region *region = new_allocation->memory.memfd.region; if (!region->zero_filled) { void *data = nine_get_pointer(allocator, new_allocation); if (!data) { ERR("INTERNAL MMAP FOR NEW ALLOCATION FAILED\n"); nine_free(allocator, new_allocation); return NULL; } DBG("memset on data=%p, size %d\n", data, region->size); memset(data, 0, region->size); region->zero_filled = true; /* Even though the user usually fills afterward, we don't weakrelease. * The reason is suballocations don't affect the weakrelease state of their * parents. Thus if only suballocations are accessed, the release would stay * weak forever. */ nine_pointer_strongrelease(allocator, new_allocation); } DBG("ALLOCATION SUCCESSFUL\n"); debug_dump_allocation_state(new_allocation); return new_allocation; } void *data = align_calloc(size, NINE_ALLOCATION_ALIGNMENT); if (!data) { DBG("ALLOCATION FAILED\n"); return NULL; } new_allocation->allocation_type = NINE_MALLOC_ALLOC; new_allocation->memory.malloc.buf = data; new_allocation->memory.malloc.allocation_size = size; list_inithead(&new_allocation->list_free); allocator->total_allocations += size; allocator->total_locked_memory += size; allocator->total_virtual_memory += size; DBG("ALLOCATION SUCCESSFUL\n"); debug_dump_allocation_state(new_allocation); return new_allocation; } /* Release memory */ static void nine_free_internal(struct nine_allocator *allocator, struct nine_allocation *allocation) { DBG("RELEASING ALLOCATION\n"); debug_dump_allocation_state(allocation); if (allocation->allocation_type == NINE_MALLOC_ALLOC) { allocator->total_allocations -= allocation->memory.malloc.allocation_size; allocator->total_locked_memory -= allocation->memory.malloc.allocation_size; allocator->total_virtual_memory -= allocation->memory.malloc.allocation_size; align_free(allocation->memory.malloc.buf); } else if (allocation->allocation_type == NINE_MEMFD_ALLOC || allocation->allocation_type == NINE_MEMFD_SUBALLOC) { struct nine_memfd_file *memfd_file = nine_get_memfd_file_backing(allocation); struct nine_memfd_file_region *region = nine_get_memfd_region_backing(allocation); if (allocation->weak_unlock) region->num_weak_unlocks--; if (allocation->allocation_type == NINE_MEMFD_ALLOC) remove_allocation(allocator, memfd_file, region); } slab_free_st(&allocator->allocation_pool, allocation); debug_dump_allocator_state(allocator); } void nine_free(struct nine_allocator *allocator, struct nine_allocation *allocation) { nine_flush_pending_frees(allocator); nine_flush_pending_releases(allocator); nine_free_internal(allocator, allocation); } /* Called from the worker thread. Similar to nine_free except we are not in the main thread, thus * we are disallowed to change the allocator structures except the fields reserved * for the worker. In addition, the allocation is allowed to not being unlocked (the release * will unlock it) */ void nine_free_worker(struct nine_allocator *allocator, struct nine_allocation *allocation) { /* Add the allocation to the list of pending allocations to free */ pthread_mutex_lock(&allocator->mutex_pending_frees); /* The order of free matters as suballocations are supposed to be released first */ list_addtail(&allocation->list_free, &allocator->pending_frees); pthread_mutex_unlock(&allocator->mutex_pending_frees); } /* Lock an allocation, and retrieve the pointer */ void * nine_get_pointer(struct nine_allocator *allocator, struct nine_allocation *allocation) { struct nine_memfd_file *memfd_file; struct nine_memfd_file_region *region; nine_flush_pending_releases(allocator); DBG("allocation_type: %d\n", allocation->allocation_type); if (allocation->allocation_type == NINE_MALLOC_ALLOC) return allocation->memory.malloc.buf; if (allocation->allocation_type == NINE_EXTERNAL_ALLOC) return allocation->memory.external.buf; memfd_file = nine_get_memfd_file_backing(allocation); region = nine_get_memfd_region_backing(allocation); if (!nine_memfd_region_map(allocator, memfd_file, region)) { DBG("Couldn't map memfd region for get_pointer\n"); return NULL; } move_region(&memfd_file->locked_mapped_allocated_regions, region); /* Note: redundant if region->num_locks */ region->num_locks++; if (region->num_locks == 1) allocator->total_locked_memory += region->size; if (allocation->weak_unlock) region->num_weak_unlocks--; allocation->weak_unlock = false; region->zero_filled = false; if (allocation->allocation_type == NINE_MEMFD_ALLOC) return region->map; if (allocation->allocation_type == NINE_MEMFD_SUBALLOC) return region->map + allocation->memory.submemfd.relative_offset; assert(false); return NULL; } /* Unlock an allocation, but with hint that we might lock again soon */ void nine_pointer_weakrelease(struct nine_allocator *allocator, struct nine_allocation *allocation) { struct nine_memfd_file_region *region; if (allocation->allocation_type > NINE_MEMFD_SUBALLOC) return; region = nine_get_memfd_region_backing(allocation); if (!allocation->weak_unlock) region->num_weak_unlocks++; allocation->weak_unlock = true; region->num_locks--; if (region->num_locks == 0) { struct nine_memfd_file *memfd_file = nine_get_memfd_file_backing(allocation); allocator->total_locked_memory -= region->size; move_region(&memfd_file->weak_unlocked_mapped_allocated_regions, region); } } /* Unlock an allocation */ void nine_pointer_strongrelease(struct nine_allocator *allocator, struct nine_allocation *allocation) { struct nine_memfd_file_region *region; if (allocation->allocation_type > NINE_MEMFD_SUBALLOC) return; region = nine_get_memfd_region_backing(allocation); region->num_locks--; if (region->num_locks == 0) { struct nine_memfd_file *memfd_file = nine_get_memfd_file_backing(allocation); allocator->total_locked_memory -= region->size; if (region->num_weak_unlocks) move_region(&memfd_file->weak_unlocked_mapped_allocated_regions, region); else move_region(&memfd_file->unlocked_mapped_allocated_regions, region); } } /* Delay a release to when a given counter becomes zero */ void nine_pointer_delayedstrongrelease(struct nine_allocator *allocator, struct nine_allocation *allocation, unsigned *counter) { if (allocation->allocation_type > NINE_MEMFD_SUBALLOC) return; assert(allocation->pending_counter == NULL || allocation->pending_counter == counter); allocation->pending_counter = counter; allocation->locks_on_counter++; if (list_is_empty(&allocation->list_release)) list_add(&allocation->list_release, &allocator->pending_releases); } /* Create a suballocation of an allocation */ struct nine_allocation * nine_suballocate(struct nine_allocator* allocator, struct nine_allocation *allocation, int offset) { struct nine_allocation *new_allocation = slab_alloc_st(&allocator->allocation_pool); if (!new_allocation) return NULL; DBG("Suballocate allocation at offset: %d\n", offset); assert(allocation->allocation_type != NINE_MEMFD_SUBALLOC); list_inithead(&new_allocation->list_free); if (allocation->allocation_type != NINE_MEMFD_ALLOC) { new_allocation->allocation_type = NINE_EXTERNAL_ALLOC; if (allocation->allocation_type == NINE_MALLOC_ALLOC) new_allocation->memory.external.buf = allocation->memory.malloc.buf + offset; else new_allocation->memory.external.buf = allocation->memory.external.buf + offset; return new_allocation; } new_allocation->allocation_type = NINE_MEMFD_SUBALLOC; new_allocation->memory.submemfd.parent = &allocation->memory.memfd; new_allocation->memory.submemfd.relative_offset = offset; new_allocation->locks_on_counter = 0; new_allocation->pending_counter = NULL; new_allocation->weak_unlock = false; list_inithead(&new_allocation->list_release); debug_dump_allocation_state(new_allocation); return new_allocation; } /* Wrap an external pointer as an allocation */ struct nine_allocation * nine_wrap_external_pointer(struct nine_allocator* allocator, void* data) { struct nine_allocation *new_allocation = slab_alloc_st(&allocator->allocation_pool); if (!new_allocation) return NULL; DBG("Wrapping external pointer: %p\n", data); new_allocation->allocation_type = NINE_EXTERNAL_ALLOC; new_allocation->memory.external.buf = data; list_inithead(&new_allocation->list_free); return new_allocation; } struct nine_allocator * nine_allocator_create(struct NineDevice9 *device, int memfd_virtualsizelimit) { struct nine_allocator* allocator = MALLOC(sizeof(struct nine_allocator)); if (!allocator) return NULL; allocator->device = device; allocator->page_size = sysconf(_SC_PAGESIZE); assert(allocator->page_size == 4 << 10); allocator->num_fd_max = (memfd_virtualsizelimit >= 0) ? MIN2(128, sysconf(_SC_OPEN_MAX)) : 0; allocator->min_file_size = DIVUP(100 * (1 << 20), allocator->page_size) * allocator->page_size; /* 100MB files */ allocator->total_allocations = 0; allocator->total_locked_memory = 0; allocator->total_virtual_memory = 0; allocator->total_virtual_memory_limit = memfd_virtualsizelimit * (1 << 20); allocator->num_fd = 0; DBG("Allocator created (ps: %d; fm: %d)\n", allocator->page_size, allocator->num_fd_max); slab_create(&allocator->allocation_pool, sizeof(struct nine_allocation), 4096); slab_create(&allocator->region_pool, sizeof(struct nine_memfd_file_region), 4096); allocator->memfd_pool = CALLOC(allocator->num_fd_max, sizeof(struct nine_memfd_file)); list_inithead(&allocator->pending_releases); list_inithead(&allocator->pending_frees); pthread_mutex_init(&allocator->mutex_pending_frees, NULL); return allocator; } void nine_allocator_destroy(struct nine_allocator* allocator) { int i; DBG("DESTROYING ALLOCATOR\n"); debug_dump_allocator_state(allocator); nine_flush_pending_releases(allocator); nine_flush_pending_frees(allocator); nine_memfd_files_unmap(allocator, true); pthread_mutex_destroy(&allocator->mutex_pending_frees); assert(list_is_empty(&allocator->pending_frees)); assert(list_is_empty(&allocator->pending_releases)); for (i = 0; i < allocator->num_fd; i++) { debug_dump_memfd_state(&allocator->memfd_pool[i], true); assert(list_is_empty(&allocator->memfd_pool[i].locked_mapped_allocated_regions)); assert(list_is_empty(&allocator->memfd_pool[i].weak_unlocked_mapped_allocated_regions)); assert(list_is_empty(&allocator->memfd_pool[i].unlocked_mapped_allocated_regions)); assert(list_is_singular(&allocator->memfd_pool[i].free_regions)); slab_free_st(&allocator->region_pool, list_first_entry(&allocator->memfd_pool[i].free_regions, struct nine_memfd_file_region, list)); close(allocator->memfd_pool[i].fd); } slab_destroy(&allocator->allocation_pool); slab_destroy(&allocator->region_pool); FREE(allocator->memfd_pool); FREE(allocator); } #else struct nine_allocation { unsigned is_external; void *external; }; struct nine_allocator { struct slab_mempool external_allocation_pool; pthread_mutex_t mutex_slab; }; struct nine_allocation * nine_allocate(struct nine_allocator *allocator, unsigned size) { struct nine_allocation *allocation; (void)allocator; assert(sizeof(struct nine_allocation) <= NINE_ALLOCATION_ALIGNMENT); allocation = align_calloc(size + NINE_ALLOCATION_ALIGNMENT, NINE_ALLOCATION_ALIGNMENT); allocation->is_external = false; return allocation; } void nine_free(struct nine_allocator *allocator, struct nine_allocation *allocation) { if (allocation->is_external) { pthread_mutex_lock(&allocator->mutex_slab); slab_free_st(&allocator->external_allocation_pool, allocation); pthread_mutex_unlock(&allocator->mutex_slab); } else align_free(allocation); } void nine_free_worker(struct nine_allocator *allocator, struct nine_allocation *allocation) { nine_free(allocator, allocation); } void *nine_get_pointer(struct nine_allocator *allocator, struct nine_allocation *allocation) { (void)allocator; if (allocation->is_external) return allocation->external; return (uint8_t *)allocation + NINE_ALLOCATION_ALIGNMENT; } void nine_pointer_weakrelease(struct nine_allocator *allocator, struct nine_allocation *allocation) { (void)allocator; (void)allocation; } void nine_pointer_strongrelease(struct nine_allocator *allocator, struct nine_allocation *allocation) { (void)allocator; (void)allocation; } void nine_pointer_delayedstrongrelease(struct nine_allocator *allocator, struct nine_allocation *allocation, unsigned *counter) { (void)allocator; (void)allocation; (void)counter; } struct nine_allocation * nine_suballocate(struct nine_allocator* allocator, struct nine_allocation *allocation, int offset) { struct nine_allocation *new_allocation; pthread_mutex_lock(&allocator->mutex_slab); new_allocation = slab_alloc_st(&allocator->external_allocation_pool); pthread_mutex_unlock(&allocator->mutex_slab); new_allocation->is_external = true; new_allocation->external = (uint8_t *)allocation + NINE_ALLOCATION_ALIGNMENT + offset; return new_allocation; } struct nine_allocation * nine_wrap_external_pointer(struct nine_allocator* allocator, void* data) { struct nine_allocation *new_allocation; pthread_mutex_lock(&allocator->mutex_slab); new_allocation = slab_alloc_st(&allocator->external_allocation_pool); pthread_mutex_unlock(&allocator->mutex_slab); new_allocation->is_external = true; new_allocation->external = data; return new_allocation; } struct nine_allocator * nine_allocator_create(struct NineDevice9 *device, int memfd_virtualsizelimit) { struct nine_allocator* allocator = MALLOC(sizeof(struct nine_allocator)); (void)device; (void)memfd_virtualsizelimit; if (!allocator) return NULL; slab_create(&allocator->external_allocation_pool, sizeof(struct nine_allocation), 4096); pthread_mutex_init(&allocator->mutex_slab, NULL); return allocator; } void nine_allocator_destroy(struct nine_allocator *allocator) { slab_destroy(&allocator->external_allocation_pool); pthread_mutex_destroy(&allocator->mutex_slab); } #endif /* NINE_ENABLE_MEMFD */