mirror of
https://github.com/trapexit/mergerfs.git
synced 2026-10-19 16:16:48 +00:00
Vendor and integrate rpmalloc 1.4.5
- Add vendored/rpmalloc/ with upstream source, license, README, changelog - Build rpmalloc.o directly linked into mergerfs and tests - USE_RPMALLOC=1 by default, disabled under SANITIZE - Static builds use rpmalloc-static.c.o with pthread_create interposer disabled - Static builds use --allow-multiple-definition for glibc malloc replacement - rpmalloc provides malloc, free, new, delete, and related overrides - Add rpmalloc to DEPENDENCIES
This commit is contained in:
Antonio SJ Musumeci
@@ -6,6 +6,7 @@
|
||||
* rapidhash: https://github.com/Nicoshev/rapidhash
|
||||
* fmt: https://github.com/fmtlib/fmt
|
||||
* concurrentqueue: https://github.com/cameron314/concurrentqueue
|
||||
* rpmalloc: https://github.com/mjansson/rpmalloc
|
||||
* scope_guard: https://github.com/Neargye/scope_guard
|
||||
* subprocess: https://github.com/arun11299/cpp-subprocess
|
||||
* CLI11: https://github.com/CLIUtils/CLI11
|
||||
|
||||
@@ -48,8 +48,19 @@ endif
|
||||
endif
|
||||
|
||||
USE_XATTR ?= 1
|
||||
ifdef SANITIZE
|
||||
USE_RPMALLOC ?= 0
|
||||
else
|
||||
USE_RPMALLOC ?= 1
|
||||
endif
|
||||
UGID_USE_RWLOCK ?= 0
|
||||
|
||||
ifdef SANITIZE
|
||||
ifeq ($(USE_RPMALLOC),1)
|
||||
$(error SANITIZE is not supported with USE_RPMALLOC=1)
|
||||
endif
|
||||
endif
|
||||
|
||||
ifdef NDEBUG
|
||||
OPT_FLAGS := -O2 -DNDEBUG
|
||||
else
|
||||
@@ -140,6 +151,23 @@ LDLIBS := \
|
||||
-pthread
|
||||
override LDLIBS += \
|
||||
$(LIBFUSE)
|
||||
ifeq ($(USE_RPMALLOC),1)
|
||||
RPMALLOC_OBJ := vendored/rpmalloc/$(BUILDDIR)/rpmalloc$(if $(filter 1,$(STATIC)),-static).c.o
|
||||
RPMALLOC_FLAGS := -DENABLE_OVERRIDE=1 -DENABLE_PRELOAD=1
|
||||
ifeq ($(STATIC),1)
|
||||
RPMALLOC_FLAGS += -DRPMALLOC_NO_PTHREAD_CREATE_OVERRIDE=1
|
||||
override LDFLAGS += \
|
||||
-Wl,--allow-multiple-definition
|
||||
else
|
||||
override LDLIBS += \
|
||||
-ldl
|
||||
endif
|
||||
LINK_OBJS := $(OBJS) $(RPMALLOC_OBJ)
|
||||
TESTS_LINK_OBJS := $(TESTS_OBJS) $(RPMALLOC_OBJ)
|
||||
else
|
||||
LINK_OBJS := $(OBJS)
|
||||
TESTS_LINK_OBJS := $(TESTS_OBJS)
|
||||
endif
|
||||
ifeq ($(CXX),g++)
|
||||
GCC_VERSION := $(shell $(CXX) -dumpversion | cut -f1 -d.)
|
||||
ifeq ($(shell test $(GCC_VERSION) -lt 9; echo $$?),0)
|
||||
@@ -172,6 +200,7 @@ all: libfuse $(BUILDDIR)/mergerfs $(BUILDDIR)/fsck.mergerfs $(BUILDDIR)/mergerfs
|
||||
help:
|
||||
@echo "usage: make ARG\n"
|
||||
@echo "USE_XATTR=0 - build program without xattrs functionality"
|
||||
@echo "USE_RPMALLOC=0 - build program without rpmalloc"
|
||||
@echo "STATIC=1 - build static binary"
|
||||
@echo "LTO=1 - build with link time optimization"
|
||||
@echo "SANITIZE=1 - build with sanitizers (address,undefined,leak)"
|
||||
@@ -181,8 +210,8 @@ help:
|
||||
@echo "GITREF=gitref - gitref to use for release builds"
|
||||
|
||||
|
||||
$(BUILDDIR)/mergerfs: $(LIBFUSE) src/version.hpp $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(OBJS) -o $@ $(LDFLAGS) $(LDLIBS)
|
||||
$(BUILDDIR)/mergerfs: $(LIBFUSE) src/version.hpp $(LINK_OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(LINK_OBJS) -o $@ $(LDFLAGS) $(LDLIBS)
|
||||
|
||||
$(BUILDDIR)/fsck.mergerfs: $(BUILDDIR)/mergerfs
|
||||
$(LN) -sf "mergerfs" $@
|
||||
@@ -190,8 +219,8 @@ $(BUILDDIR)/fsck.mergerfs: $(BUILDDIR)/mergerfs
|
||||
$(BUILDDIR)/mergerfs.collect-info: $(BUILDDIR)/mergerfs
|
||||
$(LN) -sf "mergerfs" $@
|
||||
|
||||
$(BUILDDIR)/tests: $(BUILDDIR)/mergerfs $(TESTS_OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(TESTS_OBJS) -o $@ $(LDFLAGS) $(LDLIBS)
|
||||
$(BUILDDIR)/tests: $(BUILDDIR)/mergerfs $(TESTS_LINK_OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(TESTS_LINK_OBJS) -o $@ $(LDFLAGS) $(LDLIBS)
|
||||
|
||||
.PHONY: libfuse
|
||||
libfuse: $(LIBFUSE)
|
||||
@@ -200,6 +229,12 @@ libfuse: $(LIBFUSE)
|
||||
$(LIBFUSE):
|
||||
$(MAKE) -C vendored/libfuse $(BUILDDIR)/libfuse.a
|
||||
|
||||
ifeq ($(USE_RPMALLOC),1)
|
||||
$(RPMALLOC_OBJ): vendored/rpmalloc/rpmalloc.c vendored/rpmalloc/rpmalloc.h vendored/rpmalloc/malloc.c
|
||||
$(MKDIR) -p vendored/rpmalloc/$(BUILDDIR)
|
||||
$(CC) $(CFLAGS) $(INC_FLAGS) $(CPPFLAGS) $(RPMALLOC_FLAGS) -c $< -o $@
|
||||
endif
|
||||
|
||||
tests: $(BUILDDIR)/tests
|
||||
|
||||
.PHONY:
|
||||
@@ -235,6 +270,7 @@ preload: $(BUILDDIR)/preload.so
|
||||
.PHONY: clean
|
||||
clean: rpm-clean
|
||||
$(RM) -rf $(BUILDDIR)
|
||||
$(RM) -rf vendored/rpmalloc/$(BUILDDIR)
|
||||
$(FIND) . -name "*~" -delete
|
||||
$(MAKE) -C vendored/libfuse clean
|
||||
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
1.4.5
|
||||
|
||||
Fix for issue where span cache was not properly freed on heap free all
|
||||
|
||||
Add rpmalloc_get_heap_for_ptr function to get heap for a given allocation
|
||||
|
||||
Fixed medium size block limit and alignment handling when span size is reconfigured
|
||||
|
||||
Fixed SIGILL on macOS rosetta mode (x86_64)
|
||||
|
||||
Update compatibility with newer clang versions and use intrinsics for memcpy
|
||||
|
||||
|
||||
1.4.4
|
||||
|
||||
Fixed an issue where an external thread concurrently freeing a block to the deferred list of
|
||||
a heap at the same time as owner thread freeing the last used block could cause a race condition
|
||||
ending in spen being freed multiple time.
|
||||
|
||||
Added fallback path when huge page allocation fails to allocate and promote new pages as a
|
||||
transparent huge page
|
||||
|
||||
Added option to name pages on Linux and Android.
|
||||
|
||||
Compilation compatibility updates for MSYS2, FreeBSD, MacOS/clang and tinycc.
|
||||
|
||||
1.4.3
|
||||
|
||||
Fixed an issue where certain combinations of memory page size and span map counts could cause
|
||||
a deadlock in the mapping of new memory pages.
|
||||
|
||||
Tweaked cache levels and avoid setting spans as reserved in a heap when the heap already has
|
||||
spans in the thread cache to improve cache usage.
|
||||
|
||||
Prefer flags to more actively evict physical pages in madvise calls when partially unmapping
|
||||
span ranges on POSIX systems.
|
||||
|
||||
|
||||
1.4.2
|
||||
|
||||
Fixed an issue where calling _exit might hang the main thread cleanup in rpmalloc if another
|
||||
worker thread was terminated while holding exclusive access to the global cache.
|
||||
|
||||
Improved caches to prioritize main spans in a chunk to avoid leaving main spans mapped due to
|
||||
remaining subspans in caches.
|
||||
|
||||
Improve cache reuse by allowing large blocks to use caches from slightly larger cache classes.
|
||||
|
||||
Fixed an issue where thread heap statistics would go out of sync when a free span was deferred
|
||||
to another thread heap
|
||||
|
||||
API breaking change - added flag to rpmalloc_thread_finalize to avoid releasing thread caches.
|
||||
Pass nonzero value to retain old behaviour of releasing thread caches to global cache.
|
||||
|
||||
Add option to config to set a custom error callback for assert failures (if ENABLE_ASSERT)
|
||||
|
||||
|
||||
1.4.1
|
||||
|
||||
Dual license as both released to public domain or under MIT license
|
||||
|
||||
Allow up to 4GiB memory page sizes
|
||||
|
||||
Fix an issue where large page sizes in conjunction with many threads waste a lot of memory (previously
|
||||
each heap occupied an entire memory page, now heaps can now share a memory page)
|
||||
|
||||
Fixed compilation issue on macOS when ENABLE_PRELOAD is set but not ENABLE_OVERRIDE
|
||||
|
||||
New first class heap API allowing explicit heap control and release of entire heap in a single call
|
||||
|
||||
Added rpaligned_calloc function for aligned and zero intialized allocations
|
||||
|
||||
Fixed natural alignment check in rpaligned_realloc to 16 bytes (check was 32, which is wrong)
|
||||
|
||||
Minor performance improvements for all code paths by simplified span handling
|
||||
|
||||
Minor performance improvements and for aligned allocations with alignment less or equal to 128 bytes
|
||||
by utilizing natural block alignments
|
||||
|
||||
Refactor finalization to be compatible with global scope data causing dynamic allocations and frees, like
|
||||
C++ objects with custom ctors/dtors
|
||||
|
||||
Refactor thread and global cache to be array based instead of list based for improved performance
|
||||
and cache size control
|
||||
|
||||
Added missing C++ operator overloads with ENABLE_OVERRIDE when using Microsoft C++ runtimes
|
||||
|
||||
Fixed issue in pvalloc override that could return less than a memory page in usable size
|
||||
|
||||
Added a missing null check in the non-hot allocation code paths
|
||||
|
||||
|
||||
1.4.0
|
||||
|
||||
Improved cross thread deallocations by using per-span atomic free list to minimize thread
|
||||
contention and localize free list processing to actual span
|
||||
|
||||
Change span free list to a linked list, conditionally initialized one memory page at a time
|
||||
|
||||
Reduce number of conditionals in the fast path allocation and avoid touching heap structure
|
||||
at all in best case
|
||||
|
||||
Avoid realigning block in deallocation unless span marked as used by alignment > 32 bytes
|
||||
|
||||
Revert block granularity and natural alignment to 16 bytes to reduce memory waste
|
||||
|
||||
Bugfix for preserving data when reallocating a previously aligned (>32 bytes) block
|
||||
|
||||
Use compile time span size by default for improved performance, added build time RPMALLOC_CONFIGURABLE
|
||||
preprocessor directive to reenable configurability of span and page size
|
||||
|
||||
More detailed statistics
|
||||
|
||||
Disabled adaptive thread cache by default
|
||||
|
||||
Fixed an issue where reallocations of large blocks could read outsize of memory page boundaries
|
||||
|
||||
Tag mmap requests on macOS with tag 240 for identification with vmmap tool
|
||||
|
||||
|
||||
1.3.2
|
||||
|
||||
Support for alignment equal or larger than memory page size, up to span size
|
||||
|
||||
Added adaptive thread cache size based on thread allocation load
|
||||
|
||||
Fix 32-bit MSVC Windows builds using incorrect 64-bit pointer CAS
|
||||
|
||||
Updated compatibility with clang toolchain and Python 3
|
||||
|
||||
Support preconfigured huge pages
|
||||
|
||||
Moved active heap counter to statistics
|
||||
|
||||
Moved repository to https://github.com/mjansson/rpmalloc
|
||||
|
||||
|
||||
1.3.1
|
||||
|
||||
Support for huge pages
|
||||
|
||||
Bugfix to old size in aligned realloc and usable size for aligned allocs when alignment > 32
|
||||
|
||||
Use C11 atomics for non-Microsoft compilers
|
||||
|
||||
Remove remaining spin-lock like control for caches, all operations are now lock free
|
||||
|
||||
Allow large deallocations to cross thread heaps
|
||||
|
||||
|
||||
1.3.0
|
||||
|
||||
Make span size configurable and all spans equal in size, removing span size classes and streamlining the thread cache.
|
||||
|
||||
Allow super spans to be reserved in advance and split up in multiple used spans to reduce number of system calls. This will not increase committed physical pages, only reserved virtual memory space.
|
||||
|
||||
Allow super spans to be reused for allocations of lower size, breaking up the super span and storing remainder in thread cache in order to reduce load on global cache and reduce cache overhead.
|
||||
|
||||
Fixed an issue where an allocation of zero bytes would cause a segmentation fault from indexing size class array with index -1.
|
||||
|
||||
Fixed an issue where an allocation of maximum large block size (2097120 bytes) would index the heap cache array out of bounds and potentially cause a segmentation fault depending on earlier allocation patterns.
|
||||
|
||||
Fixed an issue where memory pages at start of aligned span run was not completely unmapped on POSIX systems.
|
||||
|
||||
Fixed an issue where spans were not correctly marked as owned by the heap after traversing the global span cache.
|
||||
|
||||
Added function to access the allocator configuration after initialization to find default values.
|
||||
|
||||
Removed allocated and reserved statistics to reduce code complexity.
|
||||
|
||||
|
||||
1.2.2
|
||||
|
||||
Add configurable memory mapper providing map/unmap of memory pages. Default to VirtualAlloc/mmap if none provided. This allows rpmalloc to be used in contexts where memory is provided by internal means.
|
||||
|
||||
Avoid using explicit memory map addresses to mmap on POSIX systems. Instead use overallocation of virtual memory space to gain 64KiB alignment of spans. Since extra pages are never touched this should have no impact on real memory usage and remove the possibility of contention in virtual address space with other uses of mmap.
|
||||
|
||||
Detect system memory page size at initialization, and allow page size to be set explicitly in initialization. This allows the allocator to be used as a sub-allocator where the page granularity should be lower to reduce risk of wasting unused memory ranges, and adds support for modern iOS devices where page size is 16KiB.
|
||||
|
||||
Add build time option to use memory guards, surrounding each allocated block with a dead zone which is checked for consistency when block is freed.
|
||||
|
||||
Always finalize thread on allocator finalization, fixing issue when re-initializing allocator in the same thread.
|
||||
|
||||
Add basic allocator test cases
|
||||
|
||||
|
||||
1.2.1
|
||||
|
||||
Split library into rpmalloc only base library and preloadable malloc wrapper library.
|
||||
|
||||
Add arg validation to valloc and pvalloc.
|
||||
|
||||
Change ARM memory barrier instructions to dmb ish/ishst for compatibility.
|
||||
|
||||
Improve preload compatibility on Apple platforms by using pthread key for TLS in wrapper library.
|
||||
|
||||
Fix ABA issue in orphaned heap linked list
|
||||
|
||||
|
||||
1.2
|
||||
|
||||
Dual license under MIT
|
||||
|
||||
Fix init/fini checks in malloc entry points for preloading into binaries that does malloc/free in init or fini sections
|
||||
|
||||
Fixed an issue where freeing a block which had been realigned during allocation due to alignment request greater than 16 caused the free block link to be written in the wrong place in the block, causing next allocation from the size class to return a bad pointer
|
||||
|
||||
Improve mmap 64KiB granularity enforcement loop to avoid excessive iterations
|
||||
|
||||
Fix undersized adaptive cache counter array for large block in heap structure, causing potential abort on exit
|
||||
|
||||
Avoid hysteresis in realloc by overallocating on small size increases
|
||||
|
||||
Add entry point for realloc with alignment and optional flags to avoid preserving content
|
||||
|
||||
Add valloc/pvalloc/cfree wrappers
|
||||
|
||||
Add C++ new/delete wrappers
|
||||
|
||||
|
||||
1.1
|
||||
|
||||
Add four cache presets (unlimited, performance priority, size priority and no cache)
|
||||
|
||||
Slight performance improvement by dependent class index lookup for merged size classes
|
||||
|
||||
Adaptive cache size per thread and per size class for improved memory efficiency, and release thread caches to global cache in fixed size batches
|
||||
|
||||
Merged caches for small/medium classes using 64KiB spans with 64KiB large blocks
|
||||
|
||||
Require thread initialization with rpmalloc_thread_initialize, add pthread hooks for automatic init/fini
|
||||
|
||||
Added rpmalloc_usable_size query entry point
|
||||
|
||||
Fix invalid old size in memory copy during realloc
|
||||
|
||||
Optional statistics and integer overflow guards
|
||||
|
||||
Optional asserts for easier debugging
|
||||
|
||||
Provide malloc entry point replacements and automatic init/fini hooks, and a LD_PRELOAD:able dynamic library build
|
||||
|
||||
Improve documentation and additional code comments
|
||||
|
||||
Move benchmarks to separate repo, https://github.com/mjansson/rpmalloc-benchmark
|
||||
|
||||
|
||||
1.0
|
||||
|
||||
Initial release
|
||||
@@ -0,0 +1,52 @@
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <http://unlicense.org>
|
||||
|
||||
|
||||
You can also use this software under the MIT license if public domain
|
||||
is not recognized in your country
|
||||
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017 Mattias Jansson
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -0,0 +1,221 @@
|
||||
# rpmalloc - General Purpose Memory Allocator
|
||||
This library provides a public domain cross platform lock free thread caching 16-byte aligned memory allocator implemented in C. The latest source code is always available at https://github.com/mjansson/rpmalloc
|
||||
|
||||
Created by Mattias Jansson ([@maniccoder](https://twitter.com/maniccoder)) - Discord server for discussions at https://discord.gg/M8BwTQrt6c
|
||||
|
||||
Platforms currently supported:
|
||||
|
||||
- Windows
|
||||
- MacOS
|
||||
- iOS
|
||||
- Linux
|
||||
- Android
|
||||
- Haiku
|
||||
|
||||
The code should be easily portable to any platform with atomic operations and an mmap-style virtual memory management API. The API used to map/unmap memory pages can be configured in runtime to a custom implementation and mapping granularity/size.
|
||||
|
||||
This library is put in the public domain; you can redistribute it and/or modify it without any restrictions. Or, if you choose, you can use it under the MIT license.
|
||||
|
||||
# Performance
|
||||
We believe rpmalloc is faster than most popular memory allocators like tcmalloc, hoard, ptmalloc3 and others without causing extra allocated memory overhead in the thread caches compared to these allocators. We also believe the implementation to be easier to read and modify compared to these allocators, as it is a single source file of ~3000 lines of C code. All allocations have a natural 16-byte alignment.
|
||||
|
||||
Contained in a parallel repository is a benchmark utility that performs interleaved unaligned allocations and deallocations (both in-thread and cross-thread) in multiple threads. It measures number of memory operations performed per CPU second, as well as memory overhead by comparing the virtual memory mapped with the number of bytes requested in allocation calls. The setup of number of thread, cross-thread deallocation rate and allocation size limits is configured by command line arguments.
|
||||
|
||||
https://github.com/mjansson/rpmalloc-benchmark
|
||||
|
||||
Below is an example performance comparison chart of rpmalloc and other popular allocator implementations, with default configurations used.
|
||||
|
||||
![Ubuntu 16.10, random [16, 8000] bytes, 8 cores](https://docs.google.com/spreadsheets/d/1NWNuar1z0uPCB5iVS_Cs6hSo2xPkTmZf0KsgWS_Fb_4/pubchart?oid=301017877&format=image)
|
||||
|
||||
The benchmark producing these numbers were run on an Ubuntu 16.10 machine with 8 logical cores (4 physical, HT). The actual numbers are not to be interpreted as absolute performance figures, but rather as relative comparisons between the different allocators. For additional benchmark results, see the [BENCHMARKS](BENCHMARKS.md) file.
|
||||
|
||||
Configuration of the thread and global caches can be important depending on your use pattern. See [CACHE](CACHE.md) for a case study and some comments/guidelines.
|
||||
|
||||
# Required functions
|
||||
|
||||
Before calling any other function in the API, you __MUST__ call the initialization function, either __rpmalloc_initialize__ or __rpmalloc_initialize_config__, or you will get undefined behaviour when calling other rpmalloc entry point.
|
||||
|
||||
Before terminating your use of the allocator, you __SHOULD__ call __rpmalloc_finalize__ in order to release caches and unmap virtual memory, as well as prepare the allocator for global scope cleanup at process exit or dynamic library unload depending on your use case.
|
||||
|
||||
# Using
|
||||
The easiest way to use the library is simply adding __rpmalloc.[h|c]__ to your project and compile them along with your sources. This contains only the rpmalloc specific entry points and does not provide internal hooks to process and/or thread creation at the moment. You are required to call these functions from your own code in order to initialize and finalize the allocator in your process and threads:
|
||||
|
||||
__rpmalloc_initialize__ : Call at process start to initialize the allocator
|
||||
|
||||
__rpmalloc_initialize_config__ : Optional entry point to call at process start to initialize the allocator with a custom memory mapping backend, memory page size and mapping granularity.
|
||||
|
||||
__rpmalloc_finalize__: Call at process exit to finalize the allocator
|
||||
|
||||
__rpmalloc_thread_initialize__: Call at each thread start to initialize the thread local data for the allocator
|
||||
|
||||
__rpmalloc_thread_finalize__: Call at each thread exit to finalize and release thread cache back to global cache
|
||||
|
||||
__rpmalloc_config__: Get the current runtime configuration of the allocator
|
||||
|
||||
Then simply use the __rpmalloc__/__rpfree__ and the other malloc style replacement functions. Remember all allocations are 16-byte aligned, so no need to call the explicit rpmemalign/rpaligned_alloc/rpposix_memalign functions unless you need greater alignment, they are simply wrappers to make it easier to replace in existing code.
|
||||
|
||||
If you wish to override the standard library malloc family of functions and have automatic initialization/finalization of process and threads, define __ENABLE_OVERRIDE__ to non-zero which will include the `malloc.c` file in compilation of __rpmalloc.c__, and then rebuild the library or your project where you added the rpmalloc source. If you compile rpmalloc as a separate library you must make the linker use the override symbols from the library by referencing at least one symbol. The easiest way is to simply include `rpmalloc.h` in at least one source file and call `rpmalloc_linker_reference` somewhere - it's a dummy empty function. On Windows platforms and C++ overrides you have to `#include <rpnew.h>` in at least one source file and also manually handle the initialize/finalize of the process and all threads. The list of libc entry points replaced may not be complete, use libc/stdc++ replacement only as a convenience for testing the library on an existing code base, not a final solution.
|
||||
|
||||
For explicit first class heaps, see the __rpmalloc_heap_*__ API under [first class heaps](#first-class-heaps) section, requiring __RPMALLOC_FIRST_CLASS_HEAPS__ tp be defined to 1.
|
||||
|
||||
# Building
|
||||
To compile as a static library run the configure python script which generates a Ninja build script, then build using ninja. The ninja build produces two static libraries, one named `rpmalloc` and one named `rpmallocwrap`, where the latter includes the libc entry point overrides.
|
||||
|
||||
The configure + ninja build also produces two shared object/dynamic libraries. The `rpmallocwrap` shared library can be used with LD_PRELOAD/DYLD_INSERT_LIBRARIES to inject in a preexisting binary, replacing any malloc/free family of function calls. This is only implemented for Linux and macOS targets. The list of libc entry points replaced may not be complete, use preloading as a convenience for testing the library on an existing binary, not a final solution. The dynamic library also provides automatic init/fini of process and threads for all platforms.
|
||||
|
||||
The latest stable release is available in the master branch. For latest development code, use the develop branch.
|
||||
|
||||
# Cache configuration options
|
||||
Free memory pages are cached both per thread and in a global cache for all threads. The size of the thread caches is determined by an adaptive scheme where each cache is limited by a percentage of the maximum allocation count of the corresponding size class. The size of the global caches is determined by a multiple of the maximum of all thread caches. The factors controlling the cache sizes can be set by editing the individual defines in the `rpmalloc.c` source file for fine tuned control.
|
||||
|
||||
__ENABLE_UNLIMITED_CACHE__: By default defined to 0, set to 1 to make all caches infinite, i.e never release spans to global cache unless thread finishes and never unmap memory pages back to the OS. Highest performance but largest memory overhead.
|
||||
|
||||
__ENABLE_UNLIMITED_GLOBAL_CACHE__: By default defined to 0, set to 1 to make global caches infinite, i.e never unmap memory pages back to the OS.
|
||||
|
||||
__ENABLE_UNLIMITED_THREAD_CACHE__: By default defined to 0, set to 1 to make thread caches infinite, i.e never release spans to global cache unless thread finishes.
|
||||
|
||||
__ENABLE_GLOBAL_CACHE__: By default defined to 1, enables the global cache shared between all threads. Set to 0 to disable the global cache and directly unmap pages evicted from the thread cache.
|
||||
|
||||
__ENABLE_THREAD_CACHE__: By default defined to 1, enables the per-thread cache. Set to 0 to disable the thread cache and directly unmap pages no longer in use (also disables the global cache).
|
||||
|
||||
__ENABLE_ADAPTIVE_THREAD_CACHE__: Introduces a simple heuristics in the thread cache size, keeping 25% of the high water mark for each span count class.
|
||||
|
||||
# Other configuration options
|
||||
Detailed statistics are available if __ENABLE_STATISTICS__ is defined to 1 (default is 0, or disabled), either on compile command line or by setting the value in `rpmalloc.c`. This will cause a slight overhead in runtime to collect statistics for each memory operation, and will also add 4 bytes overhead per allocation to track sizes.
|
||||
|
||||
Integer safety checks on all calls are enabled if __ENABLE_VALIDATE_ARGS__ is defined to 1 (default is 0, or disabled), either on compile command line or by setting the value in `rpmalloc.c`. If enabled, size arguments to the global entry points are verified not to cause integer overflows in calculations.
|
||||
|
||||
Asserts are enabled if __ENABLE_ASSERTS__ is defined to 1 (default is 0, or disabled), either on compile command line or by setting the value in `rpmalloc.c`.
|
||||
|
||||
To include __malloc.c__ in compilation and provide overrides of standard library malloc entry points define __ENABLE_OVERRIDE__ to 1. To enable automatic initialization of finalization of process and threads in order to preload the library into executables using standard library malloc, define __ENABLE_PRELOAD__ to 1.
|
||||
|
||||
To enable the runtime configurable memory page and span sizes, define __RPMALLOC_CONFIGURABLE__ to 1. By default, memory page size is determined by system APIs and memory span size is set to 64KiB.
|
||||
|
||||
To enable support for first class heaps, define __RPMALLOC_FIRST_CLASS_HEAPS__ to 1. By default, the first class heap API is disabled.
|
||||
|
||||
# Huge pages
|
||||
The allocator has support for huge/large pages on Windows, Linux and MacOS. To enable it, pass a non-zero value in the config value `enable_huge_pages` when initializing the allocator with `rpmalloc_initialize_config`. If the system does not support huge pages it will be automatically disabled. You can query the status by looking at `enable_huge_pages` in the config returned from a call to `rpmalloc_config` after initialization is done.
|
||||
|
||||
# Quick overview
|
||||
The allocator is similar in spirit to tcmalloc from the [Google Performance Toolkit](https://github.com/gperftools/gperftools). It uses separate heaps for each thread and partitions memory blocks according to a preconfigured set of size classes, up to 2MiB. Larger blocks are mapped and unmapped directly. Allocations for different size classes will be served from different set of memory pages, each "span" of pages is dedicated to one size class. Spans of pages can flow between threads when the thread cache overflows and are released to a global cache, or when the thread ends. Unlike tcmalloc, single blocks do not flow between threads, only entire spans of pages.
|
||||
|
||||
# Implementation details
|
||||
The allocator is based on a fixed but configurable page alignment (defaults to 64KiB) and 16 byte block alignment, where all runs of memory pages (spans) are mapped to this alignment boundary. On Windows this is automatically guaranteed up to 64KiB by the VirtualAlloc granularity, and on mmap systems it is achieved by oversizing the mapping and aligning the returned virtual memory address to the required boundaries. By aligning to a fixed size the free operation can locate the header of the memory span without having to do a table lookup (as tcmalloc does) by simply masking out the low bits of the address (for 64KiB this would be the low 16 bits).
|
||||
|
||||
Memory blocks are divided into three categories. For 64KiB span size/alignment the small blocks are [16, 1024] bytes, medium blocks (1024, 32256] bytes, and large blocks (32256, 2097120] bytes. The three categories are further divided in size classes. If the span size is changed, the small block classes remain but medium blocks go from (1024, span size] bytes.
|
||||
|
||||
Small blocks have a size class granularity of 16 bytes each in 64 buckets. Medium blocks have a granularity of 512 bytes, 61 buckets (default). Large blocks have the same granularity as the configured span size (default 64KiB). All allocations are fitted to these size class boundaries (an allocation of 36 bytes will allocate a block of 48 bytes). Each small and medium size class has an associated span (meaning a contiguous set of memory pages) configuration describing how many pages the size class will allocate each time the cache is empty and a new allocation is requested.
|
||||
|
||||
Spans for small and medium blocks are cached in four levels to avoid calls to map/unmap memory pages. The first level is a per thread single active span for each size class. The second level is a per thread list of partially free spans for each size class. The third level is a per thread list of free spans. The fourth level is a global list of free spans.
|
||||
|
||||
Each span for a small and medium size class keeps track of how many blocks are allocated/free, as well as a list of which blocks that are free for allocation. To avoid locks, each span is completely owned by the allocating thread, and all cross-thread deallocations will be deferred to the owner thread through a separate free list per span.
|
||||
|
||||
Large blocks, or super spans, are cached in two levels. The first level is a per thread list of free super spans. The second level is a global list of free super spans.
|
||||
|
||||
# Memory mapping
|
||||
By default the allocator uses OS APIs to map virtual memory pages as needed, either `VirtualAlloc` on Windows or `mmap` on POSIX systems. If you want to use your own custom memory mapping provider you can use __rpmalloc_initialize_config__ and pass function pointers to map and unmap virtual memory. These function should reserve and free the requested number of bytes.
|
||||
|
||||
The returned memory address from the memory map function MUST be aligned to the memory page size and the memory span size (which ever is larger), both of which is configurable. Either provide the page and span sizes during initialization using __rpmalloc_initialize_config__, or use __rpmalloc_config__ to find the required alignment which is equal to the maximum of page and span size. The span size MUST be a power of two in [4096, 262144] range, and be a multiple or divisor of the memory page size.
|
||||
|
||||
Memory mapping requests are always done in multiples of the memory page size. You can specify a custom page size when initializing rpmalloc with __rpmalloc_initialize_config__, or pass 0 to let rpmalloc determine the system memory page size using OS APIs. The page size MUST be a power of two.
|
||||
|
||||
To reduce system call overhead, memory spans are mapped in batches controlled by the `span_map_count` configuration variable (which defaults to the `DEFAULT_SPAN_MAP_COUNT` value if 0, which in turn is sized according to the cache configuration define, defaulting to 64). If the memory page size is larger than the span size, the number of spans to map in a single call will be adjusted to guarantee a multiple of the page size, and the spans will be kept mapped until the entire span range can be unmapped in one call (to avoid trying to unmap partial pages).
|
||||
|
||||
On macOS and iOS mmap requests are tagged with tag 240 for easy identification with the vmmap tool.
|
||||
|
||||
# Span breaking
|
||||
Super spans (spans a multiple > 1 of the span size) can be subdivided into smaller spans to fulfill a need to map a new span of memory. By default the allocator will greedily grab and break any larger span from the available caches before mapping new virtual memory. However, spans can currently not be glued together to form larger super spans again. Subspans can traverse the cache and be used by different threads individually.
|
||||
|
||||
A span that is a subspan of a larger super span can be individually decommitted to reduce physical memory pressure when the span is evicted from caches and scheduled to be unmapped. The entire original super span will keep track of the subspans it is broken up into, and when the entire range is decommitted the super span will be unmapped. This allows platforms like Windows that require the entire virtual memory range that was mapped in a call to VirtualAlloc to be unmapped in one call to VirtualFree, while still decommitting individual pages in subspans (if the page size is smaller than the span size).
|
||||
|
||||
If you use a custom memory map/unmap function you need to take this into account by looking at the `release` parameter given to the `memory_unmap` function. It is set to 0 for decommitting individual pages and the total super span byte size for finally releasing the entire super span memory range.
|
||||
|
||||
# Memory fragmentation
|
||||
There is no memory fragmentation by the allocator in the sense that it will not leave unallocated and unusable "holes" in the memory pages by calls to allocate and free blocks of different sizes. This is due to the fact that the memory pages allocated for each size class is split up in perfectly aligned blocks which are not reused for a request of a different size. The block freed by a call to `rpfree` will always be immediately available for an allocation request within the same size class.
|
||||
|
||||
However, there is memory fragmentation in the meaning that a request for x bytes followed by a request of y bytes where x and y are at least one size class different in size will return blocks that are at least one memory page apart in virtual address space. Only blocks of the same size will potentially be within the same memory page span.
|
||||
|
||||
rpmalloc keeps an "active span" and free list for each size class. This leads to back-to-back allocations will most likely be served from within the same span of memory pages (unless the span runs out of free blocks). The rpmalloc implementation will also use any "holes" in memory pages in semi-filled spans before using a completely free span.
|
||||
|
||||
# First class heaps
|
||||
rpmalloc provides a first class heap type with explicit heap control API. Heaps are maintained with calls to __rpmalloc_heap_acquire__ and __rpmalloc_heap_release__ and allocations/frees are done with __rpmalloc_heap_alloc__ and __rpmalloc_heap_free__. See the `rpmalloc.h` documentation for the full list of functions in the heap API. The main use case of explicit heap control is to scope allocations in a heap and release everything with a single call to __rpmalloc_heap_free_all__ without having to maintain ownership of memory blocks. Note that the heap API is not thread-safe, the caller must make sure that each heap is only used in a single thread at any given time.
|
||||
|
||||
# Producer-consumer scenario
|
||||
Compared to the some other allocators, rpmalloc does not suffer as much from a producer-consumer thread scenario where one thread allocates memory blocks and another thread frees the blocks. In some allocators the free blocks need to traverse both the thread cache of the thread doing the free operations as well as the global cache before being reused in the allocating thread. In rpmalloc the freed blocks will be reused as soon as the allocating thread needs to get new spans from the thread cache. This enables faster release of completely freed memory pages as blocks in a memory page will not be aliased between different owning threads.
|
||||
|
||||
# Best case scenarios
|
||||
Threads that keep ownership of allocated memory blocks within the thread and free the blocks from the same thread will have optimal performance.
|
||||
|
||||
Threads that have allocation patterns where the difference in memory usage high and low water marks fit within the thread cache thresholds in the allocator will never touch the global cache except during thread init/fini and have optimal performance. Tweaking the cache limits can be done on a per-size-class basis.
|
||||
|
||||
# Worst case scenarios
|
||||
Since each thread cache maps spans of memory pages per size class, a thread that allocates just a few blocks of each size class (16, 32, ...) for many size classes will never fill each bucket, and thus map a lot of memory pages while only using a small fraction of the mapped memory. However, the wasted memory will always be less than 4KiB (or the configured memory page size) per size class as each span is initialized one memory page at a time. The cache for free spans will be reused by all size classes.
|
||||
|
||||
Threads that perform a lot of allocations and deallocations in a pattern that have a large difference in high and low water marks, and that difference is larger than the thread cache size, will put a lot of contention on the global cache. What will happen is the thread cache will overflow on each low water mark causing pages to be released to the global cache, then underflow on high water mark causing pages to be re-acquired from the global cache. This can be mitigated by changing the __MAX_SPAN_CACHE_DIVISOR__ define in the source code (at the cost of higher average memory overhead).
|
||||
|
||||
# Caveats
|
||||
VirtualAlloc has an internal granularity of 64KiB. However, mmap lacks this granularity control, and the implementation instead oversizes the memory mapping with configured span size to be able to always return a memory area with the required alignment. Since the extra memory pages are never touched this will not result in extra committed physical memory pages, but rather only increase virtual memory address space.
|
||||
|
||||
All entry points assume the passed values are valid, for example passing an invalid pointer to free would most likely result in a segmentation fault. __The library does not try to guard against errors!__.
|
||||
|
||||
To support global scope data doing dynamic allocation/deallocation such as C++ objects with custom constructors and destructors, the call to __rpmalloc_finalize__ will not completely terminate the allocator but rather empty all caches and put the allocator in finalization mode. Once this call has been made, the allocator is no longer thread safe and expects all remaining calls to originate from global data destruction on main thread. Any spans or heaps becoming free during this phase will be immediately unmapped to allow correct teardown of the process or dynamic library without any leaks.
|
||||
|
||||
# Other languages
|
||||
|
||||
[Johan Andersson](https://github.com/repi) at Embark has created a Rust wrapper available at [rpmalloc-rs](https://github.com/EmbarkStudios/rpmalloc-rs)
|
||||
|
||||
[Stas Denisov](https://github.com/nxrighthere) has created a C# wrapper available at [Rpmalloc-CSharp](https://github.com/nxrighthere/Rpmalloc-CSharp)
|
||||
|
||||
# License
|
||||
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <http://unlicense.org>
|
||||
|
||||
|
||||
You can also use this software under the MIT license if public domain is
|
||||
not recognized in your country
|
||||
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017 Mattias Jansson
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
@@ -0,0 +1,512 @@
|
||||
/* malloc.c - Memory allocator - Public Domain - 2016 Mattias Jansson
|
||||
*
|
||||
* This library provides a cross-platform lock free thread caching malloc implementation in C11.
|
||||
* The latest source code is always available at
|
||||
*
|
||||
* https://github.com/mjansson/rpmalloc
|
||||
*
|
||||
* This library is put in the public domain; you can redistribute it and/or modify it without any restrictions.
|
||||
*
|
||||
*/
|
||||
|
||||
//
|
||||
// This file provides overrides for the standard library malloc entry points for C and new/delete operators for C++
|
||||
// It also provides automatic initialization/finalization of process and threads
|
||||
//
|
||||
#if defined(__TINYC__)
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
|
||||
#ifndef RPMALLOC_NO_PTHREAD_CREATE_OVERRIDE
|
||||
#define RPMALLOC_NO_PTHREAD_CREATE_OVERRIDE 0
|
||||
#endif
|
||||
|
||||
#ifndef ARCH_64BIT
|
||||
# if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
|
||||
# define ARCH_64BIT 1
|
||||
_Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
|
||||
_Static_assert(sizeof(void*) == 8, "Data type size mismatch");
|
||||
# else
|
||||
# define ARCH_64BIT 0
|
||||
_Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
|
||||
_Static_assert(sizeof(void*) == 4, "Data type size mismatch");
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if (defined(__GNUC__) || defined(__clang__))
|
||||
#pragma GCC visibility push(default)
|
||||
#endif
|
||||
|
||||
#define USE_IMPLEMENT 1
|
||||
#define USE_INTERPOSE 0
|
||||
#define USE_ALIAS 0
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#undef USE_INTERPOSE
|
||||
#define USE_INTERPOSE 1
|
||||
|
||||
typedef struct interpose_t {
|
||||
void* new_func;
|
||||
void* orig_func;
|
||||
} interpose_t;
|
||||
|
||||
#define MAC_INTERPOSE_PAIR(newf, oldf) { (void*)newf, (void*)oldf }
|
||||
#define MAC_INTERPOSE_SINGLE(newf, oldf) \
|
||||
__attribute__((used)) static const interpose_t macinterpose##newf##oldf \
|
||||
__attribute__ ((section("__DATA, __interpose"))) = MAC_INTERPOSE_PAIR(newf, oldf)
|
||||
|
||||
#endif
|
||||
|
||||
#if !defined(_WIN32) && !defined(__APPLE__)
|
||||
#undef USE_IMPLEMENT
|
||||
#undef USE_ALIAS
|
||||
#define USE_IMPLEMENT 0
|
||||
#define USE_ALIAS 1
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning (disable : 4100)
|
||||
#undef malloc
|
||||
#undef free
|
||||
#undef calloc
|
||||
#define RPMALLOC_RESTRICT __declspec(restrict)
|
||||
#else
|
||||
#define RPMALLOC_RESTRICT
|
||||
#endif
|
||||
|
||||
#if ENABLE_OVERRIDE
|
||||
|
||||
typedef struct rp_nothrow_t { int __dummy; } rp_nothrow_t;
|
||||
|
||||
#if USE_IMPLEMENT
|
||||
|
||||
extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL malloc(size_t size) { return rpmalloc(size); }
|
||||
extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL calloc(size_t count, size_t size) { return rpcalloc(count, size); }
|
||||
extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL realloc(void* ptr, size_t size) { return rprealloc(ptr, size); }
|
||||
extern inline void* RPMALLOC_CDECL reallocf(void* ptr, size_t size) { return rprealloc(ptr, size); }
|
||||
extern inline void* RPMALLOC_CDECL aligned_alloc(size_t alignment, size_t size) { return rpaligned_alloc(alignment, size); }
|
||||
extern inline void* RPMALLOC_CDECL memalign(size_t alignment, size_t size) { return rpmemalign(alignment, size); }
|
||||
extern inline int RPMALLOC_CDECL posix_memalign(void** memptr, size_t alignment, size_t size) { return rpposix_memalign(memptr, alignment, size); }
|
||||
extern inline void RPMALLOC_CDECL free(void* ptr) { rpfree(ptr); }
|
||||
extern inline void RPMALLOC_CDECL cfree(void* ptr) { rpfree(ptr); }
|
||||
extern inline size_t RPMALLOC_CDECL malloc_usable_size(void* ptr) { return rpmalloc_usable_size(ptr); }
|
||||
extern inline size_t RPMALLOC_CDECL malloc_size(void* ptr) { return rpmalloc_usable_size(ptr); }
|
||||
|
||||
#ifdef _WIN32
|
||||
extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL _malloc_base(size_t size) { return rpmalloc(size); }
|
||||
extern inline void RPMALLOC_CDECL _free_base(void* ptr) { rpfree(ptr); }
|
||||
extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL _calloc_base(size_t count, size_t size) { return rpcalloc(count, size); }
|
||||
extern inline size_t RPMALLOC_CDECL _msize(void* ptr) { return rpmalloc_usable_size(ptr); }
|
||||
extern inline size_t RPMALLOC_CDECL _msize_base(void* ptr) { return rpmalloc_usable_size(ptr); }
|
||||
extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL _realloc_base(void* ptr, size_t size) { return rprealloc(ptr, size); }
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
// For Windows, #include <rpnew.h> in one source file to get the C++ operator overrides implemented in your module
|
||||
#else
|
||||
// Overload the C++ operators using the mangled names (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
|
||||
// operators delete and delete[]
|
||||
#define RPDEFVIS __attribute__((visibility("default")))
|
||||
extern void _ZdlPv(void* p); void RPDEFVIS _ZdlPv(void* p) { rpfree(p); }
|
||||
extern void _ZdaPv(void* p); void RPDEFVIS _ZdaPv(void* p) { rpfree(p); }
|
||||
#if ARCH_64BIT
|
||||
// 64-bit operators new and new[], normal and aligned
|
||||
extern void* _Znwm(uint64_t size); void* RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
|
||||
extern void* _Znam(uint64_t size); void* RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
|
||||
extern void* _Znwmm(uint64_t size, uint64_t align); void* RPDEFVIS _Znwmm(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _Znamm(uint64_t size, uint64_t align); void* RPDEFVIS _Znamm(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnwmSt11align_val_t(uint64_t size, uint64_t align); void* RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnamSt11align_val_t(uint64_t size, uint64_t align); void* RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t); void* RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
|
||||
extern void* _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t); void* RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
|
||||
extern void* _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t); void* RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t); void* RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
|
||||
// 64-bit operators sized delete and delete[], normal and aligned
|
||||
extern void _ZdlPvm(void* p, uint64_t size); void RPDEFVIS _ZdlPvm(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
|
||||
extern void _ZdaPvm(void* p, uint64_t size); void RPDEFVIS _ZdaPvm(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
|
||||
extern void _ZdlPvSt11align_val_t(void* p, uint64_t align); void RPDEFVIS _ZdlPvSt11align_val_t(void* p, uint64_t align) { rpfree(p); (void)sizeof(align); }
|
||||
extern void _ZdaPvSt11align_val_t(void* p, uint64_t align); void RPDEFVIS _ZdaPvSt11align_val_t(void* p, uint64_t align) { rpfree(p); (void)sizeof(align); }
|
||||
extern void _ZdlPvmSt11align_val_t(void* p, uint64_t size, uint64_t align); void RPDEFVIS _ZdlPvmSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(align); }
|
||||
extern void _ZdaPvmSt11align_val_t(void* p, uint64_t size, uint64_t align); void RPDEFVIS _ZdaPvmSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(align); }
|
||||
#else
|
||||
// 32-bit operators new and new[], normal and aligned
|
||||
extern void* _Znwj(uint32_t size); void* RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
|
||||
extern void* _Znaj(uint32_t size); void* RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
|
||||
extern void* _Znwjj(uint32_t size, uint32_t align); void* RPDEFVIS _Znwjj(uint32_t size, uint32_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _Znajj(uint32_t size, uint32_t align); void* RPDEFVIS _Znajj(uint32_t size, uint32_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnwjSt11align_val_t(size_t size, size_t align); void* RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnajSt11align_val_t(size_t size, size_t align); void* RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) { return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t); void* RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
|
||||
extern void* _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t); void* RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
|
||||
extern void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t); void* RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
|
||||
extern void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t); void* RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
|
||||
// 32-bit operators sized delete and delete[], normal and aligned
|
||||
extern void _ZdlPvj(void* p, uint64_t size); void RPDEFVIS _ZdlPvj(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
|
||||
extern void _ZdaPvj(void* p, uint64_t size); void RPDEFVIS _ZdaPvj(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
|
||||
extern void _ZdlPvSt11align_val_t(void* p, uint32_t align); void RPDEFVIS _ZdlPvSt11align_val_t(void* p, uint64_t a) { rpfree(p); (void)sizeof(align); }
|
||||
extern void _ZdaPvSt11align_val_t(void* p, uint32_t align); void RPDEFVIS _ZdaPvSt11align_val_t(void* p, uint64_t a) { rpfree(p); (void)sizeof(align); }
|
||||
extern void _ZdlPvjSt11align_val_t(void* p, uint32_t size, uint32_t align); void RPDEFVIS _ZdlPvjSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(a); }
|
||||
extern void _ZdaPvjSt11align_val_t(void* p, uint32_t size, uint32_t align); void RPDEFVIS _ZdaPvjSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(a); }
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if USE_INTERPOSE || USE_ALIAS
|
||||
|
||||
static void* rpmalloc_nothrow(size_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
|
||||
static void* rpaligned_alloc_reverse(size_t size, size_t align) { return rpaligned_alloc(align, size); }
|
||||
static void* rpaligned_alloc_reverse_nothrow(size_t size, size_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
|
||||
static void rpfree_size(void* p, size_t size) { (void)sizeof(size); rpfree(p); }
|
||||
static void rpfree_aligned(void* p, size_t align) { (void)sizeof(align); rpfree(p); }
|
||||
static void rpfree_size_aligned(void* p, size_t size, size_t align) { (void)sizeof(size); (void)sizeof(align); rpfree(p); }
|
||||
|
||||
#endif
|
||||
|
||||
#if USE_INTERPOSE
|
||||
|
||||
__attribute__((used)) static const interpose_t macinterpose_malloc[]
|
||||
__attribute__ ((section("__DATA, __interpose"))) = {
|
||||
//new and new[]
|
||||
MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
|
||||
MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
|
||||
MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
|
||||
MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow, _ZnwmSt11align_val_tRKSt9nothrow_t),
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow, _ZnamSt11align_val_tRKSt9nothrow_t),
|
||||
//delete and delete[]
|
||||
MAC_INTERPOSE_PAIR(rpfree, _ZdlPv),
|
||||
MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
|
||||
MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
|
||||
MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
|
||||
MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
|
||||
MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
|
||||
MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
|
||||
MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
|
||||
//libc entry points
|
||||
MAC_INTERPOSE_PAIR(rpmalloc, malloc),
|
||||
MAC_INTERPOSE_PAIR(rpmalloc, calloc),
|
||||
MAC_INTERPOSE_PAIR(rprealloc, realloc),
|
||||
MAC_INTERPOSE_PAIR(rprealloc, reallocf),
|
||||
#if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
|
||||
MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
|
||||
#endif
|
||||
MAC_INTERPOSE_PAIR(rpmemalign, memalign),
|
||||
MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
|
||||
MAC_INTERPOSE_PAIR(rpfree, free),
|
||||
MAC_INTERPOSE_PAIR(rpfree, cfree),
|
||||
MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
|
||||
MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
#if USE_ALIAS
|
||||
|
||||
#define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
|
||||
|
||||
// Alias the C++ operators using the mangled names (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
|
||||
|
||||
// operators delete and delete[]
|
||||
void _ZdlPv(void* p) RPALIAS(rpfree)
|
||||
void _ZdaPv(void* p) RPALIAS(rpfree)
|
||||
|
||||
#if ARCH_64BIT
|
||||
// 64-bit operators new and new[], normal and aligned
|
||||
void* _Znwm(uint64_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc)
|
||||
void* _Znam(uint64_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc)
|
||||
void* _Znwmm(uint64_t size, uint64_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _Znamm(uint64_t size, uint64_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _ZnwmSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _ZnamSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _ZnwmRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
|
||||
void* _ZnamRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
|
||||
void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
|
||||
void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
|
||||
// 64-bit operators delete and delete[], sized and aligned
|
||||
void _ZdlPvm(void* p, size_t n) RPALIAS(rpfree_size)
|
||||
void _ZdaPvm(void* p, size_t n) RPALIAS(rpfree_size)
|
||||
void _ZdlPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
|
||||
void _ZdaPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
|
||||
void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
|
||||
void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
|
||||
#else
|
||||
// 32-bit operators new and new[], normal and aligned
|
||||
void* _Znwj(uint32_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc)
|
||||
void* _Znaj(uint32_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc)
|
||||
void* _Znwjj(uint32_t size, uint32_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _Znajj(uint32_t size, uint32_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _ZnwjSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _ZnajSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
|
||||
void* _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
|
||||
void* _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
|
||||
void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
|
||||
void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
|
||||
// 32-bit operators delete and delete[], sized and aligned
|
||||
void _ZdlPvj(void* p, size_t n) RPALIAS(rpfree_size)
|
||||
void _ZdaPvj(void* p, size_t n) RPALIAS(rpfree_size)
|
||||
void _ZdlPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
|
||||
void _ZdaPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
|
||||
void _ZdlPvjSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
|
||||
void _ZdaPvjSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
|
||||
#endif
|
||||
|
||||
void* malloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc)
|
||||
void* calloc(size_t count, size_t size) RPALIAS(rpcalloc)
|
||||
void* realloc(void* ptr, size_t size) RPALIAS(rprealloc)
|
||||
void* reallocf(void* ptr, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc)
|
||||
void* aligned_alloc(size_t alignment, size_t size) RPALIAS(rpaligned_alloc)
|
||||
void* memalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rpmemalign)
|
||||
int posix_memalign(void** memptr, size_t alignment, size_t size) RPALIAS(rpposix_memalign)
|
||||
void free(void* ptr) RPALIAS(rpfree)
|
||||
void cfree(void* ptr) RPALIAS(rpfree)
|
||||
#if defined(__ANDROID__) || defined(__FreeBSD__)
|
||||
size_t malloc_usable_size(const void* ptr) RPALIAS(rpmalloc_usable_size)
|
||||
#else
|
||||
size_t malloc_usable_size(void* ptr) RPALIAS(rpmalloc_usable_size)
|
||||
#endif
|
||||
size_t malloc_size(void* ptr) RPALIAS(rpmalloc_usable_size)
|
||||
|
||||
#endif
|
||||
|
||||
static inline size_t
|
||||
_rpmalloc_page_size(void) {
|
||||
return _memory_page_size;
|
||||
}
|
||||
|
||||
extern void* RPMALLOC_CDECL
|
||||
reallocarray(void* ptr, size_t count, size_t size);
|
||||
|
||||
extern void* RPMALLOC_CDECL
|
||||
reallocarray(void* ptr, size_t count, size_t size) {
|
||||
size_t total;
|
||||
#if ENABLE_VALIDATE_ARGS
|
||||
#ifdef _MSC_VER
|
||||
int err = SizeTMult(count, size, &total);
|
||||
if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
|
||||
errno = EINVAL;
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
int err = __builtin_umull_overflow(count, size, &total);
|
||||
if (err || (total >= MAX_ALLOC_SIZE)) {
|
||||
errno = EINVAL;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
total = count * size;
|
||||
#endif
|
||||
return realloc(ptr, total);
|
||||
}
|
||||
|
||||
extern inline void* RPMALLOC_CDECL
|
||||
valloc(size_t size) {
|
||||
get_thread_heap();
|
||||
return rpaligned_alloc(_rpmalloc_page_size(), size);
|
||||
}
|
||||
|
||||
extern inline void* RPMALLOC_CDECL
|
||||
pvalloc(size_t size) {
|
||||
get_thread_heap();
|
||||
const size_t page_size = _rpmalloc_page_size();
|
||||
const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
|
||||
#if ENABLE_VALIDATE_ARGS
|
||||
if (aligned_size < size) {
|
||||
errno = EINVAL;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
|
||||
}
|
||||
|
||||
#endif // ENABLE_OVERRIDE
|
||||
|
||||
#if ENABLE_PRELOAD
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
#if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
|
||||
|
||||
extern __declspec(dllexport) BOOL WINAPI
|
||||
DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved);
|
||||
|
||||
extern __declspec(dllexport) BOOL WINAPI
|
||||
DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved) {
|
||||
(void)sizeof(reserved);
|
||||
(void)sizeof(instance);
|
||||
if (reason == DLL_PROCESS_ATTACH)
|
||||
rpmalloc_initialize();
|
||||
else if (reason == DLL_PROCESS_DETACH)
|
||||
rpmalloc_finalize();
|
||||
else if (reason == DLL_THREAD_ATTACH)
|
||||
rpmalloc_thread_initialize();
|
||||
else if (reason == DLL_THREAD_DETACH)
|
||||
rpmalloc_thread_finalize(1);
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
//end BUILD_DYNAMIC_LINK
|
||||
#else
|
||||
|
||||
extern void
|
||||
_global_rpmalloc_init(void) {
|
||||
rpmalloc_set_main_thread();
|
||||
rpmalloc_initialize();
|
||||
}
|
||||
|
||||
#if defined(__clang__) || defined(__GNUC__)
|
||||
|
||||
static void __attribute__((constructor))
|
||||
initializer(void) {
|
||||
_global_rpmalloc_init();
|
||||
}
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
static int
|
||||
_global_rpmalloc_xib(void) {
|
||||
_global_rpmalloc_init();
|
||||
return 0;
|
||||
}
|
||||
|
||||
#pragma section(".CRT$XIB",read)
|
||||
__declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) = _global_rpmalloc_xib;
|
||||
#pragma comment(linker, "/include:_rpmalloc_module_init")
|
||||
|
||||
#endif
|
||||
|
||||
//end !BUILD_DYNAMIC_LINK
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
#include <pthread.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
|
||||
extern void
|
||||
rpmalloc_set_main_thread(void);
|
||||
|
||||
static pthread_key_t destructor_key;
|
||||
|
||||
static void
|
||||
thread_destructor(void*);
|
||||
|
||||
static void __attribute__((constructor))
|
||||
initializer(void) {
|
||||
rpmalloc_set_main_thread();
|
||||
rpmalloc_initialize();
|
||||
pthread_key_create(&destructor_key, thread_destructor);
|
||||
}
|
||||
|
||||
static void __attribute__((destructor))
|
||||
finalizer(void) {
|
||||
rpmalloc_finalize();
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
void* (*real_start)(void*);
|
||||
void* real_arg;
|
||||
} thread_starter_arg;
|
||||
|
||||
static void*
|
||||
thread_starter(void* argptr) {
|
||||
thread_starter_arg* arg = argptr;
|
||||
void* (*real_start)(void*) = arg->real_start;
|
||||
void* real_arg = arg->real_arg;
|
||||
rpmalloc_thread_initialize();
|
||||
rpfree(argptr);
|
||||
pthread_setspecific(destructor_key, (void*)1);
|
||||
return (*real_start)(real_arg);
|
||||
}
|
||||
|
||||
static void
|
||||
thread_destructor(void* value) {
|
||||
(void)sizeof(value);
|
||||
rpmalloc_thread_finalize(1);
|
||||
}
|
||||
|
||||
#if RPMALLOC_NO_PTHREAD_CREATE_OVERRIDE
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
|
||||
static int
|
||||
pthread_create_proxy(pthread_t* thread,
|
||||
const pthread_attr_t* attr,
|
||||
void* (*start_routine)(void*),
|
||||
void* arg) {
|
||||
rpmalloc_initialize();
|
||||
thread_starter_arg* starter_arg = rpmalloc(sizeof(thread_starter_arg));
|
||||
starter_arg->real_start = start_routine;
|
||||
starter_arg->real_arg = arg;
|
||||
return pthread_create(thread, attr, thread_starter, starter_arg);
|
||||
}
|
||||
|
||||
MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
|
||||
|
||||
#else
|
||||
|
||||
#include <dlfcn.h>
|
||||
|
||||
int
|
||||
pthread_create(pthread_t* thread,
|
||||
const pthread_attr_t* attr,
|
||||
void* (*start_routine)(void*),
|
||||
void* arg) {
|
||||
#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \
|
||||
defined(__APPLE__) || defined(__HAIKU__)
|
||||
char fname[] = "pthread_create";
|
||||
#else
|
||||
char fname[] = "_pthread_create";
|
||||
#endif
|
||||
void* real_pthread_create = dlsym(RTLD_NEXT, fname);
|
||||
rpmalloc_thread_initialize();
|
||||
thread_starter_arg* starter_arg = rpmalloc(sizeof(thread_starter_arg));
|
||||
starter_arg->real_start = start_routine;
|
||||
starter_arg->real_arg = arg;
|
||||
return (*(int (*)(pthread_t*, const pthread_attr_t*, void* (*)(void*), void*))real_pthread_create)(thread, attr, thread_starter, starter_arg);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#if ENABLE_OVERRIDE
|
||||
|
||||
#if defined(__GLIBC__) && defined(__linux__)
|
||||
|
||||
void* __libc_malloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc)
|
||||
void* __libc_calloc(size_t count, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2) RPALIAS(rpcalloc)
|
||||
void* __libc_realloc(void* p, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc)
|
||||
void __libc_free(void* p) RPALIAS(rpfree)
|
||||
void __libc_cfree(void* p) RPALIAS(rpfree)
|
||||
void* __libc_memalign(size_t align, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rpmemalign)
|
||||
int __posix_memalign(void** p, size_t align, size_t size) RPALIAS(rpposix_memalign)
|
||||
|
||||
extern void* __libc_valloc(size_t size);
|
||||
extern void* __libc_pvalloc(size_t size);
|
||||
|
||||
void*
|
||||
__libc_valloc(size_t size) {
|
||||
return valloc(size);
|
||||
}
|
||||
|
||||
void*
|
||||
__libc_pvalloc(size_t size) {
|
||||
return pvalloc(size);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#if (defined(__GNUC__) || defined(__clang__))
|
||||
#pragma GCC visibility pop
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,373 @@
|
||||
/* rpmalloc.h - Memory allocator - Public Domain - 2016 Mattias Jansson
|
||||
*
|
||||
* This library provides a cross-platform lock free thread caching malloc implementation in C11.
|
||||
* The latest source code is always available at
|
||||
*
|
||||
* https://github.com/mjansson/rpmalloc
|
||||
*
|
||||
* This library is put in the public domain; you can redistribute it and/or modify it without any restrictions.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) || defined(__GNUC__)
|
||||
# define RPMALLOC_EXPORT __attribute__((visibility("default")))
|
||||
# define RPMALLOC_ALLOCATOR
|
||||
# if (defined(__clang_major__) && (__clang_major__ < 4)) || (defined(__GNUC__) && defined(ENABLE_PRELOAD) && ENABLE_PRELOAD)
|
||||
# define RPMALLOC_ATTRIB_MALLOC
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)
|
||||
# else
|
||||
# define RPMALLOC_ATTRIB_MALLOC __attribute__((__malloc__))
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE(size) __attribute__((alloc_size(size)))
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) __attribute__((alloc_size(count, size)))
|
||||
# endif
|
||||
# define RPMALLOC_CDECL
|
||||
#elif defined(_MSC_VER)
|
||||
# define RPMALLOC_EXPORT
|
||||
# define RPMALLOC_ALLOCATOR __declspec(allocator) __declspec(restrict)
|
||||
# define RPMALLOC_ATTRIB_MALLOC
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size)
|
||||
# define RPMALLOC_CDECL __cdecl
|
||||
#else
|
||||
# define RPMALLOC_EXPORT
|
||||
# define RPMALLOC_ALLOCATOR
|
||||
# define RPMALLOC_ATTRIB_MALLOC
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
|
||||
# define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size)
|
||||
# define RPMALLOC_CDECL
|
||||
#endif
|
||||
|
||||
//! Define RPMALLOC_CONFIGURABLE to enable configuring sizes. Will introduce
|
||||
// a very small overhead due to some size calculations not being compile time constants
|
||||
#ifndef RPMALLOC_CONFIGURABLE
|
||||
#define RPMALLOC_CONFIGURABLE 0
|
||||
#endif
|
||||
|
||||
//! Define RPMALLOC_FIRST_CLASS_HEAPS to enable heap based API (rpmalloc_heap_* functions).
|
||||
// Will introduce a very small overhead to track fully allocated spans in heaps
|
||||
#ifndef RPMALLOC_FIRST_CLASS_HEAPS
|
||||
#define RPMALLOC_FIRST_CLASS_HEAPS 0
|
||||
#endif
|
||||
|
||||
//! Flag to rpaligned_realloc to not preserve content in reallocation
|
||||
#define RPMALLOC_NO_PRESERVE 1
|
||||
//! Flag to rpaligned_realloc to fail and return null pointer if grow cannot be done in-place,
|
||||
// in which case the original pointer is still valid (just like a call to realloc which failes to allocate
|
||||
// a new block).
|
||||
#define RPMALLOC_GROW_OR_FAIL 2
|
||||
|
||||
typedef struct rpmalloc_global_statistics_t {
|
||||
//! Current amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_STATISTICS=1)
|
||||
size_t mapped;
|
||||
//! Peak amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_STATISTICS=1)
|
||||
size_t mapped_peak;
|
||||
//! Current amount of memory in global caches for small and medium sizes (<32KiB)
|
||||
size_t cached;
|
||||
//! Current amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1)
|
||||
size_t huge_alloc;
|
||||
//! Peak amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1)
|
||||
size_t huge_alloc_peak;
|
||||
//! Total amount of memory mapped since initialization (only if ENABLE_STATISTICS=1)
|
||||
size_t mapped_total;
|
||||
//! Total amount of memory unmapped since initialization (only if ENABLE_STATISTICS=1)
|
||||
size_t unmapped_total;
|
||||
} rpmalloc_global_statistics_t;
|
||||
|
||||
typedef struct rpmalloc_thread_statistics_t {
|
||||
//! Current number of bytes available in thread size class caches for small and medium sizes (<32KiB)
|
||||
size_t sizecache;
|
||||
//! Current number of bytes available in thread span caches for small and medium sizes (<32KiB)
|
||||
size_t spancache;
|
||||
//! Total number of bytes transitioned from thread cache to global cache (only if ENABLE_STATISTICS=1)
|
||||
size_t thread_to_global;
|
||||
//! Total number of bytes transitioned from global cache to thread cache (only if ENABLE_STATISTICS=1)
|
||||
size_t global_to_thread;
|
||||
//! Per span count statistics (only if ENABLE_STATISTICS=1)
|
||||
struct {
|
||||
//! Currently used number of spans
|
||||
size_t current;
|
||||
//! High water mark of spans used
|
||||
size_t peak;
|
||||
//! Number of spans transitioned to global cache
|
||||
size_t to_global;
|
||||
//! Number of spans transitioned from global cache
|
||||
size_t from_global;
|
||||
//! Number of spans transitioned to thread cache
|
||||
size_t to_cache;
|
||||
//! Number of spans transitioned from thread cache
|
||||
size_t from_cache;
|
||||
//! Number of spans transitioned to reserved state
|
||||
size_t to_reserved;
|
||||
//! Number of spans transitioned from reserved state
|
||||
size_t from_reserved;
|
||||
//! Number of raw memory map calls (not hitting the reserve spans but resulting in actual OS mmap calls)
|
||||
size_t map_calls;
|
||||
} span_use[64];
|
||||
//! Per size class statistics (only if ENABLE_STATISTICS=1)
|
||||
struct {
|
||||
//! Current number of allocations
|
||||
size_t alloc_current;
|
||||
//! Peak number of allocations
|
||||
size_t alloc_peak;
|
||||
//! Total number of allocations
|
||||
size_t alloc_total;
|
||||
//! Total number of frees
|
||||
size_t free_total;
|
||||
//! Number of spans transitioned to cache
|
||||
size_t spans_to_cache;
|
||||
//! Number of spans transitioned from cache
|
||||
size_t spans_from_cache;
|
||||
//! Number of spans transitioned from reserved state
|
||||
size_t spans_from_reserved;
|
||||
//! Number of raw memory map calls (not hitting the reserve spans but resulting in actual OS mmap calls)
|
||||
size_t map_calls;
|
||||
} size_use[128];
|
||||
} rpmalloc_thread_statistics_t;
|
||||
|
||||
typedef struct rpmalloc_config_t {
|
||||
//! Map memory pages for the given number of bytes. The returned address MUST be
|
||||
// aligned to the rpmalloc span size, which will always be a power of two.
|
||||
// Optionally the function can store an alignment offset in the offset variable
|
||||
// in case it performs alignment and the returned pointer is offset from the
|
||||
// actual start of the memory region due to this alignment. The alignment offset
|
||||
// will be passed to the memory unmap function. The alignment offset MUST NOT be
|
||||
// larger than 65535 (storable in an uint16_t), if it is you must use natural
|
||||
// alignment to shift it into 16 bits. If you set a memory_map function, you
|
||||
// must also set a memory_unmap function or else the default implementation will
|
||||
// be used for both. This function must be thread safe, it can be called by
|
||||
// multiple threads simultaneously.
|
||||
void* (*memory_map)(size_t size, size_t* offset);
|
||||
//! Unmap the memory pages starting at address and spanning the given number of bytes.
|
||||
// If release is set to non-zero, the unmap is for an entire span range as returned by
|
||||
// a previous call to memory_map and that the entire range should be released. The
|
||||
// release argument holds the size of the entire span range. If release is set to 0,
|
||||
// the unmap is a partial decommit of a subset of the mapped memory range.
|
||||
// If you set a memory_unmap function, you must also set a memory_map function or
|
||||
// else the default implementation will be used for both. This function must be thread
|
||||
// safe, it can be called by multiple threads simultaneously.
|
||||
void (*memory_unmap)(void* address, size_t size, size_t offset, size_t release);
|
||||
//! Called when an assert fails, if asserts are enabled. Will use the standard assert()
|
||||
// if this is not set.
|
||||
void (*error_callback)(const char* message);
|
||||
//! Called when a call to map memory pages fails (out of memory). If this callback is
|
||||
// not set or returns zero the library will return a null pointer in the allocation
|
||||
// call. If this callback returns non-zero the map call will be retried. The argument
|
||||
// passed is the number of bytes that was requested in the map call. Only used if
|
||||
// the default system memory map function is used (memory_map callback is not set).
|
||||
int (*map_fail_callback)(size_t size);
|
||||
//! Size of memory pages. The page size MUST be a power of two. All memory mapping
|
||||
// requests to memory_map will be made with size set to a multiple of the page size.
|
||||
// Used if RPMALLOC_CONFIGURABLE is defined to 1, otherwise system page size is used.
|
||||
size_t page_size;
|
||||
//! Size of a span of memory blocks. MUST be a power of two, and in [4096,262144]
|
||||
// range (unless 0 - set to 0 to use the default span size). Used if RPMALLOC_CONFIGURABLE
|
||||
// is defined to 1.
|
||||
size_t span_size;
|
||||
//! Number of spans to map at each request to map new virtual memory blocks. This can
|
||||
// be used to minimize the system call overhead at the cost of virtual memory address
|
||||
// space. The extra mapped pages will not be written until actually used, so physical
|
||||
// committed memory should not be affected in the default implementation. Will be
|
||||
// aligned to a multiple of spans that match memory page size in case of huge pages.
|
||||
size_t span_map_count;
|
||||
//! Enable use of large/huge pages. If this flag is set to non-zero and page size is
|
||||
// zero, the allocator will try to enable huge pages and auto detect the configuration.
|
||||
// If this is set to non-zero and page_size is also non-zero, the allocator will
|
||||
// assume huge pages have been configured and enabled prior to initializing the
|
||||
// allocator.
|
||||
// For Windows, see https://docs.microsoft.com/en-us/windows/desktop/memory/large-page-support
|
||||
// For Linux, see https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
|
||||
int enable_huge_pages;
|
||||
//! Respectively allocated pages and huge allocated pages names for systems
|
||||
// supporting it to be able to distinguish among anonymous regions.
|
||||
const char *page_name;
|
||||
const char *huge_page_name;
|
||||
} rpmalloc_config_t;
|
||||
|
||||
//! Initialize allocator with default configuration
|
||||
RPMALLOC_EXPORT int
|
||||
rpmalloc_initialize(void);
|
||||
|
||||
//! Initialize allocator with given configuration
|
||||
RPMALLOC_EXPORT int
|
||||
rpmalloc_initialize_config(const rpmalloc_config_t* config);
|
||||
|
||||
//! Get allocator configuration
|
||||
RPMALLOC_EXPORT const rpmalloc_config_t*
|
||||
rpmalloc_config(void);
|
||||
|
||||
//! Finalize allocator
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_finalize(void);
|
||||
|
||||
//! Initialize allocator for calling thread
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_thread_initialize(void);
|
||||
|
||||
//! Finalize allocator for calling thread
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_thread_finalize(int release_caches);
|
||||
|
||||
//! Perform deferred deallocations pending for the calling thread heap
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_thread_collect(void);
|
||||
|
||||
//! Query if allocator is initialized for calling thread
|
||||
RPMALLOC_EXPORT int
|
||||
rpmalloc_is_thread_initialized(void);
|
||||
|
||||
//! Get per-thread statistics
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_thread_statistics(rpmalloc_thread_statistics_t* stats);
|
||||
|
||||
//! Get global statistics
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_global_statistics(rpmalloc_global_statistics_t* stats);
|
||||
|
||||
//! Dump all statistics in human readable format to file (should be a FILE*)
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_dump_statistics(void* file);
|
||||
|
||||
//! Allocate a memory block of at least the given size
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1);
|
||||
|
||||
//! Free the given memory block
|
||||
RPMALLOC_EXPORT void
|
||||
rpfree(void* ptr);
|
||||
|
||||
//! Allocate a memory block of at least the given size and zero initialize it
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpcalloc(size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2);
|
||||
|
||||
//! Reallocate the given block to at least the given size
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rprealloc(void* ptr, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
|
||||
|
||||
//! Reallocate the given block to at least the given size and alignment,
|
||||
// with optional control flags (see RPMALLOC_NO_PRESERVE).
|
||||
// Alignment must be a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB)
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpaligned_realloc(void* ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3);
|
||||
|
||||
//! Allocate a memory block of at least the given size and alignment.
|
||||
// Alignment must be a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB)
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
|
||||
|
||||
//! Allocate a memory block of at least the given size and alignment, and zero initialize it.
|
||||
// Alignment must be a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB)
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpaligned_calloc(size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
|
||||
|
||||
//! Allocate a memory block of at least the given size and alignment.
|
||||
// Alignment must be a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB)
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
|
||||
|
||||
//! Allocate a memory block of at least the given size and alignment.
|
||||
// Alignment must be a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB)
|
||||
RPMALLOC_EXPORT int
|
||||
rpposix_memalign(void** memptr, size_t alignment, size_t size);
|
||||
|
||||
//! Query the usable size of the given memory block (from given pointer to the end of block)
|
||||
RPMALLOC_EXPORT size_t
|
||||
rpmalloc_usable_size(void* ptr);
|
||||
|
||||
//! Dummy empty function for forcing linker symbol inclusion
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_linker_reference(void);
|
||||
|
||||
#if RPMALLOC_FIRST_CLASS_HEAPS
|
||||
|
||||
//! Heap type
|
||||
typedef struct heap_t rpmalloc_heap_t;
|
||||
|
||||
//! Acquire a new heap. Will reuse existing released heaps or allocate memory for a new heap
|
||||
// if none available. Heap API is implemented with the strict assumption that only one single
|
||||
// thread will call heap functions for a given heap at any given time, no functions are thread safe.
|
||||
RPMALLOC_EXPORT rpmalloc_heap_t*
|
||||
rpmalloc_heap_acquire(void);
|
||||
|
||||
//! Release a heap (does NOT free the memory allocated by the heap, use rpmalloc_heap_free_all before destroying the heap).
|
||||
// Releasing a heap will enable it to be reused by other threads. Safe to pass a null pointer.
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_heap_release(rpmalloc_heap_t* heap);
|
||||
|
||||
//! Allocate a memory block of at least the given size using the given heap.
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc_heap_alloc(rpmalloc_heap_t* heap, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
|
||||
|
||||
//! Allocate a memory block of at least the given size using the given heap. The returned
|
||||
// block will have the requested alignment. Alignment must be a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB).
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc_heap_aligned_alloc(rpmalloc_heap_t* heap, size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3);
|
||||
|
||||
//! Allocate a memory block of at least the given size using the given heap and zero initialize it.
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc_heap_calloc(rpmalloc_heap_t* heap, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
|
||||
|
||||
//! Allocate a memory block of at least the given size using the given heap and zero initialize it. The returned
|
||||
// block will have the requested alignment. Alignment must either be zero, or a power of two and a multiple of sizeof(void*),
|
||||
// and should ideally be less than memory page size. A caveat of rpmalloc
|
||||
// internals is that this must also be strictly less than the span size (default 64KiB).
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc_heap_aligned_calloc(rpmalloc_heap_t* heap, size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
|
||||
|
||||
//! Reallocate the given block to at least the given size. The memory block MUST be allocated
|
||||
// by the same heap given to this function.
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc_heap_realloc(rpmalloc_heap_t* heap, void* ptr, size_t size, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3);
|
||||
|
||||
//! Reallocate the given block to at least the given size. The memory block MUST be allocated
|
||||
// by the same heap given to this function. The returned block will have the requested alignment.
|
||||
// Alignment must be either zero, or a power of two and a multiple of sizeof(void*), and should ideally be
|
||||
// less than memory page size. A caveat of rpmalloc internals is that this must also be strictly less than
|
||||
// the span size (default 64KiB).
|
||||
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void*
|
||||
rpmalloc_heap_aligned_realloc(rpmalloc_heap_t* heap, void* ptr, size_t alignment, size_t size, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(4);
|
||||
|
||||
//! Free the given memory block from the given heap. The memory block MUST be allocated
|
||||
// by the same heap given to this function.
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_heap_free(rpmalloc_heap_t* heap, void* ptr);
|
||||
|
||||
//! Free all memory allocated by the heap
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_heap_free_all(rpmalloc_heap_t* heap);
|
||||
|
||||
//! Set the given heap as the current heap for the calling thread. A heap MUST only be current heap
|
||||
// for a single thread, a heap can never be shared between multiple threads. The previous
|
||||
// current heap for the calling thread is released to be reused by other threads.
|
||||
RPMALLOC_EXPORT void
|
||||
rpmalloc_heap_thread_set_current(rpmalloc_heap_t* heap);
|
||||
|
||||
//! Returns which heap the given pointer is allocated on
|
||||
RPMALLOC_EXPORT rpmalloc_heap_t*
|
||||
rpmalloc_get_heap_for_ptr(void* ptr);
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,111 @@
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
#include <new>
|
||||
#include <rpmalloc.h>
|
||||
|
||||
#ifndef __CRTDECL
|
||||
#define __CRTDECL
|
||||
#endif
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete(void* p) noexcept {
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete[](void* p) noexcept {
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new(std::size_t size) noexcept(false) {
|
||||
return rpmalloc(size);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new[](std::size_t size) noexcept(false) {
|
||||
return rpmalloc(size);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new(std::size_t size, const std::nothrow_t& tag) noexcept {
|
||||
(void)sizeof(tag);
|
||||
return rpmalloc(size);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new[](std::size_t size, const std::nothrow_t& tag) noexcept {
|
||||
(void)sizeof(tag);
|
||||
return rpmalloc(size);
|
||||
}
|
||||
|
||||
#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete(void* p, std::size_t size) noexcept {
|
||||
(void)sizeof(size);
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete[](void* p, std::size_t size) noexcept {
|
||||
(void)sizeof(size);
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete(void* p, std::align_val_t align) noexcept {
|
||||
(void)sizeof(align);
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete[](void* p, std::align_val_t align) noexcept {
|
||||
(void)sizeof(align);
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete(void* p, std::size_t size, std::align_val_t align) noexcept {
|
||||
(void)sizeof(size);
|
||||
(void)sizeof(align);
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void __CRTDECL
|
||||
operator delete[](void* p, std::size_t size, std::align_val_t align) noexcept {
|
||||
(void)sizeof(size);
|
||||
(void)sizeof(align);
|
||||
rpfree(p);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new(std::size_t size, std::align_val_t align) noexcept(false) {
|
||||
return rpaligned_alloc(static_cast<size_t>(align), size);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new[](std::size_t size, std::align_val_t align) noexcept(false) {
|
||||
return rpaligned_alloc(static_cast<size_t>(align), size);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new(std::size_t size, std::align_val_t align, const std::nothrow_t& tag) noexcept {
|
||||
(void)sizeof(tag);
|
||||
return rpaligned_alloc(static_cast<size_t>(align), size);
|
||||
}
|
||||
|
||||
extern void* __CRTDECL
|
||||
operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t& tag) noexcept {
|
||||
(void)sizeof(tag);
|
||||
return rpaligned_alloc(static_cast<size_t>(align), size);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user