diff --git a/include/realtime_memory/free_list_resource.h b/include/realtime_memory/free_list_resource.h index 0214537..3d9fff0 100644 --- a/include/realtime_memory/free_list_resource.h +++ b/include/realtime_memory/free_list_resource.h @@ -2,8 +2,9 @@ // Copyright (c) 2019-2022 CradleApps, LLC - All Rights Reserved //============================================================================== /** A std::pmr compatible memory_resource that can allocate and - * free blocks of any size from a fixed buffer. If the buffer - * is exhausted, std::bad_alloc() is thrown. + * free blocks of any size. It can be supplied from an upstream + * resource, a fixed buffer, or a series of fixed buffers pushed + * in by an external routine (to allow async allocation techniques). * * Important notes: * - This resource is single-threaded @@ -30,14 +31,42 @@ namespace cradle::pmr class free_list_resource : public cradle::pmr::identity_equal_resource { public: - free_list_resource (void* buffer, std::size_t buffer_size) - : space (buffer_size) + /** Allocate from a buffer. Allocations throw when this runs out. */ + free_list_resource (std::byte* buffer, std::size_t buffer_size) { - if (buffer == nullptr || buffer_size < sizeof(mem_block)) + expand (buffer, buffer_size); + } + + /** Allocate from an upstream memory resource. + * + * The min_realloc_size describes the minimum size of the memory + * chunk requested from the upstream when the current one runs out. + */ + free_list_resource (memory_resource& upstream_resource, + std::size_t initial_size, + std::size_t min_chunk_size = std::size_t (64 * 1024)) + : upstream (&upstream_resource), + min_realloc_size (min_chunk_size) + { + expand (upstream->allocate (initial_size, 1), initial_size); + } + + /** Add more memory that can be used for allocations. + * If you supplied a capable upstream_resource constructor, you don't + * need to use this and more memory will be requested from the upstream + * resource if needed. + * + * However, if you are working in a domain where you can allocate/replenish + * memory at certain times and not others, this allows you to do that. + */ + void expand (void* new_memory, std::size_t new_memory_size) + { + if (new_memory == nullptr || new_memory_size < sizeof(mem_block)) throw std::bad_alloc(); - auto pos = align_block (buffer, space); - first_free = ::new (pos) mem_block (nullptr, space - mem_block::header_size()); + auto pos = align_block (new_memory, new_memory_size); + space += new_memory_size; + first_free = ::new (pos) mem_block (first_free, new_memory_size - mem_block::header_size()); } /** Returns the remaining space in the buffer, in bytes. */ @@ -59,9 +88,10 @@ class free_list_resource : public cradle::pmr::identity_equal_resource throw std::bad_alloc(); // Over-alignment is not currently supported! const auto size = cradle::pmr::detail::aligned_ceil (bytes, std::min (align, max_align_bytes)); + const auto next_chunk_size = std::max (min_realloc_size, size + mem_block::header_size()); if ((size + mem_block::header_size()) >= space) - throw std::bad_alloc(); // out of memory. + expand (upstream->allocate (next_chunk_size, max_align_bytes), next_chunk_size); for (mem_block* free = first_free, *prev = nullptr; free != nullptr; prev = free, free = free->next) if (free->size >= size) @@ -71,7 +101,9 @@ class free_list_resource : public cradle::pmr::identity_equal_resource if (auto defragmented_block = defragment (size)) return defragmented_block; - throw std::bad_alloc(); // still too fragmented, abort + // Still too fragmented: we'll have to allocate another chunk. + expand (upstream->allocate (next_chunk_size, max_align_bytes), next_chunk_size); + return do_allocate (bytes, align); } void do_deallocate (void* ptr, std::size_t, std::size_t) override @@ -134,37 +166,38 @@ class free_list_resource : public cradle::pmr::identity_equal_resource mem_block* closest_sized_block = nullptr; mem_block* closest_sized_block_prev = nullptr; - for (mem_block* free = first_free, *prev = nullptr; free != nullptr && free->next != nullptr;) + for (mem_block* free = first_free, *prev = nullptr; free != nullptr;) { - auto next = free->next; - const auto thisStart = free->data(); - const auto thisEnd = thisStart + free->size; - const auto nextHeader = reinterpret_cast (next); - - if (nextHeader - thisEnd <= std::ptrdiff_t (max_align_bytes)) + // merging next block along if it's free + while (free->next) { - const auto nextEnd = next->data() + next->size; + auto next = free->next; + auto thisEnd = free->data() + free->size; + auto nextHeader = reinterpret_cast (next); + + if (nextHeader - thisEnd > std::ptrdiff_t (max_align_bytes)) + break; - free->size = std::size_t (nextEnd - thisStart); + auto nextEnd = next->data() + next->size; + free->size = std::size_t (nextEnd - free->data()); free->next = next->next; } - else + + // look for a block that is suitable for use by the caller + if (free->size >= desired_block_size) { - if (free->size >= desired_block_size) + const auto diff = free->size - desired_block_size; + + if (diff < closest_size_diff) { - const auto diff = free->size - desired_block_size; - - if (diff < closest_size_diff) - { - closest_size_diff = diff; - closest_sized_block = free; - closest_sized_block_prev = prev; - } + closest_size_diff = diff; + closest_sized_block = free; + closest_sized_block_prev = prev; } - - prev = free; - free = free->next; } + + prev = free; + free = free->next; } if (closest_sized_block == nullptr) @@ -235,9 +268,11 @@ class free_list_resource : public cradle::pmr::identity_equal_resource static constexpr auto max_align_bytes = alignof (std::max_align_t); //============================================================================== + memory_resource* upstream = null_memory_resource(); mem_block* first_free = nullptr; std::size_t space = 0; std::size_t num_allocs = 0; + std::size_t min_realloc_size = 0; }; } // namespace cradle diff --git a/include/realtime_memory/memory_resources.h b/include/realtime_memory/memory_resources.h index dd6c1d2..854e2a0 100644 --- a/include/realtime_memory/memory_resources.h +++ b/include/realtime_memory/memory_resources.h @@ -392,7 +392,7 @@ inline monotonic_buffer_resource::monotonic_buffer_resource (std::size_t initial {} inline monotonic_buffer_resource::monotonic_buffer_resource (std::size_t initial_size, - memory_resource* up) noexcept + memory_resource* up) noexcept : upstream (up ? *up : *get_default_resource()), currentbuf (nullptr), currentbuf_size (0), @@ -405,8 +405,8 @@ inline monotonic_buffer_resource::monotonic_buffer_resource (void* buf, std::siz inline monotonic_buffer_resource::monotonic_buffer_resource (void* buf, - std::size_t bufsize, - memory_resource* up) noexcept + std::size_t bufsize, + memory_resource* up) noexcept : upstream (up ? *up : *get_default_resource()), currentbuf (buf), currentbuf_size (bufsize), diff --git a/include/realtime_memory/pmr_includes.h b/include/realtime_memory/pmr_includes.h index 0ebd07b..cdcf4db 100644 --- a/include/realtime_memory/pmr_includes.h +++ b/include/realtime_memory/pmr_includes.h @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #if defined (__apple_build_version__) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b466d7b..e01e2a9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.12) Include(FetchContent) FetchContent_Declare(Catch2 GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.0.1) # or a later release + GIT_TAG v3.11.0) # or a later release FetchContent_MakeAvailable(Catch2) add_executable(realtime_memory_tests diff --git a/tests/resource_tests.cpp b/tests/resource_tests.cpp index 4cb0fe9..a223ce0 100644 --- a/tests/resource_tests.cpp +++ b/tests/resource_tests.cpp @@ -2,6 +2,8 @@ // Copyright (c) 2019-2023 CradleApps, LLC - All Rights Reserved //============================================================================== +#include +#include #include #include @@ -343,17 +345,22 @@ TEST_CASE ("unsynchronized_pool_resource", "[memory_resource]") //============================================================================== /** Our custom 'recycling' resource. */ -TEST_CASE ("free_list_resource", "[memory_resource]") +TEST_CASE ("free_list_resource basics", "[memory_resource]") { - std::vector buf1 (256, std::byte(0)), buf2 (256, std::byte(0)); + namespace pmr = cradle::pmr; + + std::vector buf (256, std::byte (0)); + pmr::free_list_resource res (buf.data(), buf.size()); SECTION ("Only compares equal with itself") { - cradle::pmr::free_list_resource res1 (buf1.data(), buf1.size()); - cradle::pmr::free_list_resource res2 (buf2.data(), buf2.size()); + pmr::free_list_resource res2 (*pmr::get_default_resource(), 256); + pmr::free_list_resource res3 (*pmr::get_default_resource(), 256); - CHECK (res1 == res1); - CHECK_FALSE (res1 == res2); + CHECK (res == res); + CHECK (res2 == res2); + CHECK_FALSE (res == res2); + CHECK_FALSE (res2 == res3); } SECTION ("Non-copyable") @@ -364,18 +371,13 @@ TEST_CASE ("free_list_resource", "[memory_resource]") "FreeListBufferResource must not be copy assignable"); } - SECTION ("Throws bad_alloc when the buffer is exhausted") + SECTION ("Throws bad_alloc on a request for zero memory") { - cradle::pmr::free_list_resource res (buf1.data(), buf1.size()); - - CHECK_NOTHROW (res.allocate (buf1.size() - 32, 4)); - CHECK_THROWS_AS (res.allocate (32, 4), std::bad_alloc); + CHECK_THROWS_AS (res.allocate (0, 1), std::bad_alloc); } SECTION ("Throws bad_alloc on a request for overaligned memory") { - cradle::pmr::free_list_resource res (buf1.data(), buf1.size()); - auto align = alignof(std::max_align_t) * 2; CHECK_THROWS_AS (res.allocate (4, align), std::bad_alloc); @@ -383,8 +385,6 @@ TEST_CASE ("free_list_resource", "[memory_resource]") SECTION ("Delivers alignment up to alignof(std::max_align_t)") { - cradle::pmr::free_list_resource res (buf1.data(), buf1.size()); - auto align = GENERATE (as(), 1, 2, 4, alignof(std::max_align_t)); auto bytes = GENERATE (as(), 1, 3, 7, 8, 41, 77); CAPTURE (bytes, align); @@ -397,13 +397,27 @@ TEST_CASE ("free_list_resource", "[memory_resource]") res.deallocate (ptr1, 3, 1); res.deallocate (ptr2, bytes, align); } +} + +TEST_CASE ("free_list_resource (backed by buffer)", "[memory_resource]") +{ + namespace pmr = cradle::pmr; + std::vector buf1 (256, std::byte(0)), buf2 (256, std::byte(0)); + + SECTION ("Throws bad_alloc when the buffer is exhausted") + { + pmr::free_list_resource res (buf1.data(), buf1.size()); + + CHECK_NOTHROW (res.allocate (buf1.size() - 32, 4)); + CHECK_THROWS_AS (res.allocate (32, 4), std::bad_alloc); + } SECTION ("Re-merges split blocks that have been returned (defragmentation)") { std::vector buf (512, std::byte(0)); std::vector ptrs; - cradle::pmr::free_list_resource res (buf.data(), buf.size()); + pmr::free_list_resource res (buf.data(), buf.size()); const std::size_t alignment = 1; const std::size_t smallBlockSize = 2; @@ -432,12 +446,141 @@ TEST_CASE ("free_list_resource", "[memory_resource]") // Check that the two areas either side of the still-active pointer // can be re-merged in a defragmentation step. - CHECK (res.allocate (largeBlockSize, alignment)); - CHECK (res.allocate (largeBlockSize, alignment)); + CHECK (res.allocate (largeBlockSize, alignment) != nullptr); + CHECK (res.allocate (largeBlockSize, alignment) != nullptr); CHECK_THROWS_AS (res.allocate (largeBlockSize, alignment), std::bad_alloc); } } +TEST_CASE ("free_list_resource (backed by upstream resource)", "[memory_resource]") +{ + namespace pmr = cradle::pmr; + tracking_memory_resource upstream; + + SECTION ("Requests more memory from upstream when exhausted") + { + constexpr std::size_t initialSize = 256, minChunkSize = 512; + + pmr::free_list_resource res (upstream, initialSize, minChunkSize); + const auto initial = upstream.total_allocated(); + + CHECK (initial == initialSize); + + constexpr auto alloc1Size = initialSize - 32; + constexpr auto alloc2Size = 39; + + auto ptr1 = res.allocate (alloc1Size, 1); + CHECK (upstream.total_allocated() == initial); + + [[maybe_unused]] auto unused1 = res.allocate (alloc2Size, 2); + CHECK (upstream.total_allocated() == initial + minChunkSize); + + SECTION ("Freed memory from both chunks is reusable with no additional allocations") + { + res.deallocate (ptr1, alloc1Size, 1); + + [[maybe_unused]] auto unused2 = res.allocate (alloc1Size, 1); // reuse first chunk + [[maybe_unused]] auto unused3 = res.allocate (minChunkSize - alloc2Size - 64, 1); // use remainder of second chunk (minus header usage). + CHECK (upstream.total_allocated() == initial + minChunkSize); + } + } + + SECTION ("Extra memory chunks can be supplied from outside") + { + pmr::free_list_resource res (upstream, 256); + + [[maybe_unused]] auto unused1 = res.allocate (230, 1); + + res.expand (upstream.allocate (256, 1), 256); + const auto used = upstream.total_allocated(); + + CHECK (res.allocate (160, 1) != nullptr); + CHECK (upstream.total_allocated() == used); + } + + SECTION ("Additional chunks supplied from outside can all be used") + { + constexpr std::size_t minChunkSize = 512; + pmr::free_list_resource res (upstream, 256, minChunkSize); + + res.expand (upstream.allocate (256, 1), 256); + res.expand (upstream.allocate (256, 1), 256); + res.expand (upstream.allocate (256, 1), 256); + + const auto used = upstream.total_allocated(); + + auto ptr1 = res.allocate (200, 1); + auto ptr2 = res.allocate (200, 1); + auto ptr3 = res.allocate (200, 1); + auto ptr4 = res.allocate (200, 1); + CHECK_FALSE (ptr1 == nullptr); + CHECK_FALSE (ptr2 == nullptr); + CHECK_FALSE (ptr3 == nullptr); + CHECK_FALSE (ptr4 == nullptr); + + CHECK (upstream.total_allocated() == used); + + SECTION ("Allocation that cannot be satisfied (allocate from upstream)") + { + // Now it must allocate, because no single chunk has the required space left + CHECK (res.allocate (200, 1) != nullptr); + CHECK (upstream.total_allocated() == used + minChunkSize); + } + + SECTION ("Free then allocate into one of the existing chunks") + { + // check smaller, equal and larger sizes (checks de-frag). + auto size = GENERATE (as(), 160, 200, 230); + + res.deallocate (ptr2, 200, 1); + CHECK (res.allocate (size, 1) != nullptr); + CHECK (upstream.total_allocated() == used); + } + } + + SECTION ("Additional chunks that are sequential can be de-fragmented into one") + { + pmr::free_list_resource res (upstream, 256); + REQUIRE (upstream.total_allocated() == 256); + + std::vector buf (768, std::byte (0)); + res.expand (buf.data(), 256); + res.expand (buf.data() + 256, 256); + res.expand (buf.data() + 512, 256); + + auto ptr1 = res.allocate (200, 1); + auto ptr2 = res.allocate (200, 1); + auto ptr3 = res.allocate (200, 1); + auto ptr4 = res.allocate (200, 1); + REQUIRE_FALSE (ptr1 == nullptr); + REQUIRE_FALSE (ptr2 == nullptr); + REQUIRE_FALSE (ptr3 == nullptr); + REQUIRE_FALSE (ptr4 == nullptr); + REQUIRE (upstream.total_allocated() == 256); + + res.deallocate (ptr1, 200); + res.deallocate (ptr2, 200); + res.deallocate (ptr3, 200); + res.deallocate (ptr4, 200); + + SECTION ("When defragmentation can find a large enough block") + { + CHECK (res.allocate (710, 1) != nullptr); // triggers defragmentation + CHECK (upstream.total_allocated() == 256); // served from existing chunks + } + + SECTION ("When defragmentation can't find a block") + { + CHECK (res.allocate (780, 1) != nullptr); // too large, even after degfragmentation + auto used = upstream.total_allocated(); + CHECK (used > 256); // served from upstream + + CHECK (res.allocate (720, 1) != nullptr); // can still use block found during defragmentation + CHECK (upstream.total_allocated() == used); + } + } +} + //============================================================================== /** Common tests that all three resources should satisfy. * Some of these are just sanity checks that "it doesn't crash".