diff --git a/include/hwmalloc/heap.hpp b/include/hwmalloc/heap.hpp index 8d400b8..76b28cc 100644 --- a/include/hwmalloc/heap.hpp +++ b/include/hwmalloc/heap.hpp @@ -1,7 +1,7 @@ /* * ghex-org * - * Copyright (c) 2014-2023, ETH Zurich + * Copyright (c) 2014-2025, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,9 @@ class heap template using unique_ptr = unique_ptr; + // Note: sizes below are defaults and can be changed through heap_config and + // environment variables. + // // There are 5 size classes that the heap uses. For each size class it relies on a // fixed_size_heap. The size classes are: // - tiny: heaps with linearly increasing block sizes, each heap backed by 16KiB segments @@ -98,78 +102,56 @@ class heap // : : m_huge_heaps: map private: - static constexpr std::size_t log2_c(std::size_t n) noexcept + static std::size_t tiny_bucket_index(std::size_t n, std::size_t tiny_increment, + std::size_t tiny_increment_shift) noexcept { - return ((n < 2) ? 1 : 1 + log2_c(n >> 1)); + return ((n + tiny_increment - 1) >> tiny_increment_shift) - 1; } - static const std::size_t s_tiny_limit = (1u << 7); // 128 - static const std::size_t s_small_limit = (1u << 10); // 1024 - static const std::size_t s_large_limit = (1u << 16); // 65536 - - static const std::size_t s_bucket_shift = log2_c(s_tiny_limit) - 1; - - static const std::size_t s_tiny_segment = 0x04000; // 16KiB - static const std::size_t s_small_segment = 0x08000; // 32KiB - static const std::size_t s_large_segment = 0x10000; // 64KiB - - static const std::size_t s_tiny_increment_shift = 3; - static const std::size_t s_tiny_increment = (1u << s_tiny_increment_shift); // = 8 - - static const std::size_t s_num_tiny_heaps = s_tiny_limit / s_tiny_increment; - static const std::size_t s_num_small_heaps = log2_c(s_small_limit) - log2_c(s_tiny_limit); - static const std::size_t s_num_large_heaps = log2_c(s_large_limit) - log2_c(s_small_limit); - - static std::size_t tiny_bucket_index(std::size_t n) noexcept + static std::size_t bucket_index(std::size_t n, std::size_t bucket_shift) noexcept { - return ((n + s_tiny_increment - 1) >> s_tiny_increment_shift) - 1; - } - - static std::size_t bucket_index(std::size_t n) noexcept - { - return log2_c((n - 1) >> s_bucket_shift) - 1; - } - - static constexpr std::size_t round_to_pow_of_2(std::size_t n) noexcept - { - return 1u << log2_c(n - 1); + return detail::log2_c((n - 1) >> bucket_shift) - 1; } private: + heap_config m_config; Context* m_context; std::size_t m_max_size; - bool m_never_free; - std::size_t m_num_reserve_segments; heap_vector m_tiny_heaps; heap_vector m_heaps; heap_map m_huge_heaps; std::mutex m_mutex; public: - heap(Context* context, bool never_free = false, std::size_t num_reserve_segments = 1) - : m_context{context} - , m_max_size(std::max(round_to_pow_of_2(s_large_limit * 2), s_large_limit)) - , m_never_free{never_free} - , m_num_reserve_segments{num_reserve_segments} - , m_tiny_heaps(s_tiny_limit / s_tiny_increment) - , m_heaps(bucket_index(m_max_size) + 1) + heap(Context* context, heap_config const& config = get_default_heap_config()) + : m_config{config} + , m_context{context} + , m_max_size( + std::max(detail::round_to_pow_of_2(m_config.m_large_limit * 2), m_config.m_large_limit)) + , m_tiny_heaps(m_config.m_tiny_limit / m_config.m_tiny_increment) + , m_heaps(bucket_index(m_max_size, m_config.m_bucket_shift) + 1) { for (std::size_t i = 0; i < m_tiny_heaps.size(); ++i) m_tiny_heaps[i] = std::make_unique(m_context, - s_tiny_increment * (i + 1), s_tiny_segment, m_never_free, m_num_reserve_segments); + m_config.m_tiny_increment * (i + 1), m_config.m_tiny_segment_size, + m_config.m_never_free, m_config.m_num_reserve_segments); - for (std::size_t i = 0; i < s_num_small_heaps; ++i) + for (std::size_t i = 0; i < m_config.m_num_small_heaps; ++i) m_heaps[i] = std::make_unique(m_context, - (s_tiny_limit << (i + 1)), s_small_segment, m_never_free, m_num_reserve_segments); + (m_config.m_tiny_limit << (i + 1)), m_config.m_small_segment_size, + m_config.m_never_free, m_config.m_num_reserve_segments); - for (std::size_t i = 0; i < s_num_large_heaps; ++i) - m_heaps[i + s_num_small_heaps] = std::make_unique(m_context, - (s_small_limit << (i + 1)), s_large_segment, m_never_free, m_num_reserve_segments); + for (std::size_t i = 0; i < m_config.m_num_large_heaps; ++i) + m_heaps[i + m_config.m_num_small_heaps] = std::make_unique( + m_context, (m_config.m_small_limit << (i + 1)), m_config.m_large_segment_size, + m_config.m_never_free, m_config.m_num_reserve_segments); - for (std::size_t i = 0; i < m_heaps.size() - (s_num_small_heaps + s_num_large_heaps); ++i) - m_heaps[i + s_num_small_heaps + s_num_large_heaps] = - std::make_unique(m_context, (s_large_limit << (i + 1)), - (s_large_limit << (i + 1)), m_never_free, m_num_reserve_segments); + for (std::size_t i = 0; + i < m_heaps.size() - (m_config.m_num_small_heaps + m_config.m_num_large_heaps); ++i) + m_heaps[i + m_config.m_num_small_heaps + m_config.m_num_large_heaps] = + std::make_unique(m_context, + (m_config.m_large_limit << (i + 1)), (m_config.m_large_limit << (i + 1)), + m_config.m_never_free, m_config.m_num_reserve_segments); } heap(heap const&) = delete; @@ -194,20 +176,22 @@ class heap pointer allocate(std::size_t size, std::size_t numa_node) { - if (size <= s_tiny_limit) - return {m_tiny_heaps[tiny_bucket_index(size)]->allocate(numa_node)}; + if (size <= m_config.m_tiny_limit) + return {m_tiny_heaps[tiny_bucket_index(size, m_config.m_tiny_increment, + m_config.m_tiny_increment_shift)] + ->allocate(numa_node)}; else if (size <= m_max_size) - return {m_heaps[bucket_index(size)]->allocate(numa_node)}; + return {m_heaps[bucket_index(size, m_config.m_bucket_shift)]->allocate(numa_node)}; else { fixed_size_heap_type* h; { std::lock_guard lock(m_mutex); - const auto s = round_to_pow_of_2(size); + const auto s = detail::round_to_pow_of_2(size); auto& u_ptr = m_huge_heaps[s]; if (!u_ptr) - u_ptr = std::make_unique(m_context, s, s, m_never_free, - m_num_reserve_segments); + u_ptr = std::make_unique(m_context, s, s, + m_config.m_never_free, m_config.m_num_reserve_segments); h = u_ptr.get(); } return {h->allocate(numa_node)}; @@ -223,20 +207,23 @@ class heap #if HWMALLOC_ENABLE_DEVICE pointer allocate(std::size_t size, std::size_t numa_node, int device_id) { - if (size <= s_tiny_limit) - return {m_tiny_heaps[tiny_bucket_index(size)]->allocate(numa_node, device_id)}; + if (size <= m_config.m_tiny_limit) + return {m_tiny_heaps[tiny_bucket_index(size, m_config.m_tiny_increment, + m_config.m_tiny_increment_shift)] + ->allocate(numa_node, device_id)}; else if (size <= m_max_size) - return {m_heaps[bucket_index(size)]->allocate(numa_node, device_id)}; + return {m_heaps[bucket_index(size, m_config.m_bucket_shift)]->allocate(numa_node, + device_id)}; else { fixed_size_heap_type* h; { std::lock_guard lock(m_mutex); - const auto s = round_to_pow_of_2(size); + const auto s = detail::round_to_pow_of_2(size); auto& u_ptr = m_huge_heaps[s]; if (!u_ptr) - u_ptr = std::make_unique(m_context, s, s, m_never_free, - m_num_reserve_segments); + u_ptr = std::make_unique(m_context, s, s, + m_config.m_never_free, m_config.m_num_reserve_segments); h = u_ptr.get(); } return {h->allocate(numa_node, device_id)}; @@ -293,29 +280,4 @@ class heap } }; -template -const std::size_t heap::s_tiny_limit; -template -const std::size_t heap::s_small_limit; -template -const std::size_t heap::s_large_limit; -template -const std::size_t heap::s_bucket_shift; -template -const std::size_t heap::s_tiny_segment; -template -const std::size_t heap::s_small_segment; -template -const std::size_t heap::s_large_segment; -template -const std::size_t heap::s_tiny_increment_shift; -template -const std::size_t heap::s_tiny_increment; -template -const std::size_t heap::s_num_tiny_heaps; -template -const std::size_t heap::s_num_small_heaps; -template -const std::size_t heap::s_num_large_heaps; - } // namespace hwmalloc diff --git a/include/hwmalloc/heap_config.hpp b/include/hwmalloc/heap_config.hpp new file mode 100644 index 0000000..39d71ef --- /dev/null +++ b/include/hwmalloc/heap_config.hpp @@ -0,0 +1,63 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace hwmalloc +{ +namespace detail +{ +inline constexpr std::size_t +log2_c(std::size_t n) noexcept +{ + return ((n < 2) ? 1 : 1 + log2_c(n >> 1)); +} + +inline constexpr std::size_t +round_to_pow_of_2(std::size_t n) noexcept +{ + return 1u << log2_c(n - 1); +} +} // namespace detail + +struct heap_config +{ + static constexpr bool never_free_default = false; + static constexpr std::size_t num_reserve_segments_default = 16u; + static constexpr std::size_t tiny_limit_default = 128u; // 128B + static constexpr std::size_t small_limit_default = 4096u; // 4KiB + static constexpr std::size_t large_limit_default = 2097152u; // 2MiB + static constexpr std::size_t tiny_segment_size_default = 65536u; // 64KiB + static constexpr std::size_t small_segment_size_default = 65536u; // 64KiB + static constexpr std::size_t large_segment_size_default = 2097152u; // 2MiB + + bool m_never_free; + std::size_t m_num_reserve_segments; + std::size_t m_tiny_limit; + std::size_t m_small_limit; + std::size_t m_large_limit; + std::size_t m_bucket_shift = detail::log2_c(m_tiny_limit) - 1; + std::size_t m_tiny_segment_size; + std::size_t m_small_segment_size; + std::size_t m_large_segment_size; + static constexpr std::size_t m_tiny_increment_shift = 3; + static constexpr std::size_t m_tiny_increment = (1u << m_tiny_increment_shift); + std::size_t m_num_tiny_heaps = m_tiny_limit / m_tiny_increment; + std::size_t m_num_small_heaps = detail::log2_c(m_small_limit) - detail::log2_c(m_tiny_limit); + std::size_t m_num_large_heaps = detail::log2_c(m_large_limit) - detail::log2_c(m_small_limit); + + heap_config(bool never_free, std::size_t num_reserve_segments, std::size_t tiny_limit, + std::size_t small_limit, std::size_t large_limit, std::size_t tiny_segment_size, + std::size_t small_segment_size, std::size_t large_segment_size); +}; + +heap_config const& get_default_heap_config(); +} // namespace hwmalloc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 66e14a6..042a508 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,5 @@ +target_sources(hwmalloc PRIVATE heap_config.cpp) + if (NUMA_LIBRARY) target_sources(hwmalloc PRIVATE numa.cpp) else() diff --git a/src/heap_config.cpp b/src/heap_config.cpp new file mode 100644 index 0000000..ebd89ad --- /dev/null +++ b/src/heap_config.cpp @@ -0,0 +1,126 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#ifdef HWMALLOC_ENABLE_LOGGING +#include + +#include +#endif + +#include +#include +#include + +namespace hwmalloc +{ +namespace detail +{ +template +T +parse_value(const char* env_value) +{ + // This currently only supports the types needed within this file. If this + // should be used elsewhere this can be generalized to something that can be + // extended e.g. with template specializations instead a hardcoded list with + // if-constexpr. + if constexpr (std::is_same_v) { return std::stoul(env_value); } + else if constexpr (std::is_same_v) { return std::stoul(env_value) != 0; } + else { static_assert(sizeof(T) == 0, "Unsupported type for parse_value"); } +} + +template +T +get_env(const char* name, T default_value) noexcept +{ + const char* env_value = std::getenv(name); + if (env_value) + { + try + { + return parse_value(env_value); + } + catch (...) + { +#ifdef HWMALLOC_ENABLE_LOGGING + HWMALLOC_LOG("failed to parse boolean configuration option", name, "=", env_value, + "(expected 0 or 1), using default =", default_value); +#endif + return default_value; + } + } + + return default_value; +} +} // namespace detail + +heap_config::heap_config(bool never_free, std::size_t num_reserve_segments, std::size_t tiny_limit, + std::size_t small_limit, std::size_t large_limit, std::size_t tiny_segment_size, + std::size_t small_segment_size, std::size_t large_segment_size) +: m_never_free{never_free} +, m_num_reserve_segments{num_reserve_segments} +, m_tiny_limit{detail::round_to_pow_of_2(tiny_limit)} +, m_small_limit{detail::round_to_pow_of_2(small_limit)} +, m_large_limit{detail::round_to_pow_of_2(large_limit)} +, m_tiny_segment_size{detail::round_to_pow_of_2(tiny_segment_size)} +, m_small_segment_size{detail::round_to_pow_of_2(small_segment_size)} +, m_large_segment_size{detail::round_to_pow_of_2(large_segment_size)} +{ + // Validate that tiny_limit < small_limit < large_limit + if (!(m_tiny_limit < m_small_limit && m_small_limit < m_large_limit)) + { + std::ostringstream os; + os << "Invalid heap size configuration: HWMALLOC_TINY_LIMIT < HWMALLOC_SMALL_LIMIT < HWMALLOC_LARGE_LIMIT must hold. "; + os << "Got HWMALLOC_TINY_LIMIT=" << m_tiny_limit + << ", HWMALLOC_SMALL_LIMIT=" << m_small_limit + << ", HWMALLOC_LARGE_LIMIT=" << m_large_limit << "."; + throw std::runtime_error(os.str()); + } + + // Validate that limits are at most segment sizes + if (!(tiny_limit <= tiny_segment_size && small_limit <= small_segment_size && + large_limit <= large_segment_size)) + { + std::ostringstream os; + os << "Invalid heap size configuration: Limits must be at most segment sizes. "; + os << "Got HWMALLOC_TINY_LIMIT=" << m_tiny_limit + << ", HWMALLOC_TINY_SEGMENT_SIZE=" << m_tiny_segment_size + << ", HWMALLOC_SMALL_LIMIT=" << m_small_limit + << ", HWMALLOC_SMALL_SEGMENT_SIZE=" << m_small_segment_size + << ", HWMALLOC_LARGE_LIMIT=" << m_large_limit + << ", HWMALLOC_LARGE_SEGMENT_SIZE=" << m_large_segment_size << "."; + throw std::runtime_error(os.str()); + } +} + +heap_config const& +get_default_heap_config() +{ + static heap_config config{ + detail::get_env("HWMALLOC_NEVER_FREE", heap_config::never_free_default), + detail::get_env("HWMALLOC_NUM_RESERVE_SEGMENTS", + heap_config::num_reserve_segments_default), + detail::get_env("HWMALLOC_TINY_LIMIT", heap_config::tiny_limit_default), + detail::get_env("HWMALLOC_SMALL_LIMIT", heap_config::small_limit_default), + detail::get_env("HWMALLOC_LARGE_LIMIT", heap_config::large_limit_default), + detail::get_env("HWMALLOC_TINY_SEGMENT_SIZE", + heap_config::tiny_segment_size_default), + detail::get_env("HWMALLOC_SMALL_SEGMENT_SIZE", + heap_config::small_segment_size_default), + detail::get_env("HWMALLOC_LARGE_SEGMENT_SIZE", + heap_config::large_segment_size_default) + + }; + + return config; +} +} // namespace hwmalloc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f339a3c..84f7a2f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -21,6 +21,7 @@ function(reg_test t) target_link_libraries(${t} PRIVATE gtest_main) target_link_libraries(${t} PRIVATE hwmalloc) target_link_libraries(${t} PRIVATE Boost::boost) + target_include_directories(${t} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") add_test(NAME ${t} COMMAND $) endfunction() @@ -33,6 +34,9 @@ if (NUMA_LIBRARY AND NOT HWMALLOC_DISABLE_NUMA_TEST) endif() reg_test(test_ptr) reg_test(test_segment) +reg_test(test_heap_config) +reg_test(test_heap_config_default) +reg_test(test_heap_config_invalid) if (NUMA_LIBRARY) find_package(OpenMP REQUIRED) diff --git a/test/heap_config_defaults.hpp b/test/heap_config_defaults.hpp new file mode 100644 index 0000000..048bfbd --- /dev/null +++ b/test/heap_config_defaults.hpp @@ -0,0 +1,23 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +namespace hwmalloc::test +{ +// These should change accordingly if the defaults in heap_config.hpp change. +// They are defined here for convencience for tests. They are derived from +// heap_config defaults. +inline constexpr std::size_t bucket_shift_default = 7u; +inline constexpr std::size_t tiny_increment_shift_default = 3u; +inline constexpr std::size_t tiny_increment_default = 8u; +inline constexpr std::size_t num_tiny_heaps_default = 16u; +inline constexpr std::size_t num_small_heaps_default = 5u; +inline constexpr std::size_t num_large_heaps_default = 9u; +} // namespace hwmalloc::test diff --git a/test/test_heap_config.cpp b/test/test_heap_config.cpp new file mode 100644 index 0000000..030b755 --- /dev/null +++ b/test/test_heap_config.cpp @@ -0,0 +1,310 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +#include + +#include + +#include + +// Check that the default configuration values are as expected. +TEST(heap_config, defaults) +{ + hwmalloc::heap_config config = hwmalloc::get_default_heap_config(); + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +// We try changing one parameter a time compared to the defaults and check that +// the derived values change accordingly. +TEST(heap_config, never_free) +{ + hwmalloc::heap_config config{true, hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, true); // This changes to the literal value we set + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +TEST(heap_config, num_reserve_segments) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, 7, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, 7u); // This changes to the literal value we set + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +TEST(heap_config, tiny_limit) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, 256u, + hwmalloc::heap_config::small_limit_default, hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, 256u); // This changes to the literal value we set + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, 8u); // This changes based on tiny_limit + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, 32u); // This number doubles... + EXPECT_EQ(config.m_num_small_heaps, 4u); // While this is decreased by one + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +TEST(heap_config, tiny_segment) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, + hwmalloc::heap_config::large_limit_default, 8192u, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, 8192u); // This changes to the literal value we set + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +TEST(heap_config, small_limit) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, 8192u, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, 8192u); // This changes to the literal value we set + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, 6u); // This is log2(8192)-log2(128) = 6... + EXPECT_EQ(config.m_num_large_heaps, 8u); // While this is decreased accordingly +} + +TEST(heap_config, small_segment) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, 16384u, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, 16384u); // This changes to the literal value we set + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +TEST(heap_config, large_limit) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, 32768u, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, 32768u); // This changes to the literal value we set + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, hwmalloc::heap_config::large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, 3u); // This is decreased accordingly +} + +TEST(heap_config, large_segment) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, 4194304u}; + + EXPECT_EQ(config.m_never_free, hwmalloc::heap_config::never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, hwmalloc::heap_config::num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, hwmalloc::heap_config::small_limit_default); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, hwmalloc::heap_config::tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, 4194304u); // This changes to the literal value we set + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, hwmalloc::test::num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, hwmalloc::test::num_large_heaps_default); +} + +// Check that setting non-power-of-two values results in rounding to power of two +TEST(heap_config, power_of_two) +{ + hwmalloc::heap_config config{hwmalloc::heap_config::never_free_default, 17u, 200u, 500u, 20000u, + 1000u, 3000u, 100000u}; + + EXPECT_EQ(config.m_num_reserve_segments, 17u); // Not rounded up + EXPECT_EQ(config.m_tiny_limit, 256u); // Rounded up from 200 + EXPECT_EQ(config.m_small_limit, 512u); // Rounded up from 500 + EXPECT_EQ(config.m_large_limit, 32768u); // Rounded up from 20000 + EXPECT_EQ(config.m_tiny_segment_size, 1024u); // Rounded up from 1000 + EXPECT_EQ(config.m_small_segment_size, 4096u); // Rounded up from 3000 + EXPECT_EQ(config.m_large_segment_size, 131072u); // Rounded up from 100000 +} + +// Check that setting invalid values results in exceptions +TEST(heap_config, validate_tiny_small_limit) +{ + EXPECT_THROW((hwmalloc::heap_config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, 2048u, 1024u, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_small_large_limit) +{ + EXPECT_THROW((hwmalloc::heap_config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, 131072u, 65536u, + hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_tiny_limit_segment_size) +{ + EXPECT_THROW( + (hwmalloc::heap_config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, 2048u, + hwmalloc::heap_config::small_limit_default, hwmalloc::heap_config::large_limit_default, 1024u, + hwmalloc::heap_config::small_segment_size_default, + hwmalloc::heap_config::large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_small_limit_segment_size) +{ + EXPECT_THROW((hwmalloc::heap_config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, 131072u, + hwmalloc::heap_config::large_limit_default, + hwmalloc::heap_config::tiny_segment_size_default, 65536u, + hwmalloc::heap_config::large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_large_limit_segment_size) +{ + EXPECT_THROW((hwmalloc::heap_config{hwmalloc::heap_config::never_free_default, + hwmalloc::heap_config::num_reserve_segments_default, + hwmalloc::heap_config::tiny_limit_default, hwmalloc::heap_config::small_limit_default, + 262144u, hwmalloc::heap_config::tiny_segment_size_default, + hwmalloc::heap_config::small_segment_size_default, 131072u}), + std::runtime_error); +} diff --git a/test/test_heap_config_default.cpp b/test/test_heap_config_default.cpp new file mode 100644 index 0000000..885be0d --- /dev/null +++ b/test/test_heap_config_default.cpp @@ -0,0 +1,45 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +#include + +#include + +// Check that we can change the default configuration values through environment +// variables. +TEST(config, defaults) +{ + ::setenv("HWMALLOC_NEVER_FREE", "1", 1); + ::setenv("HWMALLOC_NUM_RESERVE_SEGMENTS", "19", 1); + ::setenv("HWMALLOC_TINY_LIMIT", "512", 1); + ::setenv("HWMALLOC_SMALL_LIMIT", "2000", 1); + ::setenv("HWMALLOC_LARGE_LIMIT", "131072", 1); + ::setenv("HWMALLOC_TINY_SEGMENT_SIZE", "16384", 1); + ::setenv("HWMALLOC_SMALL_SEGMENT_SIZE", "32768", 1); + ::setenv("HWMALLOC_LARGE_SEGMENT_SIZE", "262144", 1); + + hwmalloc::heap_config config = hwmalloc::get_default_heap_config(); + + EXPECT_EQ(config.m_never_free, true); + EXPECT_EQ(config.m_num_reserve_segments, 19u); + EXPECT_EQ(config.m_tiny_limit, 512u); + EXPECT_EQ(config.m_small_limit, 2048); + EXPECT_EQ(config.m_large_limit, 131072u); + EXPECT_EQ(config.m_bucket_shift, 9u); + EXPECT_EQ(config.m_tiny_segment_size, 16384u); + EXPECT_EQ(config.m_small_segment_size, 32768u); + EXPECT_EQ(config.m_large_segment_size, 262144u); + EXPECT_EQ(config.m_tiny_increment_shift, 3u); + EXPECT_EQ(config.m_tiny_increment, 8u); + EXPECT_EQ(config.m_num_tiny_heaps, 64u); + EXPECT_EQ(config.m_num_small_heaps, 2u); + EXPECT_EQ(config.m_num_large_heaps, 6u); +} diff --git a/test/test_heap_config_invalid.cpp b/test/test_heap_config_invalid.cpp new file mode 100644 index 0000000..6a739f3 --- /dev/null +++ b/test/test_heap_config_invalid.cpp @@ -0,0 +1,49 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +#include + +#include + +#include + +// Test that config falls back to defaults if environment variables are given +// non-numeric values. +// +// A warning is printed if logging is enabled. +TEST(config, defaults) +{ + ::setenv("HWMALLOC_NEVER_FREE", "foo", 1); + ::setenv("HWMALLOC_NUM_RESERVE_SEGMENTS", "9", 1); + ::setenv("HWMALLOC_TINY_LIMIT", "abcd", 1); + ::setenv("HWMALLOC_SMALL_LIMIT", "8192", 1); + ::setenv("HWMALLOC_LARGE_LIMIT", "foo", 1); + ::setenv("HWMALLOC_TINY_SEGMENT_SIZE", "16384", 1); + ::setenv("HWMALLOC_SMALL_SEGMENT_SIZE", "bar", 1); + ::setenv("HWMALLOC_LARGE_SEGMENT_SIZE", "4194304", 1); + + hwmalloc::heap_config config = hwmalloc::get_default_heap_config(); + + EXPECT_EQ(config.m_never_free, false); + EXPECT_EQ(config.m_num_reserve_segments, 9u); + EXPECT_EQ(config.m_tiny_limit, hwmalloc::heap_config::tiny_limit_default); + EXPECT_EQ(config.m_small_limit, 8192); + EXPECT_EQ(config.m_large_limit, hwmalloc::heap_config::large_limit_default); + EXPECT_EQ(config.m_bucket_shift, hwmalloc::test::bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, 16384u); + EXPECT_EQ(config.m_small_segment_size, hwmalloc::heap_config::small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, 4194304u); + EXPECT_EQ(config.m_tiny_increment_shift, hwmalloc::test::tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, hwmalloc::test::tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, hwmalloc::test::num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, 6u); + EXPECT_EQ(config.m_num_large_heaps, 8u); +} diff --git a/test/test_omp.cpp b/test/test_omp.cpp index c4b38ca..fb59fc3 100644 --- a/test/test_omp.cpp +++ b/test/test_omp.cpp @@ -73,8 +73,13 @@ register_memory(context&, void* ptr, std::size_t size) TEST(spread, neverfree) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c, 1024, true); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = true; + hc.m_num_reserve_segments = 1024; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -144,8 +149,13 @@ TEST(spread, neverfree) TEST(close, neverfree) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c, 1024, true); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = true; + hc.m_num_reserve_segments = 1024; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -212,8 +222,13 @@ TEST(close, neverfree) TEST(spread, free) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = false; + hc.m_num_reserve_segments = 1; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -275,8 +290,7 @@ TEST(spread, free) #pragma omp master { printf("SPREAD, free :: n_registrations %d\n", n_registrations); - EXPECT_TRUE( - n_registrations == (thr_per_node * NITER * nnodes * (NBUFFERS - 1) + nnodes)); + EXPECT_EQ(n_registrations, (thr_per_node * NITER * nnodes * (NBUFFERS - 1) + nnodes)); } } } @@ -284,8 +298,13 @@ TEST(spread, free) TEST(close, free) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = false; + hc.m_num_reserve_segments = 1; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -343,7 +362,7 @@ TEST(close, free) #pragma omp master { printf("CLOSE, free :: n_registrations %d\n", n_registrations); - EXPECT_TRUE(n_registrations == (nthr * NITER * (NBUFFERS - 1) + nused_nodes)); + EXPECT_EQ(n_registrations, (nthr * NITER * (NBUFFERS - 1) + nused_nodes)); } } }