Improve code coverage results (#154)

TrentHouliston · web-flow · commit 28d18fa46e25 · 2025-01-25T20:15:08.000+11:00
diff --git a/.github/workflows/gcc.yaml b/.github/workflows/gcc.yaml
@@ -92,8 +92,14 @@ jobs:
 
       - name: Test
         timeout-minutes: 2
-        working-directory: build/tests
-        run: ctest --output-on-failure -E "dsl/UDP"
+        working-directory: build
+        run: ninja run_all_tests -k 0
+
+      - name: Test Summary
+        if: ${{ !cancelled() }}
+        uses: test-summary/action@v2
+        with:
+          paths: "build/reports/tests/*.junit.xml"
 
       - name: Upload Traces
         if: ${{ !cancelled() }}
diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml
@@ -60,8 +60,14 @@ jobs:
 
       - name: Test
         timeout-minutes: 5
-        working-directory: build/tests
-        run: ctest --output-on-failure
+        working-directory: build
+        run: ninja run_all_tests -k 0
+
+      - name: Test Summary
+        if: ${{ !cancelled() }}
+        uses: test-summary/action@v2
+        with:
+          paths: "build/reports/tests/*.junit.xml"
 
       - name: Upload Traces
         if: ${{ !cancelled() }}
diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml
@@ -33,9 +33,9 @@ jobs:
         uses: SonarSource/sonarcloud-github-c-cpp@v3
 
       - name: Install CMake
-        uses: lukka/get-cmake@latest
+        uses: lukka/get-cmake@v3.30.5
         with:
-          cmakeVersion: 3.27.1
+          cmakeVersion: 3.30.5
           ninjaVersion: 1.11.1
 
       - name: Setup CCache
@@ -45,14 +45,16 @@ jobs:
           max-size: 100M
 
       - name: Configure CMake
+        env:
+          CXXFLAGS: -DNUCLEAR_TEST_TIME_UNIT_DEN=10
         run: |
           cmake -E make_directory build
           cmake -S . -B build \
             -GNinja \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-            -DBUILD_TESTS=ON \
             -DCMAKE_BUILD_TYPE=Debug \
+            -DBUILD_TESTS=ON \
             -DCI_BUILD=ON \
             -DENABLE_COVERAGE=ON \
             -DENABLE_CLANG_TIDY=OFF
@@ -63,12 +65,28 @@ jobs:
 
       - name: Run tests to generate coverage statistics
         timeout-minutes: 10
-        working-directory: build/tests
-        run: ctest --output-on-failure
+        working-directory: build
+        run: ninja run_all_tests -j1 -k 0
+
+      - name: Test Summary
+        if: ${{ !cancelled() }}
+        uses: test-summary/action@v2
+        with:
+          paths: "build/reports/tests/*.junit.xml"
 
       - name: Collect coverage into one XML report
         if: ${{ !cancelled() }}
-        run: gcovr --gcov-ignore-parse-errors=negative_hits.warn_once_per_file --exclude-unreachable-branches --exclude-noncode-lines --sonarqube > coverage.xml
+        run: |
+          gcovr \
+            --root . \
+            --object-directory build \
+            --force-color \
+            --no-markers \
+            --decisions \
+            --calls \
+            --exclude-noncode-lines \
+            --gcov-ignore-parse-errors negative_hits.warn \
+            --sonarqube "coverage.xml"
 
       - name: Upload coverage report
         if: ${{ !cancelled() }}
@@ -86,6 +104,10 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
           sonar-scanner \
+            --define sonar.projectKey=Fastcode_NUClear \
+            --define sonar.organization=fastcode \
+            --define sonar.sources=src \
+            --define sonar.tests=tests \
             --define sonar.cfamily.compile-commands=build/compile_commands.json \
             --define sonar.coverageReportPaths=coverage.xml
 
diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
@@ -81,8 +81,14 @@ jobs:
 
       - name: Test
         timeout-minutes: 5
-        working-directory: build/tests
-        run: ctest --output-on-failure
+        working-directory: build
+        run: ninja run_all_tests -k 0
+
+      - name: Test Summary
+        if: ${{ !cancelled() }}
+        uses: test-summary/action@v2
+        with:
+          paths: "build/reports/tests/*.junit.xml"
 
       - name: Upload Traces
         if: ${{ !cancelled() }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -89,6 +89,7 @@ option(BUILD_TESTS "Builds all of the NUClear unit tests." ON)
 if(BUILD_TESTS)
   enable_testing()
   add_subdirectory(tests)
+  include(TestRunner)
 endif()
 
 # Add the documentation subdirectory
diff --git a/cmake/TestRunner.cmake b/cmake/TestRunner.cmake
@@ -0,0 +1,60 @@
+# Collect all currently added targets in all subdirectories
+#
+# Parameters:
+#  - _result the list containing all found targets
+#  - _dir root directory to start looking from
+function(get_all_catch_test_targets _result _dir)
+  get_property(
+    _subdirs
+    DIRECTORY "${_dir}"
+    PROPERTY SUBDIRECTORIES
+  )
+  foreach(_subdir IN LISTS _subdirs)
+    get_all_catch_test_targets(${_result} "${_subdir}")
+  endforeach()
+
+  unset(catch_targets)
+  get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS)
+  foreach(target ${_sub_targets})
+    get_target_property(target_type ${target} TYPE)
+    if(target_type STREQUAL "EXECUTABLE")
+      get_target_property(target_link_libraries ${target} INTERFACE_LINK_LIBRARIES)
+      if(target_link_libraries MATCHES "Catch2::Catch2")
+        list(APPEND catch_targets ${target})
+      endif()
+    endif()
+  endforeach()
+
+  set(${_result}
+      ${${_result}} ${catch_targets}
+      PARENT_SCOPE
+  )
+endfunction()
+
+# Find all executable targets that link to Catch2::Catch2WithMain || Catch2::Catch2
+get_all_catch_test_targets(all_targets ${PROJECT_SOURCE_DIR})
+
+# Create a custom command for each test target to run it
+# Make sure that coverage data is written with paths relative to the source directory
+unset(reports)
+foreach(target ${all_targets})
+
+  set(sonarqube_report_file "${PROJECT_BINARY_DIR}/reports/tests/${target}.sonarqube.xml")
+  set(junit_report_file "${PROJECT_BINARY_DIR}/reports/tests/${target}.junit.xml")
+  list(APPEND reports ${sonarqube_report_file} ${junit_report_file})
+  add_custom_command(
+    OUTPUT ${sonarqube_report_file} ${junit_report_file}
+    COMMAND $<TARGET_FILE:${target}> --reporter console --reporter SonarQube::out=${sonarqube_report_file} --reporter
+            JUnit::out=${junit_report_file}
+    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    USES_TERMINAL
+    COMMENT "Running test ${target}"
+  )
+endforeach()
+
+# Create a custom target that depends on all test targets
+add_custom_target(
+  run_all_tests
+  DEPENDS ${reports}
+  COMMENT "Running all Catch2 tests"
+)
diff --git a/sonar-project.properties b/sonar-project.properties
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -37,12 +37,14 @@ target_link_libraries(nuclear ${CMAKE_THREAD_LIBS_INIT})
 set_target_properties(nuclear PROPERTIES POSITION_INDEPENDENT_CODE ON)
 target_compile_features(nuclear PUBLIC cxx_std_14)
 
-# When enabling coverage, just set it on the nuclear target so it doesn't end up on catch2
-option(ENABLE_COVERAGE "Enable coverage support" OFF)
+option(ENABLE_COVERAGE "Compile with coverage support enabled.")
 if(ENABLE_COVERAGE)
-  target_compile_options(nuclear PUBLIC "-fprofile-arcs" "-ftest-coverage" "-fprofile-abs-path")
-  target_link_options(nuclear PUBLIC "-fprofile-arcs" "-ftest-coverage" "-fprofile-abs-path")
-endif()
+  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message(WARNING "Coverage is enabled but not build in debug mode. Coverage results may be misleading.")
+  endif()
+  target_compile_options(nuclear PUBLIC -fprofile-arcs -ftest-coverage -fprofile-abs-path -fprofile-update=atomic)
+  target_link_options(nuclear PUBLIC -fprofile-arcs)
+endif(ENABLE_COVERAGE)
 
 # Enable warnings, and all warnings are errors
 if(MSVC)
diff --git a/src/util/platform.hpp b/src/util/platform.hpp
@@ -96,17 +96,6 @@
 
 #endif  // _WIN32
 
-/*******************************************
- *      SHIM FOR THREAD LOCAL STORAGE      *
- *******************************************/
-#if defined(__GNUC__)
-    #define ATTRIBUTE_TLS __thread
-#elif defined(_WIN32)
-    #define ATTRIBUTE_TLS __declspec(thread)
-#else  // !__GNUC__ && !_MSC_VER
-    #error "Define a thread local storage qualifier for your compiler/platform!"
-#endif
-
 /*******************************************
  *           SHIM FOR NETWORKING           *
  *******************************************/
diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp
@@ -28,6 +28,7 @@
 #include <utility>
 
 #include "nuclear"
+#include "test_util/TimeUnit.hpp"     // IWYU pragma: export
 #include "test_util/diff_string.hpp"  // IWYU pragma: export
 
 namespace test_util {
@@ -57,7 +58,7 @@ class TestBase : public NUClear::Reactor {
 private:
     explicit TestBase(std::unique_ptr<NUClear::Environment> environment,
                       const bool& shutdown_on_idle             = true,
-                      const std::chrono::milliseconds& timeout = std::chrono::milliseconds(1000))
+                      const std::chrono::milliseconds& timeout = TimeUnit(20))
         : Reactor(std::move(environment)) {
 
         // Shutdown if the system is idle
diff --git a/tests/test_util/has_multicast.cpp b/tests/test_util/has_multicast.cpp
@@ -0,0 +1,48 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2025 NUClear Contributors
+ *
+ * This file is part of the NUClear codebase.
+ * See https://github.com/Fastcode/NUClear for further info.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+ * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "has_multicast.hpp"
+
+#include <algorithm>
+
+#include "util/network/get_interfaces.hpp"
+#include "util/platform.hpp"
+
+namespace test_util {
+
+bool has_ipv4_multicast() {
+    // See if any interface has multicast ipv4
+    auto ifaces = NUClear::util::network::get_interfaces();
+    return std::any_of(ifaces.begin(), ifaces.end(), [](const auto& iface) {
+        return iface.ip.sock.sa_family == AF_INET && iface.flags.multicast;
+    });
+}
+
+bool has_ipv6_multicast() {
+    // See if any interface has multicast ipv6
+    auto ifaces = NUClear::util::network::get_interfaces();
+    return std::any_of(ifaces.begin(), ifaces.end(), [](const auto& iface) {
+        return iface.ip.sock.sa_family == AF_INET6 && iface.flags.multicast;
+    });
+}
+
+}  // namespace test_util
diff --git a/tests/test_util/has_multicast.hpp b/tests/test_util/has_multicast.hpp
@@ -0,0 +1,33 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2025 NUClear Contributors
+ *
+ * This file is part of the NUClear codebase.
+ * See https://github.com/Fastcode/NUClear for further info.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+ * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef TEST_UTIL_HAS_IPV4_MULTICAST_HPP
+#define TEST_UTIL_HAS_IPV4_MULTICAST_HPP
+
+namespace test_util {
+
+bool has_ipv4_multicast();
+bool has_ipv6_multicast();
+
+}  // namespace test_util
+
+#endif  // TEST_UTIL_HAS_IPV4_MULTICAST_HPP
diff --git a/tests/tests/dsl/Every.cpp b/tests/tests/dsl/Every.cpp
@@ -39,7 +39,7 @@
 class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     TestReactor(std::unique_ptr<NUClear::Environment> environment)
-        : TestBase(std::move(environment), false, std::chrono::seconds(10)) {
+        : TestBase(std::move(environment), false, test_util::TimeUnit(200)) {
 
         // Trigger on 3 different types of every
         on<Every<1000, Per<std::chrono::seconds>>>().then([this]() { every_times.push_back(NUClear::clock::now()); });
diff --git a/tests/tests/dsl/Idle.cpp b/tests/tests/dsl/Idle.cpp
@@ -57,7 +57,7 @@ class TestReactor : public test_util::TestBase<TestReactor> {
     }
 
     explicit TestReactor(std::unique_ptr<NUClear::Environment> environment)
-        : TestBase(std::move(environment), false, std::chrono::seconds(5)) {
+        : TestBase(std::move(environment), false, test_util::TimeUnit(100)) {
 
         start_time = NUClear::clock::now();
 
diff --git a/tests/tests/dsl/IdleSingleGlobal.cpp b/tests/tests/dsl/IdleSingleGlobal.cpp
@@ -59,7 +59,7 @@ class TestReactor : public test_util::TestBase<TestReactor> {
     static constexpr int n_loops = 10000;
 
     explicit TestReactor(std::unique_ptr<NUClear::Environment> environment)
-        : TestBase(std::move(environment), false, std::chrono::seconds(2)) {
+        : TestBase(std::move(environment), false, test_util::TimeUnit(200)) {
 
         /*
          * Run idle on the default pool, and a task on the main pool.
diff --git a/tests/tests/dsl/SyncOrder.cpp b/tests/tests/dsl/SyncOrder.cpp
@@ -39,7 +39,8 @@ class TestReactor : public test_util::TestBase<TestReactor> {
         Message(int val) : val(val) {};
     };
 
-    TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment)) {
+    TestReactor(std::unique_ptr<NUClear::Environment> environment)
+        : TestBase(std::move(environment), true, test_util::TimeUnit(150)) {
 
         on<Trigger<Message<'A'>>, Sync<TestReactor>>().then([this](const Message<'A'>& m) {  //
             events.emplace_back('A', m.val);
diff --git a/tests/tests/dsl/TCP.cpp b/tests/tests/dsl/TCP.cpp
@@ -84,7 +84,7 @@ class TestReactor : public test_util::TestBase<TestReactor> {
     }
 
     TestReactor(std::unique_ptr<NUClear::Environment> environment, const std::vector<TestType>& active_tests_)
-        : TestBase(std::move(environment), false, std::chrono::seconds(2)), active_tests(active_tests_) {
+        : TestBase(std::move(environment), false, test_util::TimeUnit(50)), active_tests(active_tests_) {
 
         for (const auto& t : active_tests) {
             switch (t) {
diff --git a/tests/tests/dsl/UDP.cpp b/tests/tests/dsl/UDP.cpp
diff --git a/tests/tests/dsl/emit/Delay.cpp b/tests/tests/dsl/emit/Delay.cpp
