From a3a10d4ea0d95f9c433d11aa18f748be2c6098df Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 11:51:40 +1000 Subject: [PATCH 01/87] Replace `volatile bool`s with `std::atomic`s --- src/PowerPlant.cpp | 12 ++++++------ src/PowerPlant.hpp | 2 +- src/threading/TaskScheduler.cpp | 6 +++--- src/threading/TaskScheduler.hpp | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index aa654010a..8f2774d3b 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -31,10 +31,10 @@ PowerPlant::~PowerPlant() { } void PowerPlant::on_startup(std::function&& func) { - if (is_running) { throw std::runtime_error("Unable to do on_startup as the PowerPlant has already started"); } - else { - startup_tasks.push_back(func); + if (is_running.load()) { + throw std::runtime_error("Unable to do on_startup as the PowerPlant has already started"); } + else { startup_tasks.push_back(func); } } void PowerPlant::add_thread_task(std::function&& task) { @@ -44,7 +44,7 @@ void PowerPlant::add_thread_task(std::function&& task) { void PowerPlant::start() { // We are now running - is_running = true; + is_running.store(true); // Run all our Initialise scope tasks for (auto&& func : startup_tasks) { @@ -92,7 +92,7 @@ void PowerPlant::shutdown() { // Stop running before we emit the Shutdown event // Some things such as on depend on this flag and it's possible to miss it - is_running = false; + is_running.store(false); // Emit our shutdown event emit(std::make_unique()); @@ -105,6 +105,6 @@ void PowerPlant::shutdown() { } bool PowerPlant::running() const { - return is_running; + return is_running.load(); } } // namespace NUClear diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 92bc4c39a..d05e03d25 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -262,7 +262,7 @@ class PowerPlant { /// @brief Tasks that will be run during the startup process std::vector> startup_tasks; /// @brief True if the powerplant is running - volatile bool is_running = false; + std::atomic is_running{false}; }; /** diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 39dbcfbb0..07dee2b4e 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -26,7 +26,7 @@ namespace threading { void TaskScheduler::shutdown() { { std::lock_guard lock(mutex); - running = false; + running.store(false); } condition.notify_all(); } @@ -34,7 +34,7 @@ namespace threading { void TaskScheduler::submit(std::unique_ptr&& task) { // We do not accept new tasks once we are shutdown - if (running) { + if (running.load()) { /* Mutex Scope */ { std::lock_guard lock(mutex); @@ -55,7 +55,7 @@ namespace threading { while (queue.empty()) { // If the queue is empty we either wait or shutdown - if (!running) { + if (!running.load()) { // Notify any other threads that might be waiting on this condition condition.notify_all(); diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 32cc79575..73f8584da 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -103,7 +103,7 @@ namespace threading { private: /// @brief if the scheduler is running or is shut down - volatile bool running; + std::atomic running; /// @brief our queue which sorts tasks by priority std::priority_queue> queue; /// @brief the mutex which our threads synchronize their access to this object From e07753dc01a2066b527fda07f3154070b6b8a70d Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 11:51:59 +1000 Subject: [PATCH 02/87] Add sanitisation compile options --- CMakeLists.txt | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b01536f8..7e044324e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,35 @@ if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MASTER_PROJECT ON) endif() +# Enable address sanitizer +option(USE_ASAN "Enable address sanitization" OFF) +if(USE_ASAN) + add_compile_options(-fsanitize=address -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=address) + link_libraries(asan) +endif(USE_ASAN) + +option(USE_LSAN "Enable leak sanitization" OFF) +if(USE_LSAN) + add_compile_options(-fsanitize=leak -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=leak) + link_libraries(lsan) +endif(USE_LSAN) + +option(USE_TSAN "Enable thread sanitization" OFF) +if(USE_TSAN) + add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=thread) + link_libraries(tsan) +endif(USE_TSAN) + +option(USE_UBSAN "Enable undefined behaviour sanitization" OFF) +if(USE_UBSAN) + add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=undefined) + link_libraries(ubsan) +endif(USE_UBSAN) + # Add the src directory add_subdirectory(src) From fb37432932d9817bb541209ffe1349e7e79ee9a6 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 11:52:17 +1000 Subject: [PATCH 03/87] Add colourised compiler output --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e044324e..5f63c3ddd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,14 @@ if(USE_UBSAN) link_libraries(ubsan) endif(USE_UBSAN) +# Make the compiler display colours always (even when we build with ninja) +if(CMAKE_CXX_COMPILER_ID MATCHES GNU) + add_compile_options(-fdiagnostics-color=always) +endif() +if(CMAKE_CXX_COMPILER_ID MATCHES Clang) + add_compile_options(-fcolor-diagnostics) +endif() + # Add the src directory add_subdirectory(src) From 24a7848e1389a6a2c2d9626e4ad864e01b526a9f Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 11:53:49 +1000 Subject: [PATCH 04/87] Replace `volatile bool`s with `std::atomic`s --- src/dsl/word/Sync.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index 1e8f7a8d4..39d97fd7c 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -68,7 +68,7 @@ namespace dsl { /// @brief our queue which sorts tasks by priority static std::priority_queue queue; /// @brief how many tasks are currently running - static volatile bool running; + static std::atomic running; /// @brief a mutex to ensure data consistency static std::mutex mutex; @@ -80,12 +80,12 @@ namespace dsl { std::lock_guard lock(mutex); // If we are already running then queue, otherwise return and set running - if (running) { + if (running.load()) { queue.push(std::move(task)); return std::unique_ptr(nullptr); } else { - running = true; + running.store(false); return std::move(task); } } @@ -97,7 +97,7 @@ namespace dsl { std::lock_guard lock(mutex); // We are finished running - running = false; + running.store(false); // If we have another task, add it if (!queue.empty()) { @@ -115,7 +115,7 @@ namespace dsl { std::priority_queue::task_ptr> Sync::queue; template - volatile bool Sync::running = false; + std::atomic Sync::running{false}; template std::mutex Sync::mutex; From 619ee7c43536b50baea55462b21dc123dd7a9151 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 11:54:18 +1000 Subject: [PATCH 05/87] Add unit test for sync priority queue --- tests/dsl/SingleSync.cpp | 78 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/dsl/SingleSync.cpp diff --git a/tests/dsl/SingleSync.cpp b/tests/dsl/SingleSync.cpp new file mode 100644 index 000000000..20dd4efa2 --- /dev/null +++ b/tests/dsl/SingleSync.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +namespace { + +struct Message { + int val; + Message(int val) : val(val){}; +}; + +struct ShutdownOnIdle {}; + +std::atomic counter(0); + +class TestReactor : public NUClear::Reactor { +public: + TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + + on, Sync>().then([this](const Message& m) { + // Match message value against counter value + auto value = counter.load(); + + // Increment our counter + ++counter; + + CHECK(value == m.val); + }); + + on, Priority::IDLE>().then([this] { powerplant.shutdown(); }); + + on().then([this] { + emit(std::make_unique(0)); + emit(std::make_unique(1)); + emit(std::make_unique(2)); + emit(std::make_unique(3)); + emit(std::make_unique()); + }); + } +}; +} // namespace + +TEST_CASE("Testing that the Sync priority queue word works correctly", "[api][sync][priority]") { + NUClear::PowerPlant::Configuration config; + config.thread_count = 2; + NUClear::PowerPlant plant(config); + plant.install(); + + // Repeat the test to try and hit the race condition + auto i = GENERATE(Catch::Generators::range(0, 100)); + INFO("Testing iteration " << (i + 1)); + { + plant.start(); + + // Reset the counter + auto value = counter.load(); + counter.store(0); + + REQUIRE(value == 4); + } +} From 4aa971d53682bcd0cdc4d1a0064b6c5f76b57f16 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 11:57:15 +1000 Subject: [PATCH 06/87] Add missing include --- src/PowerPlant.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index d05e03d25..1a2569608 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_POWERPLANT_HPP #define NUCLEAR_POWERPLANT_HPP +#include #include #include #include From 76526bdf7fc501ce113049e7177f65cf6417b2b3 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 12:01:29 +1000 Subject: [PATCH 07/87] Add extra source annotations to improve valgrind support --- CMakeLists.txt | 5 +++++ src/PowerPlant.cpp | 29 +++++++++++++++++++++++++++++ src/PowerPlant.hpp | 17 ++++++++++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f63c3ddd..ca6dec048 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,11 @@ if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MASTER_PROJECT ON) endif() +option(USE_VALGRIND "Add extra annotations for better valgrind handling" OFF) +if(USE_VALGRIND) + add_compile_options(-DUSE_VALGRIND) +endif(USE_VALGRIND) + # Enable address sanitizer option(USE_ASAN "Enable address sanitization" OFF) if(USE_ASAN) diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index 8f2774d3b..47b5c3f4a 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -20,6 +20,35 @@ #include "threading/ThreadPoolTask.hpp" +// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 +#if defined(USE_VALGRIND) && !defined(NDEBUG) +namespace std { +extern "C" { +static void* execute_native_thread_routine(void* __p) { + thread::_State_ptr __t{static_cast(__p)}; + __t->_M_run(); + return nullptr; +} +void thread::_M_start_thread(_State_ptr state, void (*depend)()) { + // Make sure it's not optimized out, not even with LTO. + asm("" : : "rm"(depend)); + + if (!__gthread_active_p()) { +# if __cpp_exceptions + throw system_error(make_error_code(errc::operation_not_permitted), "Enable multithreading to use std::thread"); +# else + __builtin_abort(); +# endif + } + + const int err = __gthread_create(&_M_id._M_thread, &execute_native_thread_routine, state.get()); + if (err) __throw_system_error(err); + state.release(); +} +} +} // namespace std +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) + namespace NUClear { PowerPlant* PowerPlant::powerplant = nullptr; // NOLINT diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 1a2569608..a690c9ff3 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -19,6 +19,13 @@ #ifndef NUCLEAR_POWERPLANT_HPP #define NUCLEAR_POWERPLANT_HPP +// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 +#if defined(USE_VALGRIND) && !defined(NDEBUG) +# include +# define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) +# define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) + #include #include #include @@ -28,10 +35,18 @@ #include #include #include -#include #include #include +// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 +#if defined(USE_VALGRIND) && !defined(NDEBUG) +# define _GLIBCXX_THREAD_IMPL 1 +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) +#include +#if defined(USE_VALGRIND) && !defined(NDEBUG) +# undef _GLIBCXX_THREAD_IMPL +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) + // Utilities #include "LogLevel.hpp" #include "message/LogMessage.hpp" From b76b397c53a78ee003d95ee8493c1977a39af9c9 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 12:02:09 +1000 Subject: [PATCH 08/87] Add missing includes --- src/extension/network/NUClearNetwork.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/extension/network/NUClearNetwork.cpp b/src/extension/network/NUClearNetwork.cpp index 179a7f16d..64536ad32 100644 --- a/src/extension/network/NUClearNetwork.cpp +++ b/src/extension/network/NUClearNetwork.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include "../../util/network/get_interfaces.hpp" From 75a98f6a27c6c686e5162d8b97edffb4254a6961 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 12:04:27 +1000 Subject: [PATCH 09/87] Replace `volatile bool`s with `std::atomic`s --- src/extension/IOController_Posix.hpp | 8 ++++---- src/extension/IOController_Windows.hpp | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 0502cfe3d..d55b1a939 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -111,7 +111,7 @@ namespace extension { on().then("Shutdown IO Controller", [this] { // Set shutdown to true so it won't try to poll again - shutdown = true; + shutdown.store(true); // A byte to send down the pipe char val = 0; @@ -126,7 +126,7 @@ namespace extension { on().then("IO Controller", [this] { // To make sure we don't get caught in a weird loop // shutdown keeps us out here - if (!shutdown) { + if (!shutdown.load()) { // TODO(trent): check for dirty here @@ -251,8 +251,8 @@ namespace extension { fd_t notify_recv; fd_t notify_send; - bool shutdown = false; - bool dirty = true; + std::atomic shutdown{false}; + bool dirty = true; std::mutex reaction_mutex; std::vector fds; std::vector reactions; diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 1fc9ddf2f..fe877b250 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -120,7 +120,7 @@ namespace extension { on().then("Shutdown IO Controller", [this] { // Set shutdown to true - shutdown = true; + shutdown.store(true); // Signal the notifier event to return from WSAWaitForMultipleEvents() and shutdown if (!WSASetEvent(notifier)) { @@ -130,7 +130,7 @@ namespace extension { }); on().then("IO Controller", [this] { - if (!shutdown) { + if (!shutdown.load()) { // Wait for events auto event_index = WSAWaitForMultipleEvents( static_cast(events.size()), events.data(), false, WSA_INFINITE, false); @@ -233,7 +233,7 @@ namespace extension { WSAEVENT notifier; - bool shutdown = false; + std::atomic shutdown{false}; bool reactions_list_dirty = false; std::mutex reaction_mutex; From 69480c4b1f64da6a514b84530ab6961fdbd85fc0 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 12:07:58 +1000 Subject: [PATCH 10/87] Lock mutex before calling `notify_all()` --- src/extension/ChronoController.hpp | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 9c6738997..29c6a9f01 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -35,12 +35,10 @@ namespace extension { on>().then("Add Chrono task", [this](std::shared_ptr task) { // Lock the mutex while we're doing stuff - { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); - // Add our new task to the heap - tasks.push_back(*task); - } + // Add our new task to the heap + tasks.push_back(*task); // Poke the system wait.notify_all(); @@ -49,25 +47,25 @@ namespace extension { on>>().then( "Unbind Chrono Task", [this](const dsl::operation::Unbind& unbind) { // Lock the mutex while we're doing stuff - { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); - // Find the task - auto it = std::find_if( - tasks.begin(), tasks.end(), [&](const ChronoTask& task) { return task.id == unbind.id; }); + // Find the task + auto it = std::find_if( + tasks.begin(), tasks.end(), [&](const ChronoTask& task) { return task.id == unbind.id; }); - // Remove if if it exists - if (it != tasks.end()) { - tasks.erase(it); - } - } + // Remove if if it exists + if (it != tasks.end()) { tasks.erase(it); } // Poke the system to make sure it's not waiting on something that's gone wait.notify_all(); }); // When we shutdown we notify so we quit now - on().then("Shutdown Chrono Controller", [this] { wait.notify_all(); }); + on().then("Shutdown Chrono Controller", [this] { + // Lock the mutex while we're doing stuff + std::unique_lock lock(mutex); + wait.notify_all(); + }); on().then("Chrono Controller", [this] { // Acquire the mutex lock so we can wait on it From a65fe4c8c28d55ce3f20e2b87c0719bf58bdf9a0 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 12:24:43 +1000 Subject: [PATCH 11/87] Use `lock_guard` --- src/extension/ChronoController.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 29c6a9f01..850b4685a 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -63,7 +63,7 @@ namespace extension { // When we shutdown we notify so we quit now on().then("Shutdown Chrono Controller", [this] { // Lock the mutex while we're doing stuff - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); wait.notify_all(); }); From b7a590a4f6d73efbba3301e227f612ed81fef4d1 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 6 Jun 2023 12:28:09 +1000 Subject: [PATCH 12/87] Store `true` rather than `false` --- src/dsl/word/Sync.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index 39d97fd7c..220a6f650 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -85,7 +85,7 @@ namespace dsl { return std::unique_ptr(nullptr); } else { - running.store(false); + running.store(true); return std::move(task); } } From 0d6722a70be8175257fdbc125d38944b0001fcbc Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:43:30 +1000 Subject: [PATCH 13/87] Add missing includes --- src/dsl/fusion/NoOp.hpp | 2 ++ src/dsl/word/Always.hpp | 2 ++ src/dsl/word/Buffer.hpp | 9 ++++-- src/dsl/word/Every.hpp | 1 + src/dsl/word/IO.hpp | 1 + src/dsl/word/Last.hpp | 1 + src/dsl/word/MainThread.hpp | 1 + src/dsl/word/Network.hpp | 1 + src/dsl/word/Once.hpp | 1 + src/dsl/word/Optional.hpp | 1 + src/dsl/word/Sync.hpp | 63 ++++++------------------------------- src/dsl/word/TCP.hpp | 1 + src/dsl/word/UDP.hpp | 1 + src/dsl/word/Watchdog.hpp | 1 + 14 files changed, 30 insertions(+), 56 deletions(-) diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 6d8b35080..97beca9f6 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_DSL_FUSION_NOOP_HPP #define NUCLEAR_DSL_FUSION_NOOP_HPP +#include "../../threading/Reaction.hpp" +#include "../../threading/ReactionTask.hpp" #include "../word/Priority.hpp" namespace NUClear { diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index fba5525e0..e86ab6d5b 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_DSL_WORD_ALWAYS_HPP #define NUCLEAR_DSL_WORD_ALWAYS_HPP +#include "../../threading/ReactionTask.hpp" + namespace NUClear { namespace dsl { namespace word { diff --git a/src/dsl/word/Buffer.hpp b/src/dsl/word/Buffer.hpp index bc64afe21..07a402702 100644 --- a/src/dsl/word/Buffer.hpp +++ b/src/dsl/word/Buffer.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_DSL_WORD_BUFFER_HPP #define NUCLEAR_DSL_WORD_BUFFER_HPP +#include "../../threading/Reaction.hpp" + namespace NUClear { namespace dsl { namespace word { @@ -29,9 +31,10 @@ namespace dsl { * * @details * @code on, Buffer>>() @endcode - * In the case above, when the subscribing reaction is triggered, should there be less than n existing - * tasks associated with this reaction (either executing or in the queue), then a new task will be created and - * scheduled. However, should n tasks already be allocated, then this new task request will be ignored. + * In the case above, when the subscribing reaction is triggered, should there be less than n + * existing tasks associated with this reaction (either executing or in the queue), then a new task will be + * created and scheduled. However, should n tasks already be allocated, then this new task request + * will be ignored. * * For best use, this word should be fused with at least one other binding DSL word. * diff --git a/src/dsl/word/Every.hpp b/src/dsl/word/Every.hpp index cdea390fa..f670029c3 100644 --- a/src/dsl/word/Every.hpp +++ b/src/dsl/word/Every.hpp @@ -21,6 +21,7 @@ #include +#include "../../threading/Reaction.hpp" #include "../operation/ChronoTask.hpp" #include "../operation/Unbind.hpp" #include "emit/Direct.hpp" diff --git a/src/dsl/word/IO.hpp b/src/dsl/word/IO.hpp index f3c3394e1..924d1d0a8 100644 --- a/src/dsl/word/IO.hpp +++ b/src/dsl/word/IO.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_IO_HPP #define NUCLEAR_DSL_WORD_IO_HPP +#include "../../threading/Reaction.hpp" #include "../../util/platform.hpp" #include "../operation/Unbind.hpp" #include "../store/ThreadStore.hpp" diff --git a/src/dsl/word/Last.hpp b/src/dsl/word/Last.hpp index 6611abc52..1afa53eba 100644 --- a/src/dsl/word/Last.hpp +++ b/src/dsl/word/Last.hpp @@ -22,6 +22,7 @@ #include #include +#include "../../threading/Reaction.hpp" #include "../../util/MergeTransient.hpp" namespace NUClear { diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 354b936d9..44b0341a1 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_MAINTHREAD_HPP #define NUCLEAR_DSL_WORD_MAINTHREAD_HPP +#include "../../threading/ReactionTask.hpp" #include "../../util/main_thread_id.hpp" namespace NUClear { diff --git a/src/dsl/word/Network.hpp b/src/dsl/word/Network.hpp index 8a06c5119..d3f4e9753 100644 --- a/src/dsl/word/Network.hpp +++ b/src/dsl/word/Network.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_NETWORK_HPP #define NUCLEAR_DSL_WORD_NETWORK_HPP +#include "../../threading/Reaction.hpp" #include "../../util/network/sock_t.hpp" #include "../../util/serialise/Serialise.hpp" #include "../store/ThreadStore.hpp" diff --git a/src/dsl/word/Once.hpp b/src/dsl/word/Once.hpp index f557c2762..99b59a4ed 100644 --- a/src/dsl/word/Once.hpp +++ b/src/dsl/word/Once.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_ONCE_HPP #define NUCLEAR_DSL_WORD_ONCE_HPP +#include "../../threading/ReactionTask.hpp" #include "Single.hpp" namespace NUClear { diff --git a/src/dsl/word/Optional.hpp b/src/dsl/word/Optional.hpp index b27471e63..4738a1330 100644 --- a/src/dsl/word/Optional.hpp +++ b/src/dsl/word/Optional.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_OPTIONAL_HPP #define NUCLEAR_DSL_WORD_OPTIONAL_HPP +#include "../../threading/Reaction.hpp" namespace NUClear { namespace dsl { namespace word { diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index 220a6f650..b0896cc23 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -19,6 +19,14 @@ #ifndef NUCLEAR_DSL_WORD_SYNC_HPP #define NUCLEAR_DSL_WORD_SYNC_HPP +#include +#include +#include +#include +#include + +#include "../../threading/ReactionTask.hpp" + namespace NUClear { namespace dsl { namespace word { @@ -63,63 +71,12 @@ namespace dsl { template struct Sync { - using task_ptr = std::unique_ptr; - - /// @brief our queue which sorts tasks by priority - static std::priority_queue queue; - /// @brief how many tasks are currently running - static std::atomic running; - /// @brief a mutex to ensure data consistency - static std::mutex mutex; - template - static inline std::unique_ptr reschedule( - std::unique_ptr&& task) { - - // Lock our mutex - std::lock_guard lock(mutex); - - // If we are already running then queue, otherwise return and set running - if (running.load()) { - queue.push(std::move(task)); - return std::unique_ptr(nullptr); - } - else { - running.store(true); - return std::move(task); - } - } - - template - static void postcondition(threading::ReactionTask& task) { - - // Lock our mutex - std::lock_guard lock(mutex); - - // We are finished running - running.store(false); - - // If we have another task, add it - if (!queue.empty()) { - std::unique_ptr next_task( - std::move(const_cast&>(queue.top()))); - queue.pop(); - - // Resubmit this task to the reaction queue - task.parent.reactor.powerplant.submit(std::move(next_task)); - } + static inline std::type_index group(threading::ReactionTask& task) { + return std::type_index(typeid(SyncGroup)); } }; - template - std::priority_queue::task_ptr> Sync::queue; - - template - std::atomic Sync::running{false}; - - template - std::mutex Sync::mutex; - } // namespace word } // namespace dsl } // namespace NUClear diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index e4330b140..63aec41dc 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -22,6 +22,7 @@ #include #include "../../PowerPlant.hpp" +#include "../../threading/Reaction.hpp" #include "../../util/FileDescriptor.hpp" #include "../../util/platform.hpp" #include "IO.hpp" diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index d2649fff5..ab3da723f 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -20,6 +20,7 @@ #define NUCLEAR_DSL_WORD_UDP_HPP #include "../../PowerPlant.hpp" +#include "../../threading/Reaction.hpp" #include "../../util/FileDescriptor.hpp" #include "../../util/network/get_interfaces.hpp" #include "../../util/platform.hpp" diff --git a/src/dsl/word/Watchdog.hpp b/src/dsl/word/Watchdog.hpp index 66047fb85..c01c21cc5 100644 --- a/src/dsl/word/Watchdog.hpp +++ b/src/dsl/word/Watchdog.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_WATCHDOG_HPP #define NUCLEAR_DSL_WORD_WATCHDOG_HPP +#include "../../threading/Reaction.hpp" #include "../../util/demangle.hpp" #include "../operation/Unbind.hpp" #include "../store/DataStore.hpp" From 2b90f12a021a5365855e39e81ef779a239737e7b Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:45:09 +1000 Subject: [PATCH 14/87] Add a Group fusion word --- src/dsl/Fusion.hpp | 2 + src/dsl/Parse.hpp | 5 ++ src/dsl/fusion/GroupFusion.hpp | 98 ++++++++++++++++++++++++++++++++++ src/dsl/fusion/NoOp.hpp | 9 ++++ src/dsl/fusion/has_group.hpp | 53 ++++++++++++++++++ 5 files changed, 167 insertions(+) create mode 100644 src/dsl/fusion/GroupFusion.hpp create mode 100644 src/dsl/fusion/has_group.hpp diff --git a/src/dsl/Fusion.hpp b/src/dsl/Fusion.hpp index 19c993673..e98e1805c 100644 --- a/src/dsl/Fusion.hpp +++ b/src/dsl/Fusion.hpp @@ -22,6 +22,7 @@ #include "../threading/ReactionHandle.hpp" #include "fusion/BindFusion.hpp" #include "fusion/GetFusion.hpp" +#include "fusion/GroupFusion.hpp" #include "fusion/PostconditionFusion.hpp" #include "fusion/PreconditionFusion.hpp" #include "fusion/PriorityFusion.hpp" @@ -37,6 +38,7 @@ namespace dsl { , public fusion::GetFusion , public fusion::PreconditionFusion , public fusion::PriorityFusion + , public fusion::GroupFusion , public fusion::RescheduleFusion , public fusion::PostconditionFusion {}; diff --git a/src/dsl/Parse.hpp b/src/dsl/Parse.hpp index 259a4e031..d1a5cf085 100644 --- a/src/dsl/Parse.hpp +++ b/src/dsl/Parse.hpp @@ -53,6 +53,11 @@ namespace dsl { Parse>(r); } + static inline std::type_index group(threading::Reaction& r) { + return std::conditional_t::value, DSL, fusion::NoOp>::template group< + Parse>(r); + } + static std::unique_ptr reschedule(std::unique_ptr&& task) { return std::conditional_t::value, DSL, fusion::NoOp>::template reschedule( std::move(task)); diff --git a/src/dsl/fusion/GroupFusion.hpp b/src/dsl/fusion/GroupFusion.hpp new file mode 100644 index 000000000..9ea8215a3 --- /dev/null +++ b/src/dsl/fusion/GroupFusion.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_FUSION_GROUPFUSION_HPP +#define NUCLEAR_DSL_FUSION_GROUPFUSION_HPP + +#include + +#include "../../threading/Reaction.hpp" +#include "../operation/DSLProxy.hpp" +#include "has_group.hpp" + +namespace NUClear { +namespace dsl { + namespace fusion { + + /// Type that redirects types without a group function to their proxy type + template + struct Group { + using type = std::conditional_t::value, Word, operation::DSLProxy>; + }; + + template > + struct GroupWords; + + /** + * @brief Metafunction that extracts all of the Words with a group function + * + * @tparam Word1 The word we are looking at + * @tparam WordN The words we have yet to look at + * @tparam FoundWords The words we have found with group functions + */ + template + struct GroupWords, std::tuple> + : public std::conditional_t< + has_group::type>::value, + /*T*/ GroupWords, std::tuple::type>>, + /*F*/ GroupWords, std::tuple>> {}; + + /** + * @brief Termination case for the GroupWords metafunction + * + * @tparam FoundWords The words we have found with group functions + */ + template + struct GroupWords, std::tuple> { + using type = std::tuple; + }; + + + // Default case where there are no group words + template + struct GroupFuser {}; + + // Case where there is only a single word remaining + template + struct GroupFuser> { + + template + static inline std::type_index group(threading::Reaction& reaction) { + + // Return our group + return Word::template group(reaction); + } + }; + + // Case where there are 2 or more words remaining + template + struct GroupFuser> { + + template + static inline void group(threading::Reaction& reaction) { + throw std::runtime_error("Can not be a member of more than one group"); + } + }; + + template + struct GroupFusion : public GroupFuser>::type> {}; + + } // namespace fusion +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_FUSION_GROUPFUSION_HPP diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 97beca9f6..42163b8d8 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_DSL_FUSION_NOOP_HPP #define NUCLEAR_DSL_FUSION_NOOP_HPP +#include + #include "../../threading/Reaction.hpp" #include "../../threading/ReactionTask.hpp" #include "../word/Priority.hpp" @@ -51,6 +53,11 @@ namespace dsl { return word::Priority::NORMAL::value; } + template + static inline std::type_index group(threading::Reaction&) { + return std::type_index(typeid(threading::TaskScheduler)); + } + template static inline std::unique_ptr reschedule( std::unique_ptr&& task) { @@ -76,6 +83,8 @@ namespace dsl { static inline int priority(threading::Reaction&); + static inline std::type_index group(threading::Reaction&); + static inline std::unique_ptr reschedule( std::unique_ptr&& task); diff --git a/src/dsl/fusion/has_group.hpp b/src/dsl/fusion/has_group.hpp new file mode 100644 index 000000000..8a4e0970a --- /dev/null +++ b/src/dsl/fusion/has_group.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_FUSION_HAS_GROUP_HPP +#define NUCLEAR_DSL_FUSION_HAS_GROUP_HPP + +#include "../../threading/Reaction.hpp" +#include "NoOp.hpp" + +namespace NUClear { +namespace dsl { + namespace fusion { + + /** + * @brief SFINAE struct to test if the passed class has a group function that conforms to the NUClear DSL + * + * @tparam T the class to check + */ + template + struct has_group { + private: + typedef std::true_type yes; + typedef std::false_type no; + + template + static auto test(int) + -> decltype(U::template group(std::declval()), yes()); + template + static no test(...); + + public: + static constexpr bool value = std::is_same(0)), yes>::value; + }; + + } // namespace fusion +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_FUSION_HAS_GROUP_HPP From 8e1560765214600bd2f4d5a4f84df7e0b25501b0 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:46:16 +1000 Subject: [PATCH 15/87] Add a Pool fusion word --- src/dsl/Fusion.hpp | 2 + src/dsl/Parse.hpp | 5 ++ src/dsl/fusion/NoOp.hpp | 8 +++ src/dsl/fusion/PoolFusion.hpp | 98 +++++++++++++++++++++++++++++++++++ src/dsl/fusion/has_pool.hpp | 53 +++++++++++++++++++ src/util/thread_pool.hpp | 38 ++++++++++++++ 6 files changed, 204 insertions(+) create mode 100644 src/dsl/fusion/PoolFusion.hpp create mode 100644 src/dsl/fusion/has_pool.hpp create mode 100644 src/util/thread_pool.hpp diff --git a/src/dsl/Fusion.hpp b/src/dsl/Fusion.hpp index e98e1805c..ba80d2572 100644 --- a/src/dsl/Fusion.hpp +++ b/src/dsl/Fusion.hpp @@ -23,6 +23,7 @@ #include "fusion/BindFusion.hpp" #include "fusion/GetFusion.hpp" #include "fusion/GroupFusion.hpp" +#include "fusion/PoolFusion.hpp" #include "fusion/PostconditionFusion.hpp" #include "fusion/PreconditionFusion.hpp" #include "fusion/PriorityFusion.hpp" @@ -39,6 +40,7 @@ namespace dsl { , public fusion::PreconditionFusion , public fusion::PriorityFusion , public fusion::GroupFusion + , public fusion::PoolFusion , public fusion::RescheduleFusion , public fusion::PostconditionFusion {}; diff --git a/src/dsl/Parse.hpp b/src/dsl/Parse.hpp index d1a5cf085..9cec39c36 100644 --- a/src/dsl/Parse.hpp +++ b/src/dsl/Parse.hpp @@ -58,6 +58,11 @@ namespace dsl { Parse>(r); } + static inline util::ThreadPoolDescriptor pool(threading::Reaction& r) { + return std::conditional_t::value, DSL, fusion::NoOp>::template pool< + Parse>(r); + } + static std::unique_ptr reschedule(std::unique_ptr&& task) { return std::conditional_t::value, DSL, fusion::NoOp>::template reschedule( std::move(task)); diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 42163b8d8..4bbe7aaea 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -23,6 +23,7 @@ #include "../../threading/Reaction.hpp" #include "../../threading/ReactionTask.hpp" +#include "../../util/thread_pool.hpp" #include "../word/Priority.hpp" namespace NUClear { @@ -58,6 +59,11 @@ namespace dsl { return std::type_index(typeid(threading::TaskScheduler)); } + template + static inline util::ThreadPoolDescriptor pool(threading::Reaction&) { + return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; + } + template static inline std::unique_ptr reschedule( std::unique_ptr&& task) { @@ -85,6 +91,8 @@ namespace dsl { static inline std::type_index group(threading::Reaction&); + static inline util::ThreadPoolDescriptor pool(threading::Reaction&); + static inline std::unique_ptr reschedule( std::unique_ptr&& task); diff --git a/src/dsl/fusion/PoolFusion.hpp b/src/dsl/fusion/PoolFusion.hpp new file mode 100644 index 000000000..08316da51 --- /dev/null +++ b/src/dsl/fusion/PoolFusion.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_FUSION_POOLFUSION_HPP +#define NUCLEAR_DSL_FUSION_POOLFUSION_HPP + +#include + +#include "../../threading/Reaction.hpp" +#include "../operation/DSLProxy.hpp" +#include "has_pool.hpp" + +namespace NUClear { +namespace dsl { + namespace fusion { + + /// Type that redirects types without a pool function to their proxy type + template + struct Pool { + using type = std::conditional_t::value, Word, operation::DSLProxy>; + }; + + template > + struct PoolWords; + + /** + * @brief Metafunction that extracts all of the Words with a pool function + * + * @tparam Word1 The word we are looking at + * @tparam WordN The words we have yet to look at + * @tparam FoundWords The words we have found with pool functions + */ + template + struct PoolWords, std::tuple> + : public std::conditional_t< + has_pool::type>::value, + /*T*/ PoolWords, std::tuple::type>>, + /*F*/ PoolWords, std::tuple>> {}; + + /** + * @brief Termination case for the PoolWords metafunction + * + * @tparam FoundWords The words we have found with pool functions + */ + template + struct PoolWords, std::tuple> { + using type = std::tuple; + }; + + + // Default case where there are no pool words + template + struct PoolFuser {}; + + // Case where there is only a single word remaining + template + struct PoolFuser> { + + template + static inline util::ThreadPoolDescriptor pool(threading::Reaction& reaction) { + + // Return our pool + return Word::template pool(reaction); + } + }; + + // Case where there are 2 or more words remaining + template + struct PoolFuser> { + + template + static inline util::ThreadPoolDescriptor pool(threading::Reaction& reaction) { + throw std::runtime_error("Can not be a member of more than one pool"); + } + }; + + template + struct PoolFusion : public PoolFuser>::type> {}; + + } // namespace fusion +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_FUSION_POOLFUSION_HPP diff --git a/src/dsl/fusion/has_pool.hpp b/src/dsl/fusion/has_pool.hpp new file mode 100644 index 000000000..a3a1f4366 --- /dev/null +++ b/src/dsl/fusion/has_pool.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_FUSION_HAS_POOL_HPP +#define NUCLEAR_DSL_FUSION_HAS_POOL_HPP + +#include "../../threading/Reaction.hpp" +#include "NoOp.hpp" + +namespace NUClear { +namespace dsl { + namespace fusion { + + /** + * @brief SFINAE struct to test if the passed class has a pool function that conforms to the NUClear DSL + * + * @tparam T the class to check + */ + template + struct has_pool { + private: + typedef std::true_type yes; + typedef std::false_type no; + + template + static auto test(int) + -> decltype(U::template pool(std::declval()), yes()); + template + static no test(...); + + public: + static constexpr bool value = std::is_same(0)), yes>::value; + }; + + } // namespace fusion +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_FUSION_HAS_POOL_HPP diff --git a/src/util/thread_pool.hpp b/src/util/thread_pool.hpp new file mode 100644 index 000000000..e9edade5f --- /dev/null +++ b/src/util/thread_pool.hpp @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_UTIL_THREADPOOL_HPP +#define NUCLEAR_UTIL_THREADPOOL_HPP + +#include +#include + +namespace NUClear { +namespace util { + + struct ThreadPoolDescriptor { + /// @brief Set a unique identifier for this pool + uint64_t pool_id; + + /// @brief The number of threads this thread pool will use. + size_t thread_count; + }; + +} // namespace util +} // namespace NUClear + +#endif // NUCLEAR_UTIL_THREADPOOL_HPP From f3e69e32dafe2b3cfae06696f9c2d52545b59bd7 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:48:06 +1000 Subject: [PATCH 16/87] Update CallbackGenerator to use `pool` and `group` --- src/threading/Reaction.hpp | 15 +++--- src/threading/ReactionTask.hpp | 11 +++- src/util/CallbackGenerator.hpp | 91 ++++++++++++++++++---------------- src/util/GeneratedCallback.hpp | 49 ++++++++++++++++++ 4 files changed, 116 insertions(+), 50 deletions(-) create mode 100644 src/util/GeneratedCallback.hpp diff --git a/src/threading/Reaction.hpp b/src/threading/Reaction.hpp index 7057c54ce..34b160080 100644 --- a/src/threading/Reaction.hpp +++ b/src/threading/Reaction.hpp @@ -25,6 +25,7 @@ #include #include +#include "../util/GeneratedCallback.hpp" #include "ReactionTask.hpp" namespace NUClear { @@ -50,7 +51,7 @@ namespace threading { public: // The type of the generator that is used to create functions for ReactionTask objects - using TaskGenerator = std::function(Reaction&)>; + using TaskGenerator = std::function; /** * @brief Constructs a new Reaction with the passed callback generator and options @@ -74,13 +75,15 @@ namespace threading { } // Run our generator to get a functor we can run - int priority; - std::function(std::unique_ptr &&)> func; - std::tie(priority, func) = generator(*this); + auto callback = generator(*this); // If our generator returns a valid function - if (func) { - return std::make_unique(*this, priority, std::move(func)); + if (callback) { + return std::make_unique(*this, + callback.priority, + callback.group, + callback.pool, + std::move(callback.callback)); } // Otherwise we return a null pointer diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 56b1ac7d6..a028dc257 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -27,6 +27,7 @@ #include "../message/ReactionStatistics.hpp" #include "../util/platform.hpp" +#include "../util/thread_pool.hpp" namespace NUClear { namespace threading { @@ -69,7 +70,11 @@ namespace threading { * @param priority the priority to use when executing this task. * @param callback the data bound callback to be executed in the threadpool. */ - Task(ReactionType& parent, int priority, TaskFunction&& callback) + Task(ReactionType& parent, + const int& priority, + const std::type_index& group_id, + const util::ThreadPoolDescriptor& thread_pool_descriptor, + TaskFunction&& callback) : parent(parent) , id(++task_id_source) , priority(priority) @@ -83,6 +88,8 @@ namespace threading { clock::time_point(std::chrono::seconds(0)), nullptr)) , emit_stats(parent.emit_stats && (current_task != nullptr ? current_task->emit_stats : true)) + , group_id(group_id) + , thread_pool_descriptor(thread_pool_descriptor) , callback(callback) {} @@ -121,6 +128,8 @@ namespace threading { /// reaction statistics becomes false for all created tasks. This is to stop infinite loops of death. bool emit_stats; + std::type_index group_id; + util::ThreadPoolDescriptor thread_pool_descriptor; /// @brief the data bound callback to be executed /// @attention note this must be last in the list as the this pointer is passed to the callback generator TaskFunction callback; diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 88dbc136b..c57b04e88 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -21,6 +21,7 @@ #include "../dsl/trait/is_transient.hpp" #include "../dsl/word/emit/Direct.hpp" +#include "../util/GeneratedCallback.hpp" #include "../util/MergeTransient.hpp" #include "../util/TransientDataElements.hpp" #include "../util/apply.hpp" @@ -58,7 +59,7 @@ namespace util { } - std::pair operator()(threading::Reaction& r) { + GeneratedCallback operator()(threading::Reaction& r) { // Add one to our active tasks ++r.active_tasks; @@ -69,7 +70,7 @@ namespace util { --r.active_tasks; // We cancel our execution by returning an empty function - return std::make_pair(0, threading::ReactionTask::TaskFunction()); + return {}; } else { @@ -87,53 +88,57 @@ namespace util { --r.active_tasks; // We cancel our execution by returning an empty function - return std::make_pair(0, threading::ReactionTask::TaskFunction()); + return {}; } // We have to make a copy of the callback because the "this" variable can go out of scope auto c = callback; - return std::make_pair(DSL::priority(r), [c, data](std::unique_ptr&& task) { - // Check if we are going to reschedule - task = DSL::reschedule(std::move(task)); - - // If we still control our task - if (task) { - - // Update our thread's priority to the correct level - update_current_thread_priority(task->priority); - - // Record our start time - task->stats->started = clock::now(); - - // We have to catch any exceptions - try { - // We call with only the relevant arguments to the passed function - util::apply_relevant(c, std::move(data)); - } - catch (...) { - - // Catch our exception if it happens - task->stats->exception = std::current_exception(); - } - - // Our finish time - task->stats->finished = clock::now(); - - // Run our postconditions - DSL::postcondition(*task); - - // Take one from our active tasks - --task->parent.active_tasks; - - // Emit our reaction statistics if it wouldn't cause a loop - if (task->emit_stats) { - PowerPlant::powerplant->emit_shared(task->stats); + return GeneratedCallback( + DSL::priority(r), + DSL::group(r), + DSL::pool(r), + [c, data](std::unique_ptr&& task) { + // Check if we are going to reschedule + task = DSL::reschedule(std::move(task)); + + // If we still control our task + if (task) { + + // Update our thread's priority to the correct level + update_current_thread_priority(task->priority); + + // Record our start time + task->stats->started = clock::now(); + + // We have to catch any exceptions + try { + // We call with only the relevant arguments to the passed function + util::apply_relevant(c, std::move(data)); + } + catch (...) { + + // Catch our exception if it happens + task->stats->exception = std::current_exception(); + } + + // Our finish time + task->stats->finished = clock::now(); + + // Run our postconditions + DSL::postcondition(*task); + + // Take one from our active tasks + --task->parent.active_tasks; + + // Emit our reaction statistics if it wouldn't cause a loop + if (task->emit_stats) { + PowerPlant::powerplant->emit_shared(task->stats); + } } - } - // Return our task - return std::move(task); - }); + // Return our task + return std::move(task); + }); } } diff --git a/src/util/GeneratedCallback.hpp b/src/util/GeneratedCallback.hpp new file mode 100644 index 000000000..c72176621 --- /dev/null +++ b/src/util/GeneratedCallback.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_UTIL_GENERATEDCALLBACK_HPP +#define NUCLEAR_UTIL_GENERATEDCALLBACK_HPP + +#include + +#include "../threading/ReactionTask.hpp" +#include "thread_pool.hpp" + +namespace NUClear { +namespace util { + + struct GeneratedCallback { + GeneratedCallback() = default; + GeneratedCallback(const int& priority, + const std::type_index& group, + const ThreadPoolDescriptor& pool, + threading::ReactionTask::TaskFunction callback) + : priority(priority), group(group), pool(pool), callback(std::move(callback)) {} + int priority{0}; + std::type_index group{typeid(GeneratedCallback)}; + ThreadPoolDescriptor pool{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; + threading::ReactionTask::TaskFunction callback{}; + + operator bool() const { + return bool(callback); + } + }; + +} // namespace util +} // namespace NUClear + +#endif // NUCLEAR_UTIL_GENERATEDCALLBACK_HPP From 0d4fd91ec7c3e7e8ca9577d53927e0185a22d55c Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:50:13 +1000 Subject: [PATCH 17/87] Rewrite TaskScheduler to handle everything --- src/PowerPlant.cpp | 55 +---------- src/PowerPlant.hpp | 30 +----- src/threading/TaskScheduler.cpp | 156 +++++++++++++++++++++++++------ src/threading/TaskScheduler.hpp | 66 ++++++++----- src/threading/ThreadPoolTask.hpp | 50 ---------- src/util/thread_pool.cpp | 9 ++ src/util/thread_pool.hpp | 11 +++ 7 files changed, 198 insertions(+), 179 deletions(-) delete mode 100644 src/threading/ThreadPoolTask.hpp create mode 100644 src/util/thread_pool.cpp diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index a77ba9585..60b497e20 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -18,8 +18,6 @@ #include "PowerPlant.hpp" -#include "threading/ThreadPoolTask.hpp" - // See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 #if defined(USE_VALGRIND) && !defined(NDEBUG) namespace std { @@ -60,68 +58,22 @@ PowerPlant::~PowerPlant() { powerplant = nullptr; } -void PowerPlant::on_startup(std::function&& func) { - if (is_running) { - throw std::runtime_error("Unable to do on_startup as the PowerPlant has already started"); - } - else { - startup_tasks.push_back(func); - } -} - -void PowerPlant::add_thread_task(std::function&& task) { - tasks.push_back(task); -} - void PowerPlant::start() { // We are now running is_running.store(true); - // Run all our Initialise scope tasks - for (auto&& func : startup_tasks) { - func(); - } - startup_tasks.clear(); - // Direct emit startup event emit(std::make_unique()); - // Start all our threads - for (size_t i = 0; i < configuration.thread_count; ++i) { - tasks.push_back(threading::make_thread_pool_task(scheduler)); - } - - // Start all our tasks - for (auto& task : tasks) { - threads.push_back(std::make_unique(task)); - } - - // Start our main thread using our main task scheduler - threading::make_thread_pool_task(main_thread_scheduler)(); - - // Now wait for all the threads to finish executing - for (auto& thread : threads) { - try { - if (thread->joinable()) { - thread->join(); - } - } - // This gets thrown some time if between checking if joinable and joining - // the thread is no longer joinable - catch (const std::system_error&) { - } - } + // Start all of the threads + scheduler.start(configuration.thread_count); } void PowerPlant::submit(std::unique_ptr&& task) { scheduler.submit(std::move(task)); } -void PowerPlant::submit_main(std::unique_ptr&& task) { - main_thread_scheduler.submit(std::move(task)); -} - void PowerPlant::shutdown() { // Stop running before we emit the Shutdown event @@ -133,9 +85,6 @@ void PowerPlant::shutdown() { // Shutdown the scheduler scheduler.shutdown(); - - // Shutdown the main threads scheduler - main_thread_scheduler.shutdown(); } bool PowerPlant::running() const { diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 8d8d545a9..c66370576 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -50,6 +50,7 @@ // Utilities #include "LogLevel.hpp" #include "message/LogMessage.hpp" +#include "threading/ReactionTask.hpp" #include "threading/TaskScheduler.hpp" #include "util/FunctionFusion.hpp" #include "util/demangle.hpp" @@ -139,22 +140,6 @@ class PowerPlant { */ bool running() const; - /** - * @brief Adds a function to the set of startup tasks. - * - * @param func the task being added to the set of startup tasks. - * - * @throws std::runtime_error if the PowerPlant is already running/has already started. - */ - void on_startup(std::function&& func); - - /** - * @brief Adds a function to the set of tasks to be run when the PowerPlant starts up - * - * @param task The function to add to the task list - */ - void add_thread_task(std::function&& task); - /** * @brief Installs a reactor of a particular type to the system. * @@ -176,13 +161,6 @@ class PowerPlant { */ void submit(std::unique_ptr&& task); - /** - * @brief Submits a new task to the main threads thread pool to be queued and then executed. - * - * @param task The Reaction task to be executed in the thread pool - */ - void submit_main(std::unique_ptr&& task); - /** * @brief Log a message through NUClear's system. * @@ -267,16 +245,10 @@ class PowerPlant { private: /// @brief A list of tasks that must be run when the powerplant starts up std::vector> tasks; - /// @brief A vector of the running threads in the system - std::vector> threads; /// @brief Our TaskScheduler that handles distributing task to the pool threads threading::TaskScheduler scheduler; - /// @brief Our TaskScheduler that handles distributing tasks to the main thread - threading::TaskScheduler main_thread_scheduler; /// @brief Our vector of Reactors, will get destructed when this vector is std::vector> reactors; - /// @brief Tasks that will be run during the startup process - std::vector> startup_tasks; /// @brief True if the powerplant is running std::atomic is_running{false}; }; diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 07dee2b4e..4f79a7224 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -18,16 +18,105 @@ #include "TaskScheduler.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include "../dsl/word/MainThread.hpp" +#include "../util/update_current_thread_priority.hpp" + namespace NUClear { namespace threading { - TaskScheduler::TaskScheduler() : running(true) {} + bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id) { + return task->thread_pool_descriptor.pool_id == pool_id; + } - void TaskScheduler::shutdown() { - { - std::lock_guard lock(mutex); - running.store(false); + TaskScheduler::TaskScheduler() : running(true) { + pool_map[std::this_thread::get_id()] = util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID; + } + + void TaskScheduler::create_pool(const util::ThreadPoolDescriptor& pool) { + + // Pool already exists + if (pools.count(pool.pool_id) > 0 && pools.at(pool.pool_id).thread_count > 0) { + return; + } + + // Make a copy of the pool descriptor + pools.insert({pool.pool_id, util::ThreadPoolDescriptor{pool.pool_id, pool.thread_count}}); + + auto p = pool; + auto pool_func = [this, p] { + // Wait at a high (but not realtime) priority to reduce latency + // for picking up a new task + update_current_thread_priority(1000); + + while (running.load() || !queue.empty()) { + auto task = get_task(p.pool_id); + + if (task) { + task = task->run(std::move(task)); + if (task && task->keep_alive) { + auto new_task = task->parent.get_task(); + submit(std::move(new_task)); + } + } + + // Back up to realtime while waiting + update_current_thread_priority(1000); + } + }; + + // Start all our threads + /* mutex scope */ { + std::lock_guard lock(threads_mutex); + for (size_t i = 0; i < pool.thread_count; ++i) { + threads.push_back(std::make_unique(pool_func)); + pool_map[threads.back()->get_id()] = pool.pool_id; + } } + } + + void TaskScheduler::start(const size_t& thread_count) { + + // Make the default pool + create_pool(util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, thread_count}); + + // Run main thread tasks + while (running.load()) { + auto task = get_task(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID); + + if (task) { + task = task->run(std::move(task)); + if (task && task->keep_alive) { + auto new_task = task->parent.get_task(); + submit(std::move(new_task)); + } + } + } + + // Now wait for all the threads to finish executing + for (auto& thread : threads) { + try { + if (thread->joinable()) { + thread->join(); + } + } + // This gets thrown some time if between checking if joinable and joining + // the thread is no longer joinable + catch (const std::system_error&) { + } + } + } + + void TaskScheduler::shutdown() { + std::lock_guard lock(mutex); + running.store(false); condition.notify_all(); } @@ -36,46 +125,61 @@ namespace threading { // We do not accept new tasks once we are shutdown if (running.load()) { + // Make sure the pool is created + create_pool(task->thread_pool_descriptor); + + // Check to see if this task was the result of `emit` + if (task->immediate) { + if (is_runnable(task, pool_map.at(std::this_thread::get_id())) + || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID)) { + task = task->run(std::move(task)); + return; + } + // Not runnable, stick it in the queue like nothing happened + } + /* Mutex Scope */ { std::lock_guard lock(mutex); - queue.push(std::forward>(task)); + + // Find where to insert the new task to maintain task order + auto it = + std::lower_bound(queue.begin(), queue.end(), task, std::less>()); + + // Insert before the found position + queue.insert(it, std::forward>(task)); } } // Notify a thread that it can proceed - condition.notify_one(); + std::lock_guard lock(mutex); + condition.notify_all(); } - std::unique_ptr TaskScheduler::get_task() { + std::unique_ptr TaskScheduler::get_task(const uint64_t& pool_id) { - // Obtain the lock - std::unique_lock lock(mutex); + while (running.load() || !queue.empty()) { - // While our queue is empty - while (queue.empty()) { + std::unique_lock lock(mutex); - // If the queue is empty we either wait or shutdown - if (!running.load()) { + for (auto it = queue.begin(); it != queue.end(); ++it) { + // Check if we can run it + if (is_runnable(*it, pool_id)) { + // Move the task out of the queue + std::unique_ptr task = std::move(*it); - // Notify any other threads that might be waiting on this condition - condition.notify_all(); + // Erase the old position in the queue + queue.erase(it); - // Return a nullptr to signify there is nothing on the queue - return nullptr; + // Return the task + return task; + } } // Wait for something to happen! condition.wait(lock); } - // Return the type - // If you're wondering why all the ridiculousness, it's because priority queue is not as feature complete as it - // should be its 'top' method returns a const reference (which we can't use to move a unique pointer) - std::unique_ptr task( - std::move(const_cast&>(queue.top()))); // NOLINT - queue.pop(); - - return task; + return nullptr; } } // namespace threading } // namespace NUClear diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 73f8584da..00b1d93c0 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -19,49 +19,59 @@ #ifndef NUCLEAR_THREADING_TASKSCHEDULER_HPP #define NUCLEAR_THREADING_TASKSCHEDULER_HPP -#include #include #include #include #include #include -#include -#include +#include #include +#include "../util/thread_pool.hpp" #include "Reaction.hpp" +#include "ReactionTask.hpp" namespace NUClear { namespace threading { /** - * @brief This class is responsible for scheduling tasks and distributing them amoungst threads. + * @brief This class is responsible for scheduling tasks and distributing them amongst threads. * * @details - * This task scheduler uses the options from each of the tasks to decide when to execute them in a thread. The - * rules - * are applied to the tasks in the following order. + * PRIORITY + * what priority this task should run with + * tasks are ordered by priority -> creation order + * POOL + * which thread pool this task should execute in + * 0 being execute on the main thread + * 1 being the default pool + * Work out how to create other pools later (fold in always into this?) + * GROUP + * which grouping this task belongs to for concurrency (default to the 0 group) + * CONCURRENCY + * only run if there are less than this many tasks running in this group + * INLINE + * if the submitter of this task should wait until this task is finished before returning (for DIRECT + * emits) * * @em Priority * @code Priority

@endcode - * When a priority is encountered, the task will be scheduled to execute based on this. If one of the three normal - * options are specified (HIGH, DEFAULT and LOW), then within the specified Sync group, it will run before, - * normally - * or after other reactions. + * When a priority is encountered, the task will be scheduled to execute based on this. If one of the three + * normal options are specified (HIGH, DEFAULT and LOW), then within the specified Sync group, it will run + * before, normally or after other reactions. * @attention Note that if Priority is specified, the Sync type is ignored (Single is not). * * @em Sync * @code Sync @endcode - * When a Sync type is encounterd, the system uses this as a compile time mutex flag. It will not allow two - * callbacks - * with the same Sync type to execute at the same time. It will effectivly ensure that all of the callbacks with - * this type run in sequence with eachother, rather then in parallell. It is also important to note again, that if - * the priority of a task is realtime, it will ignore Sync groups. + * When a Sync type is encountered, the system uses this as a compile time mutex flag. It will not allow two + * callbacks with the same Sync type to execute at the same time. It will effectively ensure that all of the + * callbacks with this type run in sequence with each other, rather then in parallel. It is also important to + * note again, that if the priority of a task is realtime, it will ignore Sync groups. * * @em Single * @code Single @endcode - * If single is encountered while processing the function, and a Task object for this Reaction is already running - * in a thread, or waiting in the Queue, then this task is ignored and dropped from the system. + * If single is encountered while processing the function, and a Task object for this Reaction is already + * running in a thread, or waiting in the Queue, then this task is ignored and dropped from the system. */ class TaskScheduler { public: @@ -70,6 +80,8 @@ namespace threading { */ TaskScheduler(); + void start(const size_t& thread_count); + /** * @brief * Shuts down the scheduler, all waiting threads are woken, and any attempt to get a task results in an @@ -94,22 +106,34 @@ namespace threading { * * @details * This method will get a task object to be executed from the queue. It will block until such a time as a - * task is available to be executed. For example, if a task with a paticular sync type was out, then this + * task is available to be executed. For example, if a task with a particular sync type was out, then this * thread would block until that sync type was no longer out, and then it would take a task. * * @return the task which has been given to be executed */ - std::unique_ptr get_task(); + std::unique_ptr get_task(const uint64_t& pool_id); private: + void create_pool(const util::ThreadPoolDescriptor& pool); + /// @brief if the scheduler is running or is shut down std::atomic running; + /// @brief our queue which sorts tasks by priority - std::priority_queue> queue; + std::vector> queue; + /// @brief the mutex which our threads synchronize their access to this object std::mutex mutex; /// @brief the condition object that threads wait on if they can't get a task std::condition_variable condition; + + /// @brief A vector of the running threads in the system + std::vector> threads; + /// @brief the mutex which our threads synchronize their access to this object + std::mutex threads_mutex; + + std::map pools{}; + std::map pool_map{}; }; } // namespace threading diff --git a/src/threading/ThreadPoolTask.hpp b/src/threading/ThreadPoolTask.hpp deleted file mode 100644 index 773d266b9..000000000 --- a/src/threading/ThreadPoolTask.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_THREADING_THREADPOOLTASK_HPP -#define NUCLEAR_THREADING_THREADPOOLTASK_HPP - -#include "../PowerPlant.hpp" -#include "../util/update_current_thread_priority.hpp" -#include "TaskScheduler.hpp" - -namespace NUClear { -namespace threading { - - inline std::function make_thread_pool_task(TaskScheduler& scheduler) { - return [&scheduler] { - // Wait at a high (but not realtime) priority to reduce latency - // for picking up a new task - update_current_thread_priority(1000); - - // Run while our scheduler gives us tasks - for (std::unique_ptr task(scheduler.get_task()); task; task = scheduler.get_task()) { - - // Run the task - task = task->run(std::move(task)); - - // Back up to realtime while waiting - update_current_thread_priority(1000); - } - }; - } - -} // namespace threading -} // namespace NUClear - -#endif // NUCLEAR_THREADING_THREADPOOLTASK_HPP diff --git a/src/util/thread_pool.cpp b/src/util/thread_pool.cpp new file mode 100644 index 000000000..ec19e9da0 --- /dev/null +++ b/src/util/thread_pool.cpp @@ -0,0 +1,9 @@ +#include "thread_pool.hpp" + +namespace NUClear { +namespace util { + + std::atomic ThreadPoolIDSource::source = {2}; + +} // namespace util +} // namespace NUClear diff --git a/src/util/thread_pool.hpp b/src/util/thread_pool.hpp index e9edade5f..55954cc7f 100644 --- a/src/util/thread_pool.hpp +++ b/src/util/thread_pool.hpp @@ -18,6 +18,7 @@ #ifndef NUCLEAR_UTIL_THREADPOOL_HPP #define NUCLEAR_UTIL_THREADPOOL_HPP +#include #include #include @@ -32,6 +33,16 @@ namespace util { size_t thread_count; }; + struct ThreadPoolIDSource { + static std::atomic source; + static constexpr uint64_t MAIN_THREAD_POOL_ID = 0; + static constexpr uint64_t DEFAULT_THREAD_POOL_ID = 1; + + static uint64_t get_new_pool_id() { + return source++; + } + }; + } // namespace util } // namespace NUClear From 6243f0029952cbcfb6b8fd41cbcd2cd506cdae29 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:50:44 +1000 Subject: [PATCH 18/87] Add missing `Pool` word to `Reactor` --- src/Reactor.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Reactor.hpp b/src/Reactor.hpp index 5805eaacc..28de5fe88 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -94,6 +94,9 @@ namespace dsl { template struct Sync; + template + struct Pool; + namespace emit { template struct Local; @@ -289,7 +292,7 @@ class Reactor { template auto then(const std::string& label, Function&& callback, const util::Sequence&) { - // Generate the identifer + // Generate the identifier std::vector identifier = {label, reactor.reactor_name, util::demangle(typeid(DSL).name()), @@ -412,6 +415,7 @@ class Reactor { #include "dsl/word/Network.hpp" #include "dsl/word/Once.hpp" #include "dsl/word/Optional.hpp" +#include "dsl/word/Pool.hpp" #include "dsl/word/Priority.hpp" #include "dsl/word/Shutdown.hpp" #include "dsl/word/Single.hpp" From 1f7101a9b358aaf28a8f1e5a7d77fd6cd2d130b5 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:51:18 +1000 Subject: [PATCH 19/87] Add an `immediate` flag to `ReactionTask` for `Direct` emits --- src/dsl/word/emit/Direct.hpp | 3 ++- src/threading/ReactionTask.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dsl/word/emit/Direct.hpp b/src/dsl/word/emit/Direct.hpp index 48a265277..8529f26f9 100644 --- a/src/dsl/word/emit/Direct.hpp +++ b/src/dsl/word/emit/Direct.hpp @@ -63,7 +63,8 @@ namespace dsl { auto task = reaction->get_task(); if (task) { - task = task->run(std::move(task)); + task->immediate = true; + powerplant.submit(std::move(task)); } } catch (const std::exception& ex) { diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index a028dc257..c38875959 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -130,6 +130,7 @@ namespace threading { std::type_index group_id; util::ThreadPoolDescriptor thread_pool_descriptor; + bool immediate{false}; /// @brief the data bound callback to be executed /// @attention note this must be last in the list as the this pointer is passed to the callback generator TaskFunction callback; From bfa865f27d12d8a18ffcd667f0acffd5bafff062 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:51:55 +1000 Subject: [PATCH 20/87] Rewrite `Always` to suit new `TaskScheduler` --- src/dsl/word/Always.hpp | 37 +++++++++++++++++++++------------- src/threading/ReactionTask.hpp | 2 ++ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index e86ab6d5b..3c7d7bb28 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -20,6 +20,7 @@ #define NUCLEAR_DSL_WORD_ALWAYS_HPP #include "../../threading/ReactionTask.hpp" +#include "../../util/thread_pool.hpp" namespace NUClear { namespace dsl { @@ -61,27 +62,35 @@ namespace dsl { */ struct Always { + template + static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& /*task*/) { + static uint64_t pool_id = util::ThreadPoolIDSource::source++; + return util::ThreadPoolDescriptor{pool_id, 1}; + } + template static inline void bind(const std::shared_ptr& reaction) { reaction->unbinders.push_back([](threading::Reaction& r) { r.enabled = false; }); - // This is our function that runs forever until the powerplant exits - reaction->reactor.powerplant.add_thread_task([reaction] { - while (reaction->reactor.powerplant.running()) { - try { - // Get a task - auto task = reaction->get_task(); + try { + // Get a task + auto task = reaction->get_task(); + + // If we got a real task back + if (task) { + // Set the thread pool on the task + task->thread_pool_descriptor = Always::pool(*task); + + // Make sure this task never dies + task->keep_alive = true; - // If we got a real task back - if (task) { - task = task->run(std::move(task)); - } - } - catch (...) { - } + // Submit the task to be run + reaction->reactor.powerplant.submit(std::move(task)); } - }); + } + catch (...) { + } } }; diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index c38875959..09f81d1b6 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -131,6 +131,8 @@ namespace threading { std::type_index group_id; util::ThreadPoolDescriptor thread_pool_descriptor; bool immediate{false}; + bool keep_alive{false}; + /// @brief the data bound callback to be executed /// @attention note this must be last in the list as the this pointer is passed to the callback generator TaskFunction callback; From 6f5e82b39e1444d0bded838e4bab39ac08e663d7 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:52:20 +1000 Subject: [PATCH 21/87] Change `Initialise` emits to an emit with no delay --- src/dsl/word/emit/Initialise.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dsl/word/emit/Initialise.hpp b/src/dsl/word/emit/Initialise.hpp index 4b0912bcb..4e8a3a437 100644 --- a/src/dsl/word/emit/Initialise.hpp +++ b/src/dsl/word/emit/Initialise.hpp @@ -19,7 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_EMIT_INITIALISE_HPP #define NUCLEAR_DSL_WORD_EMIT_INITIALISE_HPP -#include "Direct.hpp" +#include "Delay.hpp" namespace NUClear { namespace dsl { @@ -53,9 +53,9 @@ namespace dsl { static void emit(PowerPlant& powerplant, std::shared_ptr data) { - auto task = [&powerplant, data] { emit::Direct::emit(powerplant, data); }; - - powerplant.on_startup(task); + // Delay the emit by 0 seconds, this will delay the emit until the chrono controller starts, which + // will be when the system starts + Delay::emit(powerplant, data, std::chrono::seconds(0)); } }; From 56ee199023305efe5bf5bcecf781c830611fa6af Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:52:40 +1000 Subject: [PATCH 22/87] Change `MainThread` to use the main thread pool --- src/dsl/word/MainThread.hpp | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 44b0341a1..4e683b1e2 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -21,6 +21,7 @@ #include "../../threading/ReactionTask.hpp" #include "../../util/main_thread_id.hpp" +#include "../../util/thread_pool.hpp" namespace NUClear { namespace dsl { @@ -35,31 +36,12 @@ namespace dsl { * This will most likely be used with graphics related tasks. * * For best use, this word should be fused with at least one other binding DSL word. - * - * @par Implements - * Pre-condition */ struct MainThread { - using task_ptr = std::unique_ptr; - template - static inline std::unique_ptr reschedule( - std::unique_ptr&& task) { - - // If we are not the main thread, move us to the main thread - if (std::this_thread::get_id() != util::main_thread_id) { - - // Submit to the main thread scheduler - task->parent.reactor.powerplant.submit_main(std::move(task)); - - // We took the task away so return null - return std::unique_ptr(nullptr); - } - // Otherwise run! - else { - return std::move(task); - } + static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& task) { + return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; } }; From 6a1a7741612d82c8565b53981ced0a9c529972b4 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:53:00 +1000 Subject: [PATCH 23/87] Implement `Pool` `DSL` --- src/dsl/word/Pool.hpp | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/dsl/word/Pool.hpp diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp new file mode 100644 index 000000000..ca8b739d9 --- /dev/null +++ b/src/dsl/word/Pool.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_WORD_POOL_HPP +#define NUCLEAR_DSL_WORD_POOL_HPP + +#include "../../threading/ReactionTask.hpp" +#include "../../util/thread_pool.hpp" + +namespace NUClear { +namespace dsl { + namespace word { + + template + struct Pool { + template + static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& task) { + static uint64_t pool_id = util::ThreadPoolIDSource::source++; + return util::ThreadPoolDescriptor{pool_id, PoolType::concurrency}; + } + }; + + } // namespace word +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_WORD_POOL_HPP From ee85a92e6a32b45e4c5750e7f379c280dc94d083 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 8 Jun 2023 14:53:30 +1000 Subject: [PATCH 24/87] `#undef` valgrind symbols before setting them --- src/PowerPlant.hpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index c66370576..7e2bbc604 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -21,9 +21,11 @@ // See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 #if defined(USE_VALGRIND) && !defined(NDEBUG) -# include -# define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) -# define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) + #include + #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE + #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER + #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) + #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) #endif // defined(USE_VALGRIND) && !defined(NDEBUG) #include @@ -40,11 +42,11 @@ // See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 #if defined(USE_VALGRIND) && !defined(NDEBUG) -# define _GLIBCXX_THREAD_IMPL 1 + #define _GLIBCXX_THREAD_IMPL 1 #endif // defined(USE_VALGRIND) && !defined(NDEBUG) #include #if defined(USE_VALGRIND) && !defined(NDEBUG) -# undef _GLIBCXX_THREAD_IMPL + #undef _GLIBCXX_THREAD_IMPL #endif // defined(USE_VALGRIND) && !defined(NDEBUG) // Utilities From 7a3c4ef1798dcae38bd5a79d68c6beabbdd3b658 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 9 Jun 2023 14:54:09 +1000 Subject: [PATCH 25/87] Remove reschedule --- src/dsl/Fusion.hpp | 2 - src/dsl/Parse.hpp | 5 -- src/dsl/fusion/NoOp.hpp | 9 --- src/dsl/fusion/RescheduleFusion.hpp | 111 ---------------------------- src/dsl/fusion/has_reschedule.hpp | 56 -------------- src/threading/ReactionTask.hpp | 12 ++- src/threading/TaskScheduler.cpp | 10 +-- src/util/CallbackGenerator.hpp | 56 ++++++-------- 8 files changed, 33 insertions(+), 228 deletions(-) delete mode 100644 src/dsl/fusion/RescheduleFusion.hpp delete mode 100644 src/dsl/fusion/has_reschedule.hpp diff --git a/src/dsl/Fusion.hpp b/src/dsl/Fusion.hpp index ba80d2572..0b7608e2e 100644 --- a/src/dsl/Fusion.hpp +++ b/src/dsl/Fusion.hpp @@ -27,7 +27,6 @@ #include "fusion/PostconditionFusion.hpp" #include "fusion/PreconditionFusion.hpp" #include "fusion/PriorityFusion.hpp" -#include "fusion/RescheduleFusion.hpp" namespace NUClear { namespace dsl { @@ -41,7 +40,6 @@ namespace dsl { , public fusion::PriorityFusion , public fusion::GroupFusion , public fusion::PoolFusion - , public fusion::RescheduleFusion , public fusion::PostconditionFusion {}; } // namespace dsl diff --git a/src/dsl/Parse.hpp b/src/dsl/Parse.hpp index 9cec39c36..0b2cc646d 100644 --- a/src/dsl/Parse.hpp +++ b/src/dsl/Parse.hpp @@ -63,11 +63,6 @@ namespace dsl { Parse>(r); } - static std::unique_ptr reschedule(std::unique_ptr&& task) { - return std::conditional_t::value, DSL, fusion::NoOp>::template reschedule( - std::move(task)); - } - static inline void postcondition(threading::ReactionTask& r) { std::conditional_t::value, DSL, fusion::NoOp>::template postcondition< Parse>(r); diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 4bbe7aaea..efb0d4775 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -64,12 +64,6 @@ namespace dsl { return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; } - template - static inline std::unique_ptr reschedule( - std::unique_ptr&& task) { - return std::move(task); - } - template static inline void postcondition(threading::ReactionTask&) {} }; @@ -93,9 +87,6 @@ namespace dsl { static inline util::ThreadPoolDescriptor pool(threading::Reaction&); - static inline std::unique_ptr reschedule( - std::unique_ptr&& task); - static inline void postcondition(threading::ReactionTask&); }; diff --git a/src/dsl/fusion/RescheduleFusion.hpp b/src/dsl/fusion/RescheduleFusion.hpp deleted file mode 100644 index a5c96e5f4..000000000 --- a/src/dsl/fusion/RescheduleFusion.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_FUSION_RESCHEDULEFUSION_HPP -#define NUCLEAR_DSL_FUSION_RESCHEDULEFUSION_HPP - -#include "../../threading/ReactionTask.hpp" -#include "../operation/DSLProxy.hpp" -#include "has_reschedule.hpp" - -namespace NUClear { -namespace dsl { - namespace fusion { - - /// Type that redirects types without a reschedule function to their proxy type - template - struct Reschedule { - using type = std::conditional_t::value, Word, operation::DSLProxy>; - }; - - template > - struct RescheduleWords; - - /** - * @brief Metafunction that extracts all of the Words with a reschedule function - * - * @tparam Word1 The word we are looking at - * @tparam WordN The words we have yet to look at - * @tparam FoundWords The words we have found with reschedule functions - */ - template - struct RescheduleWords, std::tuple> - : public std::conditional_t< - has_reschedule::type>::value, - /*T*/ - RescheduleWords, std::tuple::type>>, - /*F*/ RescheduleWords, std::tuple>> {}; - - /** - * @brief Termination case for the RescheduleWords metafunction - * - * @tparam FoundWords The words we have found with reschedule functions - */ - template - struct RescheduleWords, std::tuple> { - using type = std::tuple; - }; - - - // Default case where there are no reschedule words - template - struct RescheduleFuser {}; - - // Case where there is only a single word remaining - template - struct RescheduleFuser> { - - template - static inline std::unique_ptr reschedule( - std::unique_ptr&& task) { - - // Pass our task to see if it gets rescheduled and return the result - return Word::template reschedule(std::move(task)); - } - }; - - // Case where there is more 2 more more words remaining - template - struct RescheduleFuser> { - - template - static inline std::unique_ptr reschedule( - std::unique_ptr&& task) { - - // Pass our task to see if it gets rescheduled - auto ptr = Word1::template reschedule(std::move(task)); - - // If it was not rescheduled pass to the next rescheduler - if (ptr) { - return RescheduleFuser>::template reschedule(std::move(ptr)); - } - else { - return ptr; - } - } - }; - - template - struct RescheduleFusion - : public RescheduleFuser>::type> {}; - - } // namespace fusion -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_FUSION_RESCHEDULEFUSION_HPP diff --git a/src/dsl/fusion/has_reschedule.hpp b/src/dsl/fusion/has_reschedule.hpp deleted file mode 100644 index 77e57037a..000000000 --- a/src/dsl/fusion/has_reschedule.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_FUSION_HAS_RESCHEDULE_HPP -#define NUCLEAR_DSL_FUSION_HAS_RESCHEDULE_HPP - -#include "../../threading/ReactionTask.hpp" -#include "NoOp.hpp" - -namespace NUClear { -namespace dsl { - namespace fusion { - - /** - * @brief SFINAE struct to test if the passed class has a reschedule function that conforms to the - * NUClear DSL - * - * @tparam T the class to check - */ - template - struct has_reschedule { - private: - typedef std::true_type yes; - typedef std::false_type no; - - template - static auto test(int) -> decltype(U::template reschedule( - std::declval>()), - yes()); - template - static no test(...); - - public: - static constexpr bool value = std::is_same(0)), yes>::value; - }; - - } // namespace fusion -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_FUSION_HAS_RESCHEDULE_HPP diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 09f81d1b6..bff38cbef 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -52,7 +52,7 @@ namespace threading { public: /// Type of the functions that ReactionTasks execute - using TaskFunction = std::function>(std::unique_ptr>&&)>; + using TaskFunction = std::function&)>; /** * @brief Gets the current executing task, or nullptr if there isn't one. @@ -100,20 +100,18 @@ namespace threading { * This runs the internal data bound task and times how long the execution takes. These figures can then be * used in a debugging context to calculate how long callbacks are taking to run. */ - inline std::unique_ptr> run(std::unique_ptr>&& us) { + inline void run() { // Update our current task + // TODO RAII THIS Task* old_task = current_task; current_task = this; - // Run our callback at catch the returned task (to see if it rescheduled itself) - us = callback(std::move(us)); + // Run our callback + callback(*this); // Reset our task back current_task = old_task; - - // Return our original task - return std::move(us); } /// @brief the parent Reaction object which spawned this diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 4f79a7224..7fb3291f9 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -60,8 +60,8 @@ namespace threading { auto task = get_task(p.pool_id); if (task) { - task = task->run(std::move(task)); - if (task && task->keep_alive) { + task->run(); + if (task->keep_alive) { auto new_task = task->parent.get_task(); submit(std::move(new_task)); } @@ -92,8 +92,8 @@ namespace threading { auto task = get_task(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID); if (task) { - task = task->run(std::move(task)); - if (task && task->keep_alive) { + task->run(); + if (task->keep_alive) { auto new_task = task->parent.get_task(); submit(std::move(new_task)); } @@ -132,7 +132,7 @@ namespace threading { if (task->immediate) { if (is_runnable(task, pool_map.at(std::this_thread::get_id())) || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID)) { - task = task->run(std::move(task)); + task->run(); return; } // Not runnable, stick it in the queue like nothing happened diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index c57b04e88..e84bb7702 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -97,47 +97,37 @@ namespace util { DSL::priority(r), DSL::group(r), DSL::pool(r), - [c, data](std::unique_ptr&& task) { - // Check if we are going to reschedule - task = DSL::reschedule(std::move(task)); + [c, data](threading::ReactionTask& task) { + // Update our thread's priority to the correct level + update_current_thread_priority(task.priority); - // If we still control our task - if (task) { + // Record our start time + task.stats->started = clock::now(); - // Update our thread's priority to the correct level - update_current_thread_priority(task->priority); - - // Record our start time - task->stats->started = clock::now(); - - // We have to catch any exceptions - try { - // We call with only the relevant arguments to the passed function - util::apply_relevant(c, std::move(data)); - } - catch (...) { + // We have to catch any exceptions + try { + // We call with only the relevant arguments to the passed function + util::apply_relevant(c, std::move(data)); + } + catch (...) { - // Catch our exception if it happens - task->stats->exception = std::current_exception(); - } + // Catch our exception if it happens + task.stats->exception = std::current_exception(); + } - // Our finish time - task->stats->finished = clock::now(); + // Our finish time + task.stats->finished = clock::now(); - // Run our postconditions - DSL::postcondition(*task); + // Run our postconditions + DSL::postcondition(task); - // Take one from our active tasks - --task->parent.active_tasks; + // Take one from our active tasks + --task.parent.active_tasks; - // Emit our reaction statistics if it wouldn't cause a loop - if (task->emit_stats) { - PowerPlant::powerplant->emit_shared(task->stats); - } + // Emit our reaction statistics if it wouldn't cause a loop + if (task.emit_stats) { + PowerPlant::powerplant->emit_shared(task.stats); } - - // Return our task - return std::move(task); }); } } From daa6ce9ad99d7b943d51e02118a8542806dc8a24 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 10 Jun 2023 11:17:54 +1000 Subject: [PATCH 26/87] formatting --- src/util/CallbackGenerator.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index e84bb7702..7c1967ae7 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -45,9 +45,10 @@ namespace util { template struct CallbackGenerator { - CallbackGenerator(Function&& callback) - : callback(std::forward(callback)) - , transients(std::make_shared::type>()){}; + template + CallbackGenerator(F&& callback) + : callback(std::forward(callback)) + , transients(std::make_shared::type>()) {} template void merge_transients(std::tuple& data, const Sequence&, const Sequence&) { @@ -58,7 +59,6 @@ namespace util { std::get(data))...); } - GeneratedCallback operator()(threading::Reaction& r) { // Add one to our active tasks From 7eb0215b4a70a166a78eacaf1e94fb785e448156 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 11:02:45 +1000 Subject: [PATCH 27/87] Apply clang-tidy fixes --- src/PowerPlant.cpp | 2 ++ src/PowerPlant.hpp | 5 +++-- src/dsl/fusion/GroupFusion.hpp | 2 +- src/dsl/fusion/NoOp.hpp | 6 +++--- src/dsl/fusion/PoolFusion.hpp | 2 +- src/dsl/fusion/has_pool.hpp | 4 ++-- src/dsl/word/MainThread.hpp | 2 +- src/dsl/word/Pool.hpp | 4 ++-- src/dsl/word/Sync.hpp | 4 ++-- src/dsl/word/emit/Direct.hpp | 2 +- src/threading/ReactionTask.hpp | 2 +- src/threading/TaskScheduler.cpp | 11 +++++------ src/util/CallbackGenerator.hpp | 10 +++++----- src/util/thread_pool.cpp | 1 + src/util/thread_pool.hpp | 1 + 15 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index 60b497e20..382f1ecfc 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -20,6 +20,7 @@ // See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 #if defined(USE_VALGRIND) && !defined(NDEBUG) +// NOLINTBEGIN namespace std { extern "C" { static void* execute_native_thread_routine(void* __p) { @@ -46,6 +47,7 @@ void thread::_M_start_thread(_State_ptr state, void (*depend)()) { } } } // namespace std +// NOLINTEND #endif // defined(USE_VALGRIND) && !defined(NDEBUG) namespace NUClear { diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 1a2c99404..99d8617bb 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -24,8 +24,8 @@ #include #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER - #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) - #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) + #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) // NOLINT + #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) // NOLINT #endif // defined(USE_VALGRIND) && !defined(NDEBUG) #include @@ -42,6 +42,7 @@ // See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 #if defined(USE_VALGRIND) && !defined(NDEBUG) + // NOLINTNEXTLINE #define _GLIBCXX_THREAD_IMPL 1 #endif // defined(USE_VALGRIND) && !defined(NDEBUG) #include diff --git a/src/dsl/fusion/GroupFusion.hpp b/src/dsl/fusion/GroupFusion.hpp index 9ea8215a3..4bb706527 100644 --- a/src/dsl/fusion/GroupFusion.hpp +++ b/src/dsl/fusion/GroupFusion.hpp @@ -83,7 +83,7 @@ namespace dsl { struct GroupFuser> { template - static inline void group(threading::Reaction& reaction) { + static inline void group(threading::Reaction& /*reaction*/) { throw std::runtime_error("Can not be a member of more than one group"); } }; diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 685d3374c..7c0fd6f4b 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -55,12 +55,12 @@ namespace dsl { } template - static inline std::type_index group(threading::Reaction&) { - return std::type_index(typeid(threading::TaskScheduler)); + static inline std::type_index group(threading::Reaction& /*reaction*/) { + return typeid(threading::TaskScheduler); } template - static inline util::ThreadPoolDescriptor pool(threading::Reaction&) { + static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; } diff --git a/src/dsl/fusion/PoolFusion.hpp b/src/dsl/fusion/PoolFusion.hpp index 08316da51..8c4c67221 100644 --- a/src/dsl/fusion/PoolFusion.hpp +++ b/src/dsl/fusion/PoolFusion.hpp @@ -83,7 +83,7 @@ namespace dsl { struct PoolFuser> { template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& reaction) { + static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { throw std::runtime_error("Can not be a member of more than one pool"); } }; diff --git a/src/dsl/fusion/has_pool.hpp b/src/dsl/fusion/has_pool.hpp index a3a1f4366..160916972 100644 --- a/src/dsl/fusion/has_pool.hpp +++ b/src/dsl/fusion/has_pool.hpp @@ -33,8 +33,8 @@ namespace dsl { template struct has_pool { private: - typedef std::true_type yes; - typedef std::false_type no; + using yes = std::true_type; + using no = std::false_type; template static auto test(int) diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 4e683b1e2..725462587 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -40,7 +40,7 @@ namespace dsl { struct MainThread { template - static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& task) { + static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& /*task*/) { return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; } }; diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index ca8b739d9..e6d7a4963 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -28,8 +28,8 @@ namespace dsl { template struct Pool { template - static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& task) { - static uint64_t pool_id = util::ThreadPoolIDSource::source++; + static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& /*task*/) { + const static uint64_t pool_id = util::ThreadPoolIDSource::source++; return util::ThreadPoolDescriptor{pool_id, PoolType::concurrency}; } }; diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index b0896cc23..f79040bf5 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -72,8 +72,8 @@ namespace dsl { struct Sync { template - static inline std::type_index group(threading::ReactionTask& task) { - return std::type_index(typeid(SyncGroup)); + static inline std::type_index group(threading::ReactionTask& /*task*/) { + return typeid(SyncGroup); } }; diff --git a/src/dsl/word/emit/Direct.hpp b/src/dsl/word/emit/Direct.hpp index 131219ef8..ae5229241 100644 --- a/src/dsl/word/emit/Direct.hpp +++ b/src/dsl/word/emit/Direct.hpp @@ -52,7 +52,7 @@ namespace dsl { template struct Direct { - static void emit(PowerPlant& /*powerplant*/, std::shared_ptr data) { + static void emit(PowerPlant& powerplant, std::shared_ptr data) { // Run all our reactions that are interested for (auto& reaction : store::TypeCallbackStore::get()) { diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 87b77acd1..8de96b851 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -103,7 +103,7 @@ namespace threading { inline void run() { // Update our current task - // TODO RAII THIS + // TODO(Trent) RAII THIS Task* old_task = current_task; current_task = this; diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 7fb3291f9..e43e49ec0 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -74,7 +74,7 @@ namespace threading { // Start all our threads /* mutex scope */ { - std::lock_guard lock(threads_mutex); + const std::lock_guard threads_lock(threads_mutex); for (size_t i = 0; i < pool.thread_count; ++i) { threads.push_back(std::make_unique(pool_func)); pool_map[threads.back()->get_id()] = pool.pool_id; @@ -115,7 +115,7 @@ namespace threading { } void TaskScheduler::shutdown() { - std::lock_guard lock(mutex); + const std::lock_guard lock(mutex); running.store(false); condition.notify_all(); } @@ -139,11 +139,10 @@ namespace threading { } /* Mutex Scope */ { - std::lock_guard lock(mutex); + const std::lock_guard lock(mutex); // Find where to insert the new task to maintain task order - auto it = - std::lower_bound(queue.begin(), queue.end(), task, std::less>()); + auto it = std::lower_bound(queue.begin(), queue.end(), task, std::less<>()); // Insert before the found position queue.insert(it, std::forward>(task)); @@ -151,7 +150,7 @@ namespace threading { } // Notify a thread that it can proceed - std::lock_guard lock(mutex); + const std::lock_guard lock(mutex); condition.notify_all(); } diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 7dff6e2e8..b420e06c0 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -45,7 +45,8 @@ namespace util { template struct CallbackGenerator { - template + // Don't using this constructor is F is of type CallbackGenerator + template ::value, bool> = true> CallbackGenerator(F&& callback) : callback(std::forward(callback)) , transients(std::make_shared::type>()) {} @@ -130,11 +131,10 @@ namespace util { } }); } - } - Function callback; - std::shared_ptr::type> transients; -}; + Function callback; + std::shared_ptr::type> transients; + }; } // namespace util } // namespace NUClear diff --git a/src/util/thread_pool.cpp b/src/util/thread_pool.cpp index ec19e9da0..472439c67 100644 --- a/src/util/thread_pool.cpp +++ b/src/util/thread_pool.cpp @@ -3,6 +3,7 @@ namespace NUClear { namespace util { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::atomic ThreadPoolIDSource::source = {2}; } // namespace util diff --git a/src/util/thread_pool.hpp b/src/util/thread_pool.hpp index 55954cc7f..f341be3b4 100644 --- a/src/util/thread_pool.hpp +++ b/src/util/thread_pool.hpp @@ -34,6 +34,7 @@ namespace util { }; struct ThreadPoolIDSource { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::atomic source; static constexpr uint64_t MAIN_THREAD_POOL_ID = 0; static constexpr uint64_t DEFAULT_THREAD_POOL_ID = 1; From 0b9c0d84dac0e5fecc86a772035e1c012ffb7e24 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:01:21 +1000 Subject: [PATCH 28/87] Make Always work --- src/dsl/word/Always.hpp | 87 ++++++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 3c7d7bb28..45720445a 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -19,6 +19,9 @@ #ifndef NUCLEAR_DSL_WORD_ALWAYS_HPP #define NUCLEAR_DSL_WORD_ALWAYS_HPP +#include +#include + #include "../../threading/ReactionTask.hpp" #include "../../util/thread_pool.hpp" @@ -63,33 +66,87 @@ namespace dsl { struct Always { template - static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& /*task*/) { - static uint64_t pool_id = util::ThreadPoolIDSource::source++; - return util::ThreadPoolDescriptor{pool_id, 1}; + static inline util::ThreadPoolDescriptor pool(threading::Reaction& reaction) { + static std::map pool_id; + static std::mutex mutex; + + const std::lock_guard lock(mutex); + if (pool_id.count(reaction.id) == 0) { + pool_id[reaction.id] = util::ThreadPoolIDSource::get_new_pool_id(); + } + return util::ThreadPoolDescriptor{pool_id[reaction.id], 1}; } template - static inline void bind(const std::shared_ptr& reaction) { + static inline void bind(const std::shared_ptr& always_reaction) { + static std::map, std::shared_ptr>> + reaction_store = {}; - reaction->unbinders.push_back([](threading::Reaction& r) { r.enabled = false; }); + always_reaction->unbinders.push_back([](threading::Reaction& r) { r.enabled = false; }); - try { - // Get a task - auto task = reaction->get_task(); + auto idle_callback = [always_reaction](threading::Reaction& idle_reaction) -> util::GeneratedCallback { + // Reaction is still bound re-add the reaction and this + auto callback = [&idle_reaction, always_reaction](threading::ReactionTask& /*task*/) { + // Get new task for the actual reaction + auto always_task = always_reaction->get_task(); + if (always_task) { + // Set the thread pool on the task + always_task->thread_pool_descriptor = Always::pool(always_task->parent); - // If we got a real task back - if (task) { + // Submit the task to be run + always_reaction->reactor.powerplant.submit(std::move(always_task)); + } + + // Get new task for the actual reaction + auto idle_task = idle_reaction.get_task(); + if (idle_task) { // Set the thread pool on the task - task->thread_pool_descriptor = Always::pool(*task); + idle_task->thread_pool_descriptor = Always::pool(idle_task->parent); - // Make sure this task never dies - task->keep_alive = true; + // Only let this task run when the always thread pool is idle + idle_task->priority = Priority::IDLE::value - 1; // Submit the task to be run - reaction->reactor.powerplant.submit(std::move(task)); + idle_reaction.reactor.powerplant.submit(std::move(idle_task)); + } + }; + + return {Priority::IDLE::value - 1, DSL::group(idle_reaction), DSL::pool(idle_reaction), callback}; + }; + auto idle_reaction = std::make_shared( + always_reaction->reactor, + std::vector{always_reaction->identifier[0] + " - IDLE Task", + always_reaction->identifier[1], + always_reaction->identifier[2], + always_reaction->identifier[3]}, + idle_callback); + + // Keep this reaction handy so it doesn't go out of scope + reaction_store[always_reaction->id] = {always_reaction, idle_reaction}; + + // Get a task for the always reaction and submit it to the scheduler + auto always_task = always_reaction->get_task(); + if (always_task) { + always_reaction->reactor.powerplant.submit(std::move(always_task)); + } + + // Get a task for the idle reaction and submit it to the scheduler + auto idle_task = idle_reaction->get_task(); + if (idle_task) { + idle_reaction->reactor.powerplant.submit(std::move(idle_task)); } } - catch (...) { + + template + static inline void postcondition(threading::ReactionTask& task) { + auto new_task = task.parent.get_task(); + if (new_task) { + // Set the thread pool on the task + new_task->thread_pool_descriptor = Always::pool(new_task->parent); + + // Submit the task to be run + task.parent.reactor.powerplant.submit(std::move(new_task)); } } }; From c193a1991e6668fa511c32d3759988c0a1a851a7 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:01:37 +1000 Subject: [PATCH 29/87] Fix MainThread --- src/dsl/word/MainThread.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 725462587..2e9178525 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -40,7 +40,7 @@ namespace dsl { struct MainThread { template - static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& /*task*/) { + static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; } }; From 06c2bcc2b7125796f97f01af83c68dbf0c45b84e Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:02:08 +1000 Subject: [PATCH 30/87] Fix parsing --- src/dsl/Parse.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsl/Parse.hpp b/src/dsl/Parse.hpp index 0b2cc646d..8b4e35534 100644 --- a/src/dsl/Parse.hpp +++ b/src/dsl/Parse.hpp @@ -59,7 +59,7 @@ namespace dsl { } static inline util::ThreadPoolDescriptor pool(threading::Reaction& r) { - return std::conditional_t::value, DSL, fusion::NoOp>::template pool< + return std::conditional_t::value, DSL, fusion::NoOp>::template pool< Parse>(r); } From 3984a075be55e57141078742f40ded70ae17b1da Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:02:26 +1000 Subject: [PATCH 31/87] Fix ChronoController shutdown --- src/extension/ChronoController.hpp | 70 ++++++++++++++++-------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 764c364bc..040aaec28 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -37,8 +37,10 @@ namespace extension { // Lock the mutex while we're doing stuff const std::lock_guard lock(mutex); - // Add our new task to the heap - tasks.push_back(*task); + // Add our new task to the heap if we are still running + if (running) { + tasks.push_back(*task); + } // Poke the system wait.notify_all(); @@ -55,7 +57,7 @@ namespace extension { return task.id == unbind.id; }); - // Remove if if it exists + // Remove if it exists if (it != tasks.end()) { tasks.erase(it); } @@ -67,6 +69,7 @@ namespace extension { // When we shutdown we notify so we quit now on().then("Shutdown Chrono Controller", [this] { const std::lock_guard lock(mutex); + running = false; wait.notify_all(); }); @@ -74,48 +77,50 @@ namespace extension { // Acquire the mutex lock so we can wait on it std::unique_lock lock(mutex); - // If we have tasks to do - if (!tasks.empty()) { + if (running) { + // If we have tasks to do + if (!tasks.empty()) { - // Make the list into a heap so we can remove the soonest ones - std::make_heap(tasks.begin(), tasks.end(), std::greater<>()); + // Make the list into a heap so we can remove the soonest ones + std::make_heap(tasks.begin(), tasks.end(), std::greater<>()); - // If we are within the wait offset of the time, spinlock until we get there for greater - // accuracy - if (NUClear::clock::now() + wait_offset > tasks.front().time) { + // If we are within the wait offset of the time, spinlock until we get there for greater + // accuracy + if (NUClear::clock::now() + wait_offset > tasks.front().time) { - // Spinlock! - while (NUClear::clock::now() < tasks.front().time) { - } + // Spinlock! + while (NUClear::clock::now() < tasks.front().time) { + } - const NUClear::clock::time_point now = NUClear::clock::now(); + const NUClear::clock::time_point now = NUClear::clock::now(); - // Move back from the end poping the heap - for (auto end = tasks.end(); end != tasks.begin() && tasks.front().time < now;) { - // Run our task and if it returns false remove it - const bool renew = tasks.front()(); + // Move back from the end poping the heap + for (auto end = tasks.end(); end != tasks.begin() && tasks.front().time < now;) { + // Run our task and if it returns false remove it + const bool renew = tasks.front()(); - // Move this to the back of the list - std::pop_heap(tasks.begin(), end, std::greater<>()); + // Move this to the back of the list + std::pop_heap(tasks.begin(), end, std::greater<>()); - if (!renew) { - end = tasks.erase(--end); - } - else { - --end; + if (!renew) { + end = tasks.erase(--end); + } + else { + --end; + } } } + // Otherwise we wait for the next event using a wait_for (with a small offset for greater + // accuracy) Either that or until we get interrupted with a new event + else { + wait.wait_until(lock, tasks.front().time - wait_offset); + } } - // Otherwise we wait for the next event using a wait_for (with a small offset for greater - // accuracy) Either that or until we get interrupted with a new event + // Otherwise we wait for something to happen else { - wait.wait_until(lock, tasks.front().time - wait_offset); + wait.wait(lock); } } - // Otherwise we wait for something to happen - else { - wait.wait(lock); - } }); } @@ -123,6 +128,7 @@ namespace extension { std::vector tasks; std::mutex mutex; std::condition_variable wait; + bool running{true}; NUClear::clock::duration wait_offset; }; From 2bc1ea23fc2ad2e67a9984b20029619421d535d8 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:03:47 +1000 Subject: [PATCH 32/87] Remove `keep_alive` --- src/threading/ReactionTask.hpp | 1 - src/threading/TaskScheduler.cpp | 8 -------- 2 files changed, 9 deletions(-) diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 8de96b851..3c0530bcf 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -129,7 +129,6 @@ namespace threading { std::type_index group_id; util::ThreadPoolDescriptor thread_pool_descriptor; bool immediate{false}; - bool keep_alive{false}; /// @brief the data bound callback to be executed /// @attention note this must be last in the list as the this pointer is passed to the callback generator diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index e43e49ec0..022065469 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -61,10 +61,6 @@ namespace threading { if (task) { task->run(); - if (task->keep_alive) { - auto new_task = task->parent.get_task(); - submit(std::move(new_task)); - } } // Back up to realtime while waiting @@ -93,10 +89,6 @@ namespace threading { if (task) { task->run(); - if (task->keep_alive) { - auto new_task = task->parent.get_task(); - submit(std::move(new_task)); - } } } From d0da0b71a38b68b98d708eb4ba24fb5697eba187 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:05:47 +1000 Subject: [PATCH 33/87] Control when threads are started --- src/threading/TaskScheduler.cpp | 73 +++++++++++++++++++++------------ src/threading/TaskScheduler.hpp | 8 +++- 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 022065469..13388e0a1 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -36,28 +36,13 @@ namespace threading { return task->thread_pool_descriptor.pool_id == pool_id; } - TaskScheduler::TaskScheduler() : running(true) { - pool_map[std::this_thread::get_id()] = util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID; - } - - void TaskScheduler::create_pool(const util::ThreadPoolDescriptor& pool) { - - // Pool already exists - if (pools.count(pool.pool_id) > 0 && pools.at(pool.pool_id).thread_count > 0) { - return; - } - - // Make a copy of the pool descriptor - pools.insert({pool.pool_id, util::ThreadPoolDescriptor{pool.pool_id, pool.thread_count}}); - - auto p = pool; - auto pool_func = [this, p] { + void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { // Wait at a high (but not realtime) priority to reduce latency // for picking up a new task update_current_thread_priority(1000); while (running.load() || !queue.empty()) { - auto task = get_task(p.pool_id); + auto task = get_task(pool.pool_id); if (task) { task->run(); @@ -66,25 +51,60 @@ namespace threading { // Back up to realtime while waiting update_current_thread_priority(1000); } - }; + } - // Start all our threads + TaskScheduler::TaskScheduler() { + pools[util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID] = + util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; + pool_map[std::this_thread::get_id()] = util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID; + } + + void TaskScheduler::start_threads(const util::ThreadPoolDescriptor& pool) { /* mutex scope */ { const std::lock_guard threads_lock(threads_mutex); + const std::lock_guard pool_lock(pool_mutex); for (size_t i = 0; i < pool.thread_count; ++i) { - threads.push_back(std::make_unique(pool_func)); + threads.push_back(std::make_unique(&TaskScheduler::pool_func, this, pool)); pool_map[threads.back()->get_id()] = pool.pool_id; } } } + void TaskScheduler::create_pool(const util::ThreadPoolDescriptor& pool) { + // Pool already exists + /* mutex scope */ { + const std::lock_guard pool_lock(pool_mutex); + if (pools.count(pool.pool_id) > 0 && pools.at(pool.pool_id).thread_count > 0) { + return; + } + + // Make a copy of the pool descriptor + pools[pool.pool_id] = util::ThreadPoolDescriptor{pool.pool_id, pool.thread_count}; + } + + // If the scheduler has not yet started then don't start the threads for this pool yet + if (started.load()) { + start_threads(pool); + } + } + void TaskScheduler::start(const size_t& thread_count) { // Make the default pool create_pool(util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, thread_count}); + // The scheduler is now started + started.store(true); + + // Start all our threads + /* mutex scope */ { + for (const auto& pool : pools) { + start_threads(pool.second); + } + } + // Run main thread tasks - while (running.load()) { + while (running.load() || !queue.empty()) { auto task = get_task(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID); if (task) { @@ -109,6 +129,7 @@ namespace threading { void TaskScheduler::shutdown() { const std::lock_guard lock(mutex); running.store(false); + started.store(false); condition.notify_all(); } @@ -121,9 +142,7 @@ namespace threading { create_pool(task->thread_pool_descriptor); // Check to see if this task was the result of `emit` - if (task->immediate) { - if (is_runnable(task, pool_map.at(std::this_thread::get_id())) - || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID)) { + if (started.load() && task->immediate) { task->run(); return; } @@ -141,16 +160,16 @@ namespace threading { } } - // Notify a thread that it can proceed + // Notify all threads that there is a new task to be processed const std::lock_guard lock(mutex); condition.notify_all(); } std::unique_ptr TaskScheduler::get_task(const uint64_t& pool_id) { - while (running.load() || !queue.empty()) { + std::unique_lock lock(mutex); - std::unique_lock lock(mutex); + while (running.load() || !queue.empty()) { for (auto it = queue.begin(); it != queue.end(); ++it) { // Check if we can run it diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 00b1d93c0..460a0eeb7 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -115,9 +115,12 @@ namespace threading { private: void create_pool(const util::ThreadPoolDescriptor& pool); + void pool_func(const util::ThreadPoolDescriptor& pool); + void start_threads(const util::ThreadPoolDescriptor& pool); - /// @brief if the scheduler is running or is shut down - std::atomic running; + /// @brief if the scheduler is running + std::atomic running{true}; + std::atomic started{false}; /// @brief our queue which sorts tasks by priority std::vector> queue; @@ -134,6 +137,7 @@ namespace threading { std::map pools{}; std::map pool_map{}; + std::mutex pool_mutex; }; } // namespace threading From 6f91b63c7ccc77bf52c7763f0b4c07fea39b7b70 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:07:18 +1000 Subject: [PATCH 34/87] Update unit tests --- tests/dsl/Always.cpp | 26 ++++++++++++++++++++------ tests/dsl/BlockNoData.cpp | 2 ++ tests/dsl/FlagMessage.cpp | 4 +--- tests/dsl/IO.cpp | 2 +- tests/dsl/Last.cpp | 2 +- tests/dsl/MainThread.cpp | 4 ++-- tests/dsl/Optional.cpp | 3 --- tests/dsl/With.cpp | 3 ++- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/tests/dsl/Always.cpp b/tests/dsl/Always.cpp index d48e52e59..293195ae1 100644 --- a/tests/dsl/Always.cpp +++ b/tests/dsl/Always.cpp @@ -20,18 +20,31 @@ #include namespace { +struct BlankMessage {}; -int i = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +int i = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +bool emitted_message = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { on().then([this] { - ++i; - - // Run until it's 11 then shutdown + // Run until it's 11 then emit the blank message if (i > 10) { + if (!emitted_message) { + emitted_message = true; + emit(std::make_unique()); + } + } + else { + ++i; + } + }); + + on>().then([this] { + if (i == 11) { + ++i; powerplant.shutdown(); } }); @@ -39,7 +52,7 @@ class TestReactor : public NUClear::Reactor { }; } // namespace -TEST_CASE("Testing on functionality (permanant run)", "[api][always]") { +TEST_CASE("Testing on functionality (permanent run)", "[api][always]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; @@ -52,5 +65,6 @@ TEST_CASE("Testing on functionality (permanant run)", "[api][always]") { plant.start(); - REQUIRE(i == 11); + REQUIRE(emitted_message); + REQUIRE(i == 12); } diff --git a/tests/dsl/BlockNoData.cpp b/tests/dsl/BlockNoData.cpp index b88318c10..59b86401b 100644 --- a/tests/dsl/BlockNoData.cpp +++ b/tests/dsl/BlockNoData.cpp @@ -62,4 +62,6 @@ TEST_CASE("Testing that when a trigger does not have it's data satisfied it does plant.emit(message); plant.start(); + + REQUIRE(a != nullptr); } diff --git a/tests/dsl/FlagMessage.cpp b/tests/dsl/FlagMessage.cpp index 4ba9686bd..db333ce87 100644 --- a/tests/dsl/FlagMessage.cpp +++ b/tests/dsl/FlagMessage.cpp @@ -62,9 +62,7 @@ class TestReactor : public NUClear::Reactor { // We make this high priority to ensure it runs first (will check for more errors) on, With, Priority::HIGH>().then([](const MessageA&, const MessageB&) { - // Check A and B have been emitted - REQUIRE(a != nullptr); - REQUIRE(b != nullptr); + FAIL("A was never emitted after B so this should not be possible"); }); } }; diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index a5153df7a..165b16c8f 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -30,7 +30,7 @@ namespace { class TestReactor : public NUClear::Reactor { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)), in(0), out(0) { + TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { std::array fds{-1, -1}; diff --git a/tests/dsl/Last.cpp b/tests/dsl/Last.cpp index 1ad91c7b7..927bfaf9c 100644 --- a/tests/dsl/Last.cpp +++ b/tests/dsl/Last.cpp @@ -54,7 +54,7 @@ class TestReactor : public NUClear::Reactor { REQUIRE(int(messages.size()) == messages.back()->value); } - // Check that our numbers are decreasing + // Check that our numbers are increasing int i = messages.front()->value; for (auto& m : messages) { REQUIRE(m->value == i); diff --git a/tests/dsl/MainThread.cpp b/tests/dsl/MainThread.cpp index 8fa2c5921..3e4513598 100644 --- a/tests/dsl/MainThread.cpp +++ b/tests/dsl/MainThread.cpp @@ -26,7 +26,7 @@ class TestReactor : public NUClear::Reactor { TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { // Run a task without MainThread to make sure it isn't on the main thread - on>().then([this] { + on>().then("Non-MainThread reaction", [this] { // We shouldn't be on the main thread REQUIRE(NUClear::util::main_thread_id != std::this_thread::get_id()); @@ -34,7 +34,7 @@ class TestReactor : public NUClear::Reactor { }); // Run a task with MainTHread and ensure that it is on the main thread - on, MainThread>().then([this] { + on, MainThread>().then("MainThread reaction", [this] { // We should be on the main thread REQUIRE(NUClear::util::main_thread_id == std::this_thread::get_id()); diff --git a/tests/dsl/Optional.cpp b/tests/dsl/Optional.cpp index 535792268..c5ce9fa85 100644 --- a/tests/dsl/Optional.cpp +++ b/tests/dsl/Optional.cpp @@ -27,9 +27,6 @@ int trigger3 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) int trigger4 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) struct MessageA {}; - -template -auto resolve_function_type(R (*)(A...)) -> R (*)(A...); struct MessageB {}; class TestReactor : public NUClear::Reactor { diff --git a/tests/dsl/With.cpp b/tests/dsl/With.cpp index d7be3dafb..d51b4077f 100644 --- a/tests/dsl/With.cpp +++ b/tests/dsl/With.cpp @@ -58,5 +58,6 @@ TEST_CASE("Testing poorly ordered on arguments", "[api][with]") { plant.emit(std::make_unique()); plant.emit(std::make_unique()); - plant.start(); + REQUIRE_NOTHROW(plant.start()); + REQUIRE_FALSE(plant.running()); } From 706653a2bcfe8688c87b5aafa72cf5677eee2fdc Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:08:47 +1000 Subject: [PATCH 35/87] Add a group descriptor to handle group concurrency --- src/dsl/Parse.hpp | 2 +- src/dsl/fusion/GroupFusion.hpp | 2 +- src/dsl/fusion/NoOp.hpp | 11 ++-- src/dsl/word/Always.hpp | 8 +-- src/dsl/word/Group.hpp | 42 +++++++++++++ src/dsl/word/MainThread.hpp | 2 +- src/dsl/word/Pool.hpp | 6 +- src/dsl/word/Sync.hpp | 6 +- src/threading/ReactionTask.hpp | 9 +-- src/threading/TaskScheduler.cpp | 63 +++++++++++++++---- src/threading/TaskScheduler.hpp | 6 +- src/util/GeneratedCallback.hpp | 9 +-- src/util/GroupDescriptor.hpp | 46 ++++++++++++++ ...read_pool.hpp => ThreadPoolDescriptor.hpp} | 19 +++--- src/util/thread_pool.cpp | 10 --- 15 files changed, 184 insertions(+), 57 deletions(-) create mode 100644 src/dsl/word/Group.hpp create mode 100644 src/util/GroupDescriptor.hpp rename src/util/{thread_pool.hpp => ThreadPoolDescriptor.hpp} (90%) delete mode 100644 src/util/thread_pool.cpp diff --git a/src/dsl/Parse.hpp b/src/dsl/Parse.hpp index 8b4e35534..239f5d66a 100644 --- a/src/dsl/Parse.hpp +++ b/src/dsl/Parse.hpp @@ -53,7 +53,7 @@ namespace dsl { Parse>(r); } - static inline std::type_index group(threading::Reaction& r) { + static inline util::GroupDescriptor group(threading::Reaction& r) { return std::conditional_t::value, DSL, fusion::NoOp>::template group< Parse>(r); } diff --git a/src/dsl/fusion/GroupFusion.hpp b/src/dsl/fusion/GroupFusion.hpp index 4bb706527..552d0b25e 100644 --- a/src/dsl/fusion/GroupFusion.hpp +++ b/src/dsl/fusion/GroupFusion.hpp @@ -71,7 +71,7 @@ namespace dsl { struct GroupFuser> { template - static inline std::type_index group(threading::Reaction& reaction) { + static inline util::GroupDescriptor group(threading::Reaction& reaction) { // Return our group return Word::template group(reaction); diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 7c0fd6f4b..70372e93e 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -23,7 +23,8 @@ #include "../../threading/Reaction.hpp" #include "../../threading/ReactionTask.hpp" -#include "../../util/thread_pool.hpp" +#include "../../util/GroupDescriptor.hpp" +#include "../../util/ThreadPoolDescriptor.hpp" #include "../word/Priority.hpp" namespace NUClear { @@ -55,13 +56,13 @@ namespace dsl { } template - static inline std::type_index group(threading::Reaction& /*reaction*/) { - return typeid(threading::TaskScheduler); + static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { + return util::GroupDescriptor{}; } template static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; + return util::ThreadPoolDescriptor{}; } template @@ -83,7 +84,7 @@ namespace dsl { static inline int priority(threading::Reaction&); - static inline std::type_index group(threading::Reaction&); + static inline util::GroupDescriptor group(threading::Reaction&); static inline util::ThreadPoolDescriptor pool(threading::Reaction&); diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 45720445a..822337734 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -23,7 +23,7 @@ #include #include "../../threading/ReactionTask.hpp" -#include "../../util/thread_pool.hpp" +#include "../../util/ThreadPoolDescriptor.hpp" namespace NUClear { namespace dsl { @@ -101,13 +101,13 @@ namespace dsl { // Get new task for the actual reaction auto idle_task = idle_reaction.get_task(); if (idle_task) { - // Set the thread pool on the task + // Set the thread pool on the task idle_task->thread_pool_descriptor = Always::pool(idle_task->parent); // Only let this task run when the always thread pool is idle idle_task->priority = Priority::IDLE::value - 1; - // Submit the task to be run + // Submit the task to be run idle_reaction.reactor.powerplant.submit(std::move(idle_task)); } }; @@ -135,8 +135,8 @@ namespace dsl { auto idle_task = idle_reaction->get_task(); if (idle_task) { idle_reaction->reactor.powerplant.submit(std::move(idle_task)); - } } + } template static inline void postcondition(threading::ReactionTask& task) { diff --git a/src/dsl/word/Group.hpp b/src/dsl/word/Group.hpp new file mode 100644 index 000000000..9f1ba0987 --- /dev/null +++ b/src/dsl/word/Group.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_WORD_GROUP_HPP +#define NUCLEAR_DSL_WORD_GROUP_HPP + +#include "../../threading/ReactionTask.hpp" +#include "../../util/GroupDescriptor.hpp" + +namespace NUClear { +namespace dsl { + namespace word { + + template + struct Group { + + template + static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { + const static uint64_t group_id = util::GroupDescriptor::get_new_group_id(); + return util::GroupDescriptor{group_id, GroupType::concurrency}; + } + }; + + } // namespace word +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_WORD_GROUP_HPP diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 2e9178525..f7170f9f8 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -20,8 +20,8 @@ #define NUCLEAR_DSL_WORD_MAINTHREAD_HPP #include "../../threading/ReactionTask.hpp" +#include "../../util/ThreadPoolDescriptor.hpp" #include "../../util/main_thread_id.hpp" -#include "../../util/thread_pool.hpp" namespace NUClear { namespace dsl { diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index e6d7a4963..edb6db365 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -19,7 +19,7 @@ #define NUCLEAR_DSL_WORD_POOL_HPP #include "../../threading/ReactionTask.hpp" -#include "../../util/thread_pool.hpp" +#include "../../util/ThreadPoolDescriptor.hpp" namespace NUClear { namespace dsl { @@ -28,8 +28,8 @@ namespace dsl { template struct Pool { template - static inline util::ThreadPoolDescriptor pool(threading::ReactionTask& /*task*/) { - const static uint64_t pool_id = util::ThreadPoolIDSource::source++; + static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { + const static uint64_t pool_id = util::ThreadPoolIDSource::get_new_pool_id(); return util::ThreadPoolDescriptor{pool_id, PoolType::concurrency}; } }; diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index f79040bf5..cd7b2f662 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -26,6 +26,7 @@ #include #include "../../threading/ReactionTask.hpp" +#include "../../util/GroupDescriptor.hpp" namespace NUClear { namespace dsl { @@ -72,8 +73,9 @@ namespace dsl { struct Sync { template - static inline std::type_index group(threading::ReactionTask& /*task*/) { - return typeid(SyncGroup); + static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { + const static uint64_t group_id = util::GroupDescriptor::get_new_group_id(); + return util::GroupDescriptor{group_id, 1}; } }; diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 3c0530bcf..c29dbe3cc 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -26,8 +26,9 @@ #include #include "../message/ReactionStatistics.hpp" +#include "../util/GroupDescriptor.hpp" +#include "../util/ThreadPoolDescriptor.hpp" #include "../util/platform.hpp" -#include "../util/thread_pool.hpp" namespace NUClear { namespace threading { @@ -72,7 +73,7 @@ namespace threading { */ Task(ReactionType& parent, const int& priority, - const std::type_index& group_id, + const util::GroupDescriptor& group_descriptor, const util::ThreadPoolDescriptor& thread_pool_descriptor, TaskFunction&& callback) : parent(parent) @@ -88,7 +89,7 @@ namespace threading { clock::time_point(std::chrono::seconds(0)), nullptr)) , emit_stats(parent.emit_stats && (current_task != nullptr ? current_task->emit_stats : true)) - , group_id(group_id) + , group_descriptor(group_descriptor) , thread_pool_descriptor(thread_pool_descriptor) , callback(callback) {} @@ -126,7 +127,7 @@ namespace threading { /// reaction statistics becomes false for all created tasks. This is to stop infinite loops of death. bool emit_stats; - std::type_index group_id; + util::GroupDescriptor group_descriptor; util::ThreadPoolDescriptor thread_pool_descriptor; bool immediate{false}; diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 13388e0a1..bf855fd8d 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -32,25 +32,41 @@ namespace NUClear { namespace threading { - bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id) { - return task->thread_pool_descriptor.pool_id == pool_id; + bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id, const size_t& group_count) { + return + // Task can run if it is meant to run on the current thread pool + task->thread_pool_descriptor.pool_id == pool_id && + // Task can run if the group is belongs to has spare threads + group_count < task->group_descriptor.thread_count; } void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { - // Wait at a high (but not realtime) priority to reduce latency - // for picking up a new task - update_current_thread_priority(1000); + // Wait at a high (but not realtime) priority to reduce latency + // for picking up a new task + update_current_thread_priority(1000); - while (running.load() || !queue.empty()) { + while (running.load() || !queue.empty()) { auto task = get_task(pool.pool_id); - if (task) { - task->run(); + if (task) { + // This task is about to run in this group, increase the number of active tasks in the group + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + groups.at(task->group_descriptor.group_id)++; } - // Back up to realtime while waiting - update_current_thread_priority(1000); + task->run(); + + // This task is no longer running, decrease the number of active tasks in the group + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + groups.at(task->group_descriptor.group_id)--; + } } + + // Back up to realtime while waiting + update_current_thread_priority(1000); + } } TaskScheduler::TaskScheduler() { @@ -143,6 +159,18 @@ namespace threading { // Check to see if this task was the result of `emit` if (started.load() && task->immediate) { + uint64_t current_pool = util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID; + size_t group_count = 1; + /* mutex scope */ { + const std::lock_guard pool_lock(pool_mutex); + current_pool = pool_map.at(std::this_thread::get_id()); + } + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + group_count = groups.at(task->group_descriptor.group_id); + } + if ((is_runnable(task, current_pool, group_count) + || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, group_count))) { task->run(); return; } @@ -172,8 +200,21 @@ namespace threading { while (running.load() || !queue.empty()) { for (auto it = queue.begin(); it != queue.end(); ++it) { + + size_t group_count = 0; + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + uint64_t group_id = (*it)->group_descriptor.group_id; + if (groups.count(group_id) > 0) { + group_count = groups.at(group_id); + } + else { + groups[group_id] = 0; + } + } + // Check if we can run it - if (is_runnable(*it, pool_id)) { + if (is_runnable(*it, pool_id, group_count)) { // Move the task out of the queue std::unique_ptr task = std::move(*it); diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 460a0eeb7..415591a47 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -27,7 +27,8 @@ #include #include -#include "../util/thread_pool.hpp" +#include "../util/GroupDescriptor.hpp" +#include "../util/ThreadPoolDescriptor.hpp" #include "Reaction.hpp" #include "ReactionTask.hpp" @@ -138,6 +139,9 @@ namespace threading { std::map pools{}; std::map pool_map{}; std::mutex pool_mutex; + + std::map groups{}; + std::mutex group_mutex; }; } // namespace threading diff --git a/src/util/GeneratedCallback.hpp b/src/util/GeneratedCallback.hpp index c72176621..cd0e55892 100644 --- a/src/util/GeneratedCallback.hpp +++ b/src/util/GeneratedCallback.hpp @@ -18,10 +18,11 @@ #ifndef NUCLEAR_UTIL_GENERATEDCALLBACK_HPP #define NUCLEAR_UTIL_GENERATEDCALLBACK_HPP -#include +#include #include "../threading/ReactionTask.hpp" -#include "thread_pool.hpp" +#include "GroupDescriptor.hpp" +#include "ThreadPoolDescriptor.hpp" namespace NUClear { namespace util { @@ -29,12 +30,12 @@ namespace util { struct GeneratedCallback { GeneratedCallback() = default; GeneratedCallback(const int& priority, - const std::type_index& group, + const GroupDescriptor& group, const ThreadPoolDescriptor& pool, threading::ReactionTask::TaskFunction callback) : priority(priority), group(group), pool(pool), callback(std::move(callback)) {} int priority{0}; - std::type_index group{typeid(GeneratedCallback)}; + GroupDescriptor group{0, std::numeric_limits::max()}; ThreadPoolDescriptor pool{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; threading::ReactionTask::TaskFunction callback{}; diff --git a/src/util/GroupDescriptor.hpp b/src/util/GroupDescriptor.hpp new file mode 100644 index 000000000..159fcd95a --- /dev/null +++ b/src/util/GroupDescriptor.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_UTIL_GROUPDESCRIPTOR_HPP +#define NUCLEAR_UTIL_GROUPDESCRIPTOR_HPP + +#include +#include +#include +#include + +namespace NUClear { +namespace util { + + struct GroupDescriptor { + /// @brief Set a unique identifier for this pool + uint64_t group_id{0}; + + /// @brief The number of threads this thread pool will use. + size_t thread_count{std::numeric_limits::max()}; + + static uint64_t get_new_group_id() { + // Make group 0 the default group + static std::atomic source{1}; + return source++; + } + }; + +} // namespace util +} // namespace NUClear + +#endif // NUCLEAR_UTIL_GROUPDESCRIPTOR_HPP diff --git a/src/util/thread_pool.hpp b/src/util/ThreadPoolDescriptor.hpp similarity index 90% rename from src/util/thread_pool.hpp rename to src/util/ThreadPoolDescriptor.hpp index f341be3b4..961b2a738 100644 --- a/src/util/thread_pool.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -25,25 +25,24 @@ namespace NUClear { namespace util { - struct ThreadPoolDescriptor { - /// @brief Set a unique identifier for this pool - uint64_t pool_id; - - /// @brief The number of threads this thread pool will use. - size_t thread_count; - }; - struct ThreadPoolIDSource { - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::atomic source; static constexpr uint64_t MAIN_THREAD_POOL_ID = 0; static constexpr uint64_t DEFAULT_THREAD_POOL_ID = 1; static uint64_t get_new_pool_id() { + static std::atomic source{2}; return source++; } }; + struct ThreadPoolDescriptor { + /// @brief Set a unique identifier for this pool + uint64_t pool_id{ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID}; + + /// @brief The number of threads this thread pool will use. + size_t thread_count{1}; + }; + } // namespace util } // namespace NUClear diff --git a/src/util/thread_pool.cpp b/src/util/thread_pool.cpp deleted file mode 100644 index 472439c67..000000000 --- a/src/util/thread_pool.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "thread_pool.hpp" - -namespace NUClear { -namespace util { - - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::atomic ThreadPoolIDSource::source = {2}; - -} // namespace util -} // namespace NUClear From 5acdda809e677211fe4869bd71fc0a22d0ff36b0 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 14 Jun 2023 17:08:57 +1000 Subject: [PATCH 36/87] Fix task sort order --- src/threading/ReactionTask.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index c29dbe3cc..4ca144cef 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -162,7 +162,7 @@ namespace threading { return a == nullptr ? false : b == nullptr ? true : a->priority == b->priority ? a->id > b->id - : a->priority < b->priority; + : a->priority > b->priority; } // Alias the templated Task so that public API remains intact From 975e27630574fd7a65c2ac85d993ebfea9b4892a Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:51:50 +1000 Subject: [PATCH 37/87] Fix task sort order --- src/threading/ReactionTask.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 4ca144cef..743025efb 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -161,7 +161,7 @@ namespace threading { // tasks created first (smaller ids) should run before tasks created later return a == nullptr ? false : b == nullptr ? true - : a->priority == b->priority ? a->id > b->id + : a->priority == b->priority ? a->id < b->id : a->priority > b->priority; } From 91041a50c3763bf3693d32cd5b4779685e341636 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:52:41 +1000 Subject: [PATCH 38/87] Move task running to separate function --- src/threading/TaskScheduler.cpp | 28 ++++++++++++---------------- src/threading/TaskScheduler.hpp | 1 + 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index bf855fd8d..711816120 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -40,14 +40,7 @@ namespace threading { group_count < task->group_descriptor.thread_count; } - void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { - // Wait at a high (but not realtime) priority to reduce latency - // for picking up a new task - update_current_thread_priority(1000); - - while (running.load() || !queue.empty()) { - auto task = get_task(pool.pool_id); - + void TaskScheduler::run_task(std::unique_ptr&& task) { if (task) { // This task is about to run in this group, increase the number of active tasks in the group /* mutex scope */ { @@ -63,6 +56,15 @@ namespace threading { groups.at(task->group_descriptor.group_id)--; } } + } + + void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { + // Wait at a high (but not realtime) priority to reduce latency + // for picking up a new task + update_current_thread_priority(1000); + + while (running.load() || !queue.empty()) { + run_task(std::move(get_task(pool.pool_id))); // Back up to realtime while waiting update_current_thread_priority(1000); @@ -120,13 +122,7 @@ namespace threading { } // Run main thread tasks - while (running.load() || !queue.empty()) { - auto task = get_task(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID); - - if (task) { - task->run(); - } - } + pool_func(pools.at(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID)); // Now wait for all the threads to finish executing for (auto& thread : threads) { @@ -171,7 +167,7 @@ namespace threading { } if ((is_runnable(task, current_pool, group_count) || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, group_count))) { - task->run(); + run_task(std::move(task)); return; } // Not runnable, stick it in the queue like nothing happened diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 415591a47..2a0ef9c56 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -118,6 +118,7 @@ namespace threading { void create_pool(const util::ThreadPoolDescriptor& pool); void pool_func(const util::ThreadPoolDescriptor& pool); void start_threads(const util::ThreadPoolDescriptor& pool); + void run_task(std::unique_ptr&& task); /// @brief if the scheduler is running std::atomic running{true}; From bf567e57613a2bc90d482ac7f1a0480bfeff0b67 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:52:56 +1000 Subject: [PATCH 39/87] Initialise thread pools to zero threads --- src/util/ThreadPoolDescriptor.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp index 961b2a738..e962b3192 100644 --- a/src/util/ThreadPoolDescriptor.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -40,7 +40,7 @@ namespace util { uint64_t pool_id{ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID}; /// @brief The number of threads this thread pool will use. - size_t thread_count{1}; + size_t thread_count{0}; }; } // namespace util From d5e5f65702aa822dd3096b8e768b627fcaa23ff7 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:53:19 +1000 Subject: [PATCH 40/87] Add labels to single reactions --- tests/dsl/Single.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/dsl/Single.cpp b/tests/dsl/Single.cpp index d7262919f..17522e1e0 100644 --- a/tests/dsl/Single.cpp +++ b/tests/dsl/Single.cpp @@ -47,7 +47,7 @@ class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - on, Single>().then([this](const SimpleMessage1&) { + on, Single>().then("SimpleMessage1", [this](const SimpleMessage1&) { // Increment our run count ++message_count.message1; @@ -67,12 +67,14 @@ class TestReactor : public NUClear::Reactor { powerplant.shutdown(); }); - on, Single>().then([](const SimpleMessage2&) { ++message_count.message2; }); + on, Single>().then("SimpleMessage2", + [](const SimpleMessage2&) { ++message_count.message2; }); on, With, Single>().then( + "SimpleMessage2 With SimpleMessage3", [](const SimpleMessage2&, const SimpleMessage3&) { ++message_count.message3; }); - on().then([this]() { + on().then("Startup", [this]() { // Emit two events, only one should run emit(std::make_unique()); emit(std::make_unique()); From f9656467d89b86e1a7fc54222ae44d78488a85ac Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:53:33 +1000 Subject: [PATCH 41/87] Shutdown before require in main thread --- tests/dsl/MainThread.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/dsl/MainThread.cpp b/tests/dsl/MainThread.cpp index 3e4513598..55387b1cf 100644 --- a/tests/dsl/MainThread.cpp +++ b/tests/dsl/MainThread.cpp @@ -35,10 +35,11 @@ class TestReactor : public NUClear::Reactor { // Run a task with MainTHread and ensure that it is on the main thread on, MainThread>().then("MainThread reaction", [this] { + // Shutdown first so the test will end even if the next check fails + powerplant.shutdown(); + // We should be on the main thread REQUIRE(NUClear::util::main_thread_id == std::this_thread::get_id()); - - powerplant.shutdown(); }); on().then([this]() { From ea2cd2378f7498d2242424431d683213d592c5e6 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:54:07 +1000 Subject: [PATCH 42/87] Use vector of strings for single sync test --- tests/dsl/SingleSync.cpp | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/tests/dsl/SingleSync.cpp b/tests/dsl/SingleSync.cpp index 20dd4efa2..8b7b3dfad 100644 --- a/tests/dsl/SingleSync.cpp +++ b/tests/dsl/SingleSync.cpp @@ -18,6 +18,8 @@ #include #include +#include +#include namespace { @@ -28,25 +30,19 @@ struct Message { struct ShutdownOnIdle {}; -std::atomic counter(0); +std::vector values; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - on, Sync>().then([this](const Message& m) { - // Match message value against counter value - auto value = counter.load(); - - // Increment our counter - ++counter; - - CHECK(value == m.val); - }); + on, Sync>().then( + [](const Message& m) { values.push_back("Received value " + std::to_string(m.val)); }); on, Priority::IDLE>().then([this] { powerplant.shutdown(); }); on().then([this] { + values.clear(); emit(std::make_unique(0)); emit(std::make_unique(1)); emit(std::make_unique(2)); @@ -61,18 +57,12 @@ TEST_CASE("Testing that the Sync priority queue word works correctly", "[api][sy NUClear::PowerPlant::Configuration config; config.thread_count = 2; NUClear::PowerPlant plant(config); - plant.install(); - - // Repeat the test to try and hit the race condition - auto i = GENERATE(Catch::Generators::range(0, 100)); - INFO("Testing iteration " << (i + 1)); - { - plant.start(); - // Reset the counter - auto value = counter.load(); - counter.store(0); + plant.install(); + plant.start(); - REQUIRE(value == 4); + REQUIRE(values.size() == 4); + for (int i = 0; i < 4; ++i) { + CHECK(values[i] == "Received value " + std::to_string(i)); } } From 594304d7db4a99dd268ff7bd99b36090a227aafe Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:54:26 +1000 Subject: [PATCH 43/87] Fix group usage in scheduler --- src/threading/TaskScheduler.cpp | 41 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 711816120..5660402cc 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -41,21 +41,21 @@ namespace threading { } void TaskScheduler::run_task(std::unique_ptr&& task) { - if (task) { - // This task is about to run in this group, increase the number of active tasks in the group - /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); - groups.at(task->group_descriptor.group_id)++; - } + if (task) { + // This task is about to run in this group, increase the number of active tasks in the group + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + groups.at(task->group_descriptor.group_id)++; + } - task->run(); + task->run(); - // This task is no longer running, decrease the number of active tasks in the group - /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); - groups.at(task->group_descriptor.group_id)--; - } + // This task is no longer running, decrease the number of active tasks in the group + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + groups.at(task->group_descriptor.group_id)--; } + } } void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { @@ -153,6 +153,15 @@ namespace threading { // Make sure the pool is created create_pool(task->thread_pool_descriptor); + // Make sure we know about this group + /* mutex scope */ { + const std::lock_guard group_lock(group_mutex); + uint64_t group_id = task->group_descriptor.group_id; + if (groups.count(group_id) == 0) { + groups[group_id] = 0; + } + } + // Check to see if this task was the result of `emit` if (started.load() && task->immediate) { uint64_t current_pool = util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID; @@ -200,13 +209,7 @@ namespace threading { size_t group_count = 0; /* mutex scope */ { const std::lock_guard group_lock(group_mutex); - uint64_t group_id = (*it)->group_descriptor.group_id; - if (groups.count(group_id) > 0) { - group_count = groups.at(group_id); - } - else { - groups[group_id] = 0; - } + group_count = groups.at((*it)->group_descriptor.group_id); } // Check if we can run it From b3e1e1f6e7394eb9d54ba80ba93f02098392c4c0 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 09:59:09 +1000 Subject: [PATCH 44/87] Apply clang-tidy fixes --- src/threading/TaskScheduler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 5660402cc..170dfc945 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -64,7 +64,7 @@ namespace threading { update_current_thread_priority(1000); while (running.load() || !queue.empty()) { - run_task(std::move(get_task(pool.pool_id))); + run_task(get_task(pool.pool_id)); // Back up to realtime while waiting update_current_thread_priority(1000); @@ -156,7 +156,7 @@ namespace threading { // Make sure we know about this group /* mutex scope */ { const std::lock_guard group_lock(group_mutex); - uint64_t group_id = task->group_descriptor.group_id; + const uint64_t group_id = task->group_descriptor.group_id; if (groups.count(group_id) == 0) { groups[group_id] = 0; } From d51511521fa3da6959fc3af515c4792ef1415b05 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Thu, 15 Jun 2023 12:00:00 +1000 Subject: [PATCH 45/87] Fix main thread issue --- src/PowerPlant.hpp | 1 + src/dsl/word/MainThread.hpp | 1 - src/threading/TaskScheduler.cpp | 9 ++++----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 99d8617bb..13c3682b0 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -57,6 +57,7 @@ #include "threading/TaskScheduler.hpp" #include "util/FunctionFusion.hpp" #include "util/demangle.hpp" +#include "util/main_thread_id.hpp" #include "util/unpack.hpp" namespace NUClear { diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index f7170f9f8..8857c5178 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -21,7 +21,6 @@ #include "../../threading/ReactionTask.hpp" #include "../../util/ThreadPoolDescriptor.hpp" -#include "../../util/main_thread_id.hpp" namespace NUClear { namespace dsl { diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 170dfc945..e42fd4db9 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -78,7 +78,8 @@ namespace threading { } void TaskScheduler::start_threads(const util::ThreadPoolDescriptor& pool) { - /* mutex scope */ { + // The main thread never needs to be started + if (pool.pool_id != util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID) { const std::lock_guard threads_lock(threads_mutex); const std::lock_guard pool_lock(pool_mutex); for (size_t i = 0; i < pool.thread_count; ++i) { @@ -115,10 +116,8 @@ namespace threading { started.store(true); // Start all our threads - /* mutex scope */ { - for (const auto& pool : pools) { - start_threads(pool.second); - } + for (const auto& pool : pools) { + start_threads(pool.second); } // Run main thread tasks From c595a35b24c5425ee4d6eb5b98c5844e2832f43c Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 14:53:28 +1000 Subject: [PATCH 46/87] Simplify `Always` code --- src/dsl/word/Always.hpp | 67 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 822337734..16ad31a9a 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -21,6 +21,7 @@ #include #include +#include #include "../../threading/ReactionTask.hpp" #include "../../util/ThreadPoolDescriptor.hpp" @@ -85,42 +86,45 @@ namespace dsl { always_reaction->unbinders.push_back([](threading::Reaction& r) { r.enabled = false; }); - auto idle_callback = [always_reaction](threading::Reaction& idle_reaction) -> util::GeneratedCallback { - // Reaction is still bound re-add the reaction and this - auto callback = [&idle_reaction, always_reaction](threading::ReactionTask& /*task*/) { - // Get new task for the actual reaction - auto always_task = always_reaction->get_task(); - if (always_task) { - // Set the thread pool on the task - always_task->thread_pool_descriptor = Always::pool(always_task->parent); - - // Submit the task to be run - always_reaction->reactor.powerplant.submit(std::move(always_task)); - } - - // Get new task for the actual reaction - auto idle_task = idle_reaction.get_task(); - if (idle_task) { - // Set the thread pool on the task - idle_task->thread_pool_descriptor = Always::pool(idle_task->parent); - - // Only let this task run when the always thread pool is idle - idle_task->priority = Priority::IDLE::value - 1; - - // Submit the task to be run - idle_reaction.reactor.powerplant.submit(std::move(idle_task)); - } - }; - - return {Priority::IDLE::value - 1, DSL::group(idle_reaction), DSL::pool(idle_reaction), callback}; - }; auto idle_reaction = std::make_shared( always_reaction->reactor, std::vector{always_reaction->identifier[0] + " - IDLE Task", always_reaction->identifier[1], always_reaction->identifier[2], always_reaction->identifier[3]}, - idle_callback); + [always_reaction](threading::Reaction& idle_reaction) -> util::GeneratedCallback { + // Reaction is still bound re-add the reaction and this + auto callback = [&idle_reaction, always_reaction](threading::ReactionTask& /*task*/) { + // Get new task for the actual reaction + auto always_task = always_reaction->get_task(); + if (always_task) { + // Submit the task to be run + always_reaction->reactor.powerplant.submit(std::move(always_task)); + } + + // Get new task for the actual reaction + auto idle_task = idle_reaction.get_task(); + if (idle_task) { + // Set the thread pool on the task + idle_task->thread_pool_descriptor = DSL::pool(*always_reaction); + + // Only let this task run when the always thread pool is idle + idle_task->priority = Priority::IDLE::value - 1; + + // Submit the task to be run + idle_reaction.reactor.powerplant.submit(std::move(idle_task)); + } + }; + + // Make sure that idle reaction always has lower priority than the always reaction + return {DSL::priority(*always_reaction) - 1, + DSL::group(*always_reaction), + DSL::pool(*always_reaction), + callback}; + }); + + // Don't emit stats for the idle reaction + idle_reaction->emit_stats = false; // Keep this reaction handy so it doesn't go out of scope reaction_store[always_reaction->id] = {always_reaction, idle_reaction}; @@ -142,9 +146,6 @@ namespace dsl { static inline void postcondition(threading::ReactionTask& task) { auto new_task = task.parent.get_task(); if (new_task) { - // Set the thread pool on the task - new_task->thread_pool_descriptor = Always::pool(new_task->parent); - // Submit the task to be run task.parent.reactor.powerplant.submit(std::move(new_task)); } From 3fdecbdbed258cbc0044d3e28fea1418924b183f Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 14:53:53 +1000 Subject: [PATCH 47/87] Fix group descriptor generation --- src/dsl/word/Group.hpp | 21 +++++++++++++++++++-- src/dsl/word/Sync.hpp | 21 ++++++++++++++++----- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/dsl/word/Group.hpp b/src/dsl/word/Group.hpp index 9f1ba0987..22e02d654 100644 --- a/src/dsl/word/Group.hpp +++ b/src/dsl/word/Group.hpp @@ -18,6 +18,10 @@ #ifndef NUCLEAR_DSL_WORD_GROUP_HPP #define NUCLEAR_DSL_WORD_GROUP_HPP +#include +#include +#include + #include "../../threading/ReactionTask.hpp" #include "../../util/GroupDescriptor.hpp" @@ -28,13 +32,26 @@ namespace dsl { template struct Group { + static std::map group_id; + static std::mutex mutex; + template static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - const static uint64_t group_id = util::GroupDescriptor::get_new_group_id(); - return util::GroupDescriptor{group_id, GroupType::concurrency}; + + const std::lock_guard lock(mutex); + if (group_id.count(typeid(GroupType)) == 0) { + group_id[typeid(GroupType)] = util::GroupDescriptor::get_new_group_id(); + } + return util::GroupDescriptor{group_id[typeid(GroupType)], 1}; } }; + // Initialise the static map and mutex + template + std::map Group::group_id; + template + std::mutex Group::mutex; + } // namespace word } // namespace dsl } // namespace NUClear diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index cd7b2f662..e4fdb69ea 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -19,10 +19,8 @@ #ifndef NUCLEAR_DSL_WORD_SYNC_HPP #define NUCLEAR_DSL_WORD_SYNC_HPP -#include -#include +#include #include -#include #include #include "../../threading/ReactionTask.hpp" @@ -72,13 +70,26 @@ namespace dsl { template struct Sync { + static std::map group_id; + static std::mutex mutex; + template static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - const static uint64_t group_id = util::GroupDescriptor::get_new_group_id(); - return util::GroupDescriptor{group_id, 1}; + + const std::lock_guard lock(mutex); + if (group_id.count(typeid(SyncGroup)) == 0) { + group_id[typeid(SyncGroup)] = util::GroupDescriptor::get_new_group_id(); + } + return util::GroupDescriptor{group_id[typeid(SyncGroup)], 1}; } }; + // Initialise the static map and mutex + template + std::map Sync::group_id; + template + std::mutex Sync::mutex; + } // namespace word } // namespace dsl } // namespace NUClear From f0e81b8e4d075b8830b58416e23b4d4f8355ecaf Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 14:54:07 +1000 Subject: [PATCH 48/87] Fix pool descriptor generation --- src/dsl/word/Pool.hpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index edb6db365..40e3ca1b4 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -18,6 +18,10 @@ #ifndef NUCLEAR_DSL_WORD_POOL_HPP #define NUCLEAR_DSL_WORD_POOL_HPP +#include +#include +#include + #include "../../threading/ReactionTask.hpp" #include "../../util/ThreadPoolDescriptor.hpp" @@ -27,13 +31,26 @@ namespace dsl { template struct Pool { + static std::map pool_id; + static std::mutex mutex; + template static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - const static uint64_t pool_id = util::ThreadPoolIDSource::get_new_pool_id(); - return util::ThreadPoolDescriptor{pool_id, PoolType::concurrency}; + + const std::lock_guard lock(mutex); + if (pool_id.count(typeid(PoolType)) == 0) { + pool_id[typeid(PoolType)] = util::ThreadPoolIDSource::get_new_pool_id(); + } + return util::ThreadPoolDescriptor{pool_id[typeid(PoolType)], PoolType::concurrency}; } }; + // Initialise the static map and mutex + template + std::map Pool::pool_id; + template + std::mutex Pool::mutex; + } // namespace word } // namespace dsl } // namespace NUClear From 0dd75b620c9ee95922ace902d031b35ccd4624f7 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 14:54:51 +1000 Subject: [PATCH 49/87] Fix log test --- tests/log/Log.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/log/Log.cpp b/tests/log/Log.cpp index a8fa92938..faece41b6 100644 --- a/tests/log/Log.cpp +++ b/tests/log/Log.cpp @@ -165,7 +165,6 @@ TEST_CASE("Testing the Log<>() function", "[api][log]") { expected_count += levels.size() * (levels.size() + 1) / 2; // Direct reaction logs expected_count += levels.size() * (levels.size() + 1) / 2; // Indirect reaction logs expected_count += levels.size() * levels.size(); // Non reaction logs - expected_count += 2 + levels.size(); // Post shutdown logs REQUIRE(messages.size() == expected_count); // Test that each of the messages are correct for each log level @@ -198,24 +197,4 @@ TEST_CASE("Testing the Log<>() function", "[api][log]") { REQUIRE_FALSE(messages[i++].from_reaction); } } - - // Test post-shutdown logs - { - const std::string expected = "Post Powerplant Shutdown " + std::to_string(NUClear::FATAL); - REQUIRE(messages[i].message == expected); - REQUIRE(messages[i].level == NUClear::FATAL); - REQUIRE(messages[i++].from_reaction); - REQUIRE(messages[i].message == expected); - REQUIRE(messages[i].level == NUClear::FATAL); - REQUIRE(messages[i++].from_reaction); - } - - // Test logs from free floating functions - for (const auto& log_level : levels) { - // No filter here, free floating prints everything - const std::string expected = "Non Reaction " + std::to_string(log_level); - REQUIRE(messages[i].message == expected); - REQUIRE(messages[i].level == log_level); - REQUIRE_FALSE(messages[i++].from_reaction); - } } From 2ebcfe70bbc42b5fb0cd6d0b0ee4b773819b3fb8 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 14:57:05 +1000 Subject: [PATCH 50/87] Create a task queue per thread pool --- src/threading/TaskScheduler.cpp | 93 +++++++++++++++++++++++---------- src/threading/TaskScheduler.hpp | 6 +-- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index e42fd4db9..f54cac332 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -59,22 +59,25 @@ namespace threading { } void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { - // Wait at a high (but not realtime) priority to reduce latency - // for picking up a new task - update_current_thread_priority(1000); + while (running.load() || !queue.at(pool.pool_id).empty()) { + // Wait at a high (but not realtime) priority to reduce latency + // for picking up a new task + update_current_thread_priority(1000); - while (running.load() || !queue.empty()) { run_task(get_task(pool.pool_id)); - - // Back up to realtime while waiting - update_current_thread_priority(1000); } } TaskScheduler::TaskScheduler() { + // Setup everything (thread pool, task queue, mutex, condition variable) for the main thread here pools[util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID] = util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; pool_map[std::this_thread::get_id()] = util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID; + + queue.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, std::vector>{}); + queue_mutex.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, std::make_unique()); + queue_condition.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, + std::make_unique()); } void TaskScheduler::start_threads(const util::ThreadPoolDescriptor& pool) { @@ -99,6 +102,20 @@ namespace threading { // Make a copy of the pool descriptor pools[pool.pool_id] = util::ThreadPoolDescriptor{pool.pool_id, pool.thread_count}; + + // Make sure the mutex and condition variable are created for this pool + if (queue_mutex.count(pool.pool_id) == 0) { + queue_mutex.emplace(pool.pool_id, std::make_unique()); + queue_condition.emplace(pool.pool_id, std::make_unique()); + } + } + + // Make sure the task queue is created for this pool + /* mutex scope */ { + const std::lock_guard queue_lock(*queue_mutex[pool.pool_id]); + if (queue.count(pool.pool_id) == 0) { + queue.emplace(pool.pool_id, std::vector>{}); + } } // If the scheduler has not yet started then don't start the threads for this pool yet @@ -138,72 +155,92 @@ namespace threading { } void TaskScheduler::shutdown() { - const std::lock_guard lock(mutex); - running.store(false); started.store(false); - condition.notify_all(); + running.store(false); + for (auto& mutex : queue_mutex) { + const std::lock_guard queue_lock(*mutex.second); + queue_condition.at(mutex.first)->notify_all(); + } } void TaskScheduler::submit(std::unique_ptr&& task) { + // Extract the thread pool descriptor from the current task + const util::ThreadPoolDescriptor current_pool = task->thread_pool_descriptor; + // We do not accept new tasks once we are shutdown if (running.load()) { // Make sure the pool is created - create_pool(task->thread_pool_descriptor); + create_pool(current_pool); // Make sure we know about this group /* mutex scope */ { const std::lock_guard group_lock(group_mutex); const uint64_t group_id = task->group_descriptor.group_id; if (groups.count(group_id) == 0) { - groups[group_id] = 0; + groups.emplace(group_id, 0); } } // Check to see if this task was the result of `emit` if (started.load() && task->immediate) { - uint64_t current_pool = util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID; - size_t group_count = 1; + // Map the current thread to the thread pool it belongs to + uint64_t thread_pool = util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID; /* mutex scope */ { const std::lock_guard pool_lock(pool_mutex); - current_pool = pool_map.at(std::this_thread::get_id()); + if (pool_map.count(std::this_thread::get_id()) > 0) { + thread_pool = pool_map.at(std::this_thread::get_id()); + } } + + size_t group_count = 0; /* mutex scope */ { const std::lock_guard group_lock(group_mutex); - group_count = groups.at(task->group_descriptor.group_id); + const uint64_t group_id = task->group_descriptor.group_id; + group_count = groups.at(group_id); } - if ((is_runnable(task, current_pool, group_count) + + // Because this is a direct emit we allow it to run on the default thread pool if + // (a) the default thread pool isn't the thread pool of the calling thread, and + // (b) the default thread pool has a spare thread + // + // If this task is not immediately runnable (neither thread pool has a spare thread and the group is + // already at full concurrency) then this task is just queued up like all of the other non-immediate + // tasks + if ((is_runnable(task, thread_pool, group_count) || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, group_count))) { run_task(std::move(task)); return; } - // Not runnable, stick it in the queue like nothing happened } /* Mutex Scope */ { - const std::lock_guard lock(mutex); + const std::lock_guard queue_lock(*queue_mutex.at(current_pool.pool_id)); // Find where to insert the new task to maintain task order - auto it = std::lower_bound(queue.begin(), queue.end(), task, std::less<>()); + auto it = std::lower_bound(queue.at(current_pool.pool_id).begin(), + queue.at(current_pool.pool_id).end(), + task, + std::less<>()); // Insert before the found position - queue.insert(it, std::forward>(task)); + queue.at(current_pool.pool_id).insert(it, std::forward>(task)); } } // Notify all threads that there is a new task to be processed - const std::lock_guard lock(mutex); - condition.notify_all(); + const std::lock_guard queue_lock(*queue_mutex.at(current_pool.pool_id)); + queue_condition.at(current_pool.pool_id)->notify_all(); } std::unique_ptr TaskScheduler::get_task(const uint64_t& pool_id) { - std::unique_lock lock(mutex); + std::unique_lock queue_lock(*queue_mutex.at(pool_id)); - while (running.load() || !queue.empty()) { + while (running.load() || !queue[pool_id].empty()) { - for (auto it = queue.begin(); it != queue.end(); ++it) { + for (auto it = queue.at(pool_id).begin(); it != queue.at(pool_id).end(); ++it) { size_t group_count = 0; /* mutex scope */ { @@ -217,7 +254,7 @@ namespace threading { std::unique_ptr task = std::move(*it); // Erase the old position in the queue - queue.erase(it); + queue.at(pool_id).erase(it); // Return the task return task; @@ -225,7 +262,7 @@ namespace threading { } // Wait for something to happen! - condition.wait(lock); + queue_condition.at(pool_id)->wait(queue_lock); } return nullptr; diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 2a0ef9c56..a35d76c13 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -125,12 +125,12 @@ namespace threading { std::atomic started{false}; /// @brief our queue which sorts tasks by priority - std::vector> queue; + std::map>> queue; /// @brief the mutex which our threads synchronize their access to this object - std::mutex mutex; + std::map> queue_mutex; /// @brief the condition object that threads wait on if they can't get a task - std::condition_variable condition; + std::map> queue_condition; /// @brief A vector of the running threads in the system std::vector> threads; From cfb3a04ca0d18617d68d44700043f8439fd2eba9 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 15:27:54 +1000 Subject: [PATCH 51/87] Silence clang-tidy --- src/dsl/word/Group.hpp | 4 ++++ src/dsl/word/Pool.hpp | 4 ++++ src/dsl/word/Sync.hpp | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/dsl/word/Group.hpp b/src/dsl/word/Group.hpp index 22e02d654..d6809335e 100644 --- a/src/dsl/word/Group.hpp +++ b/src/dsl/word/Group.hpp @@ -32,7 +32,9 @@ namespace dsl { template struct Group { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::map group_id; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::mutex mutex; template @@ -48,8 +50,10 @@ namespace dsl { // Initialise the static map and mutex template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::map Group::group_id; template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::mutex Group::mutex; } // namespace word diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index 40e3ca1b4..52e600fa1 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -31,7 +31,9 @@ namespace dsl { template struct Pool { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::map pool_id; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::mutex mutex; template @@ -47,8 +49,10 @@ namespace dsl { // Initialise the static map and mutex template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::map Pool::pool_id; template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::mutex Pool::mutex; } // namespace word diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index e4fdb69ea..3f13deac9 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -70,7 +70,9 @@ namespace dsl { template struct Sync { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::map group_id; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static std::mutex mutex; template @@ -86,8 +88,10 @@ namespace dsl { // Initialise the static map and mutex template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::map Sync::group_id; template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::mutex Sync::mutex; } // namespace word From b7c6ef5c46e72cbc321adab14dc95a7324672f79 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 16:07:12 +1000 Subject: [PATCH 52/87] Keep running even if one gcc job fails --- .github/workflows/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c7227e7d1..3672ec5b0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -28,6 +28,7 @@ jobs: # The type of runner that the job will run on runs-on: ubuntu-latest + continue-on-error: true # Use the container for this specific version of gcc container: ${{ matrix.container }} From c4724410925041a7570e3adf664ca7b4b5d3d6e0 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 16:17:15 +1000 Subject: [PATCH 53/87] Move thread pool descriptor constant initalisers to cpp file --- src/util/ThreadPoolDescriptor.cpp | 27 +++++++++++++++++++++++++++ src/util/ThreadPoolDescriptor.hpp | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 src/util/ThreadPoolDescriptor.cpp diff --git a/src/util/ThreadPoolDescriptor.cpp b/src/util/ThreadPoolDescriptor.cpp new file mode 100644 index 000000000..71e76c770 --- /dev/null +++ b/src/util/ThreadPoolDescriptor.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "ThreadPoolDescriptor.hpp" + +namespace NUClear { +namespace util { + + const uint64_t ThreadPoolIDSource::MAIN_THREAD_POOL_ID = 0; + const uint64_t ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID = 1; + +} // namespace util +} // namespace NUClear diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp index e962b3192..4fdc2983c 100644 --- a/src/util/ThreadPoolDescriptor.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -26,8 +26,8 @@ namespace NUClear { namespace util { struct ThreadPoolIDSource { - static constexpr uint64_t MAIN_THREAD_POOL_ID = 0; - static constexpr uint64_t DEFAULT_THREAD_POOL_ID = 1; + static const uint64_t MAIN_THREAD_POOL_ID; + static const uint64_t DEFAULT_THREAD_POOL_ID; static uint64_t get_new_pool_id() { static std::atomic source{2}; From 8003149f2d153f816afa38970ded66c568d86524 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 16 Jun 2023 16:28:30 +1000 Subject: [PATCH 54/87] Don't use `std::enable_if_t` --- src/util/CallbackGenerator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index b420e06c0..5cc11c6df 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -46,7 +46,7 @@ namespace util { struct CallbackGenerator { // Don't using this constructor is F is of type CallbackGenerator - template ::value, bool> = true> + template ::value, bool>::type = true> CallbackGenerator(F&& callback) : callback(std::forward(callback)) , transients(std::make_shared::type>()) {} From 7028145c79dace1b74de642a08e50438cb969023 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Mon, 19 Jun 2023 09:12:10 +1000 Subject: [PATCH 55/87] Apply `remove_cvref` to `enable_if` call --- src/util/CallbackGenerator.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 5cc11c6df..822088af9 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_UTIL_CALLBACKGENERATOR_HPP #define NUCLEAR_UTIL_CALLBACKGENERATOR_HPP +#include + #include "../dsl/trait/is_transient.hpp" #include "../dsl/word/emit/Direct.hpp" #include "../util/GeneratedCallback.hpp" @@ -46,7 +48,11 @@ namespace util { struct CallbackGenerator { // Don't using this constructor is F is of type CallbackGenerator - template ::value, bool>::type = true> + template ::type>::type, + CallbackGenerator>::value, + bool>::type = true> CallbackGenerator(F&& callback) : callback(std::forward(callback)) , transients(std::make_shared::type>()) {} From 0429f98557b28922cbacc9b902eab35dad57231b Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Mon, 19 Jun 2023 16:13:17 +1000 Subject: [PATCH 56/87] Increase group count earlier --- src/threading/TaskScheduler.cpp | 30 +++++++++++++----------------- src/threading/TaskScheduler.hpp | 3 ++- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index f54cac332..7fe459fce 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -32,22 +32,24 @@ namespace NUClear { namespace threading { - bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id, const size_t& group_count) { - return + bool TaskScheduler::is_runnable(const std::unique_ptr& task, const uint64_t& pool_id) { + // Task can run if it is meant to run on the current thread pool - task->thread_pool_descriptor.pool_id == pool_id && - // Task can run if the group is belongs to has spare threads - group_count < task->group_descriptor.thread_count; - } + const bool correct_pool = pool_id == task->thread_pool_descriptor.pool_id; - void TaskScheduler::run_task(std::unique_ptr&& task) { - if (task) { + // Task can run if the group it belongs to has spare threads + const std::lock_guard group_lock(group_mutex); + if (groups.at(task->group_descriptor.group_id) < task->group_descriptor.thread_count && correct_pool) { // This task is about to run in this group, increase the number of active tasks in the group - /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); groups.at(task->group_descriptor.group_id)++; + return true; } + return false; + } + + void TaskScheduler::run_task(std::unique_ptr&& task) { + if (task) { task->run(); // This task is no longer running, decrease the number of active tasks in the group @@ -242,14 +244,8 @@ namespace threading { for (auto it = queue.at(pool_id).begin(); it != queue.at(pool_id).end(); ++it) { - size_t group_count = 0; - /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); - group_count = groups.at((*it)->group_descriptor.group_id); - } - // Check if we can run it - if (is_runnable(*it, pool_id, group_count)) { + if (is_runnable(*it, pool_id)) { // Move the task out of the queue std::unique_ptr task = std::move(*it); diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index a35d76c13..37a936ee7 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -102,6 +102,7 @@ namespace threading { */ void submit(std::unique_ptr&& task); + private: /** * @brief Get a task object to be executed by a thread. * @@ -114,11 +115,11 @@ namespace threading { */ std::unique_ptr get_task(const uint64_t& pool_id); - private: void create_pool(const util::ThreadPoolDescriptor& pool); void pool_func(const util::ThreadPoolDescriptor& pool); void start_threads(const util::ThreadPoolDescriptor& pool); void run_task(std::unique_ptr&& task); + bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id); /// @brief if the scheduler is running std::atomic running{true}; From 7f5fe1824e24aa3c6880f3293cddd7a50e5d3adb Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Mon, 19 Jun 2023 16:13:49 +1000 Subject: [PATCH 57/87] Rewrite immediate logic --- src/threading/TaskScheduler.cpp | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 7fe459fce..a5bdb65cf 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -186,6 +186,7 @@ namespace threading { } // Check to see if this task was the result of `emit` + // If we aren't already started then just queue the task up to run when we are started if (started.load() && task->immediate) { // Map the current thread to the thread pool it belongs to uint64_t thread_pool = util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID; @@ -194,24 +195,14 @@ namespace threading { if (pool_map.count(std::this_thread::get_id()) > 0) { thread_pool = pool_map.at(std::this_thread::get_id()); } + else { + throw std::runtime_error("Task submitted by unknown thread"); } - - size_t group_count = 0; - /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); - const uint64_t group_id = task->group_descriptor.group_id; - group_count = groups.at(group_id); } - // Because this is a direct emit we allow it to run on the default thread pool if - // (a) the default thread pool isn't the thread pool of the calling thread, and - // (b) the default thread pool has a spare thread - // - // If this task is not immediately runnable (neither thread pool has a spare thread and the group is - // already at full concurrency) then this task is just queued up like all of the other non-immediate - // tasks - if ((is_runnable(task, thread_pool, group_count) - || is_runnable(task, util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, group_count))) { + // Check to see if this task is runnable in the current thread + // If it isn't we can just queue it up with all of the other non-immediate task + if (is_runnable(task, thread_pool)) { run_task(std::move(task)); return; } From ceeb1163b6ad75da845c94ec44056d3bb43f7c55 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Mon, 19 Jun 2023 16:14:27 +1000 Subject: [PATCH 58/87] Poke thread pools before joining --- src/threading/TaskScheduler.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index a5bdb65cf..58b33d695 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -34,16 +34,16 @@ namespace threading { bool TaskScheduler::is_runnable(const std::unique_ptr& task, const uint64_t& pool_id) { - // Task can run if it is meant to run on the current thread pool + // Task can run if it is meant to run on the current thread pool const bool correct_pool = pool_id == task->thread_pool_descriptor.pool_id; // Task can run if the group it belongs to has spare threads const std::lock_guard group_lock(group_mutex); if (groups.at(task->group_descriptor.group_id) < task->group_descriptor.thread_count && correct_pool) { // This task is about to run in this group, increase the number of active tasks in the group - groups.at(task->group_descriptor.group_id)++; + groups.at(task->group_descriptor.group_id)++; return true; - } + } return false; } @@ -146,6 +146,15 @@ namespace threading { for (auto& thread : threads) { try { if (thread->joinable()) { + // Poke the thread pool that this thread belongs to to make sure it has woken up + /* mutex scope */ { + const std::lock_guard pool_lock(pool_mutex); + if (pool_map.count(thread->get_id()) > 0) { + const uint64_t thread_pool = pool_map.at(thread->get_id()); + const std::lock_guard queue_lock(*queue_mutex.at(thread_pool)); + queue_condition.at(thread_pool)->notify_all(); + } + } thread->join(); } } @@ -197,7 +206,7 @@ namespace threading { } else { throw std::runtime_error("Task submitted by unknown thread"); - } + } } // Check to see if this task is runnable in the current thread From 82ca926684f40c556cb7c61cf3aeaacbc0f139aa Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Mon, 19 Jun 2023 16:14:36 +1000 Subject: [PATCH 59/87] Fix comment --- src/util/CallbackGenerator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 822088af9..3ab3e9181 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -47,7 +47,7 @@ namespace util { template struct CallbackGenerator { - // Don't using this constructor is F is of type CallbackGenerator + // Don't use this constructor if F is of type CallbackGenerator template ::type>::type, From d7172f25da32c51ff3119590d2bc36c4d35f3500 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Mon, 19 Jun 2023 16:38:53 +1000 Subject: [PATCH 60/87] Fix segfault --- src/threading/TaskScheduler.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 58b33d695..891763c24 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -204,9 +204,6 @@ namespace threading { if (pool_map.count(std::this_thread::get_id()) > 0) { thread_pool = pool_map.at(std::this_thread::get_id()); } - else { - throw std::runtime_error("Task submitted by unknown thread"); - } } // Check to see if this task is runnable in the current thread From 14112a21fb1a5e8f9ba9b10ea481f3ddf837a89a Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 15:30:35 +1000 Subject: [PATCH 61/87] Reduce number of mutexes --- src/threading/TaskScheduler.cpp | 38 +++++++++++---------------------- src/threading/TaskScheduler.hpp | 8 ++----- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 891763c24..126a2f211 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -38,7 +38,6 @@ namespace threading { const bool correct_pool = pool_id == task->thread_pool_descriptor.pool_id; // Task can run if the group it belongs to has spare threads - const std::lock_guard group_lock(group_mutex); if (groups.at(task->group_descriptor.group_id) < task->group_descriptor.thread_count && correct_pool) { // This task is about to run in this group, increase the number of active tasks in the group groups.at(task->group_descriptor.group_id)++; @@ -54,7 +53,7 @@ namespace threading { // This task is no longer running, decrease the number of active tasks in the group /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); + const std::lock_guard group_lock(queue_mutex); groups.at(task->group_descriptor.group_id)--; } } @@ -77,15 +76,11 @@ namespace threading { pool_map[std::this_thread::get_id()] = util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID; queue.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, std::vector>{}); - queue_mutex.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, std::make_unique()); - queue_condition.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, - std::make_unique()); } void TaskScheduler::start_threads(const util::ThreadPoolDescriptor& pool) { // The main thread never needs to be started if (pool.pool_id != util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID) { - const std::lock_guard threads_lock(threads_mutex); const std::lock_guard pool_lock(pool_mutex); for (size_t i = 0; i < pool.thread_count; ++i) { threads.push_back(std::make_unique(&TaskScheduler::pool_func, this, pool)); @@ -104,17 +99,11 @@ namespace threading { // Make a copy of the pool descriptor pools[pool.pool_id] = util::ThreadPoolDescriptor{pool.pool_id, pool.thread_count}; - - // Make sure the mutex and condition variable are created for this pool - if (queue_mutex.count(pool.pool_id) == 0) { - queue_mutex.emplace(pool.pool_id, std::make_unique()); - queue_condition.emplace(pool.pool_id, std::make_unique()); - } } // Make sure the task queue is created for this pool /* mutex scope */ { - const std::lock_guard queue_lock(*queue_mutex[pool.pool_id]); + const std::lock_guard queue_lock(queue_mutex); if (queue.count(pool.pool_id) == 0) { queue.emplace(pool.pool_id, std::vector>{}); } @@ -150,9 +139,8 @@ namespace threading { /* mutex scope */ { const std::lock_guard pool_lock(pool_mutex); if (pool_map.count(thread->get_id()) > 0) { - const uint64_t thread_pool = pool_map.at(thread->get_id()); - const std::lock_guard queue_lock(*queue_mutex.at(thread_pool)); - queue_condition.at(thread_pool)->notify_all(); + const std::lock_guard queue_lock(queue_mutex); + queue_condition.notify_all(); } } thread->join(); @@ -168,10 +156,8 @@ namespace threading { void TaskScheduler::shutdown() { started.store(false); running.store(false); - for (auto& mutex : queue_mutex) { - const std::lock_guard queue_lock(*mutex.second); - queue_condition.at(mutex.first)->notify_all(); - } + const std::lock_guard queue_lock(queue_mutex); + queue_condition.notify_all(); } void TaskScheduler::submit(std::unique_ptr&& task) { @@ -187,7 +173,7 @@ namespace threading { // Make sure we know about this group /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); + const std::lock_guard group_lock(queue_mutex); const uint64_t group_id = task->group_descriptor.group_id; if (groups.count(group_id) == 0) { groups.emplace(group_id, 0); @@ -215,7 +201,7 @@ namespace threading { } /* Mutex Scope */ { - const std::lock_guard queue_lock(*queue_mutex.at(current_pool.pool_id)); + const std::lock_guard queue_lock(queue_mutex); // Find where to insert the new task to maintain task order auto it = std::lower_bound(queue.at(current_pool.pool_id).begin(), @@ -229,13 +215,13 @@ namespace threading { } // Notify all threads that there is a new task to be processed - const std::lock_guard queue_lock(*queue_mutex.at(current_pool.pool_id)); - queue_condition.at(current_pool.pool_id)->notify_all(); + const std::lock_guard queue_lock(queue_mutex); + queue_condition.notify_all(); } std::unique_ptr TaskScheduler::get_task(const uint64_t& pool_id) { - std::unique_lock queue_lock(*queue_mutex.at(pool_id)); + std::unique_lock queue_lock(queue_mutex); while (running.load() || !queue[pool_id].empty()) { @@ -255,7 +241,7 @@ namespace threading { } // Wait for something to happen! - queue_condition.at(pool_id)->wait(queue_lock); + queue_condition.wait(queue_lock); } return nullptr; diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 37a936ee7..cdecbf8f9 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -129,21 +129,17 @@ namespace threading { std::map>> queue; /// @brief the mutex which our threads synchronize their access to this object - std::map> queue_mutex; + std::mutex queue_mutex; /// @brief the condition object that threads wait on if they can't get a task - std::map> queue_condition; + std::condition_variable queue_condition; /// @brief A vector of the running threads in the system std::vector> threads; - /// @brief the mutex which our threads synchronize their access to this object - std::mutex threads_mutex; - std::map pools{}; std::map pool_map{}; std::mutex pool_mutex; std::map groups{}; - std::mutex group_mutex; }; } // namespace threading From e0295407887074b03a9a64ccdb22e09546c59e65 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 15:32:19 +1000 Subject: [PATCH 62/87] Fix up comments for always --- src/dsl/word/Always.hpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 16ad31a9a..199da8b6a 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -1,6 +1,7 @@ /* * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston + * 2014-2022 Trent Houliston + * 2023 Trent Houliston , Alex Biddulph * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the @@ -93,23 +94,21 @@ namespace dsl { always_reaction->identifier[2], always_reaction->identifier[3]}, [always_reaction](threading::Reaction& idle_reaction) -> util::GeneratedCallback { - // Reaction is still bound re-add the reaction and this auto callback = [&idle_reaction, always_reaction](threading::ReactionTask& /*task*/) { - // Get new task for the actual reaction + // Get a task for the always reaction and submit it to the scheduler auto always_task = always_reaction->get_task(); if (always_task) { - // Submit the task to be run always_reaction->reactor.powerplant.submit(std::move(always_task)); } - // Get new task for the actual reaction + // Get a task for the idle reaction and submit it to the scheduler auto idle_task = idle_reaction.get_task(); if (idle_task) { // Set the thread pool on the task idle_task->thread_pool_descriptor = DSL::pool(*always_reaction); - // Only let this task run when the always thread pool is idle - idle_task->priority = Priority::IDLE::value - 1; + // Make sure that idle reaction always has lower priority than the always reaction + idle_task->priority = DSL::priority(*always_reaction) - 1; // Submit the task to be run idle_reaction.reactor.powerplant.submit(std::move(idle_task)); @@ -144,9 +143,9 @@ namespace dsl { template static inline void postcondition(threading::ReactionTask& task) { + // Get a task for the always reaction and submit it to the scheduler auto new_task = task.parent.get_task(); if (new_task) { - // Submit the task to be run task.parent.reactor.powerplant.submit(std::move(new_task)); } } From b3a266e390d7626dbeccd51326d036349f93f6b7 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 15:32:33 +1000 Subject: [PATCH 63/87] Fix unbinder for always --- src/dsl/word/Always.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 199da8b6a..8742c1475 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -85,8 +85,6 @@ namespace dsl { std::pair, std::shared_ptr>> reaction_store = {}; - always_reaction->unbinders.push_back([](threading::Reaction& r) { r.enabled = false; }); - auto idle_reaction = std::make_shared( always_reaction->reactor, std::vector{always_reaction->identifier[0] + " - IDLE Task", @@ -128,6 +126,12 @@ namespace dsl { // Keep this reaction handy so it doesn't go out of scope reaction_store[always_reaction->id] = {always_reaction, idle_reaction}; + // Create an unbinder for the always reaction + always_reaction->unbinders.push_back([](threading::Reaction& r) { + r.enabled = false; + reaction_store.erase(r.id); + }); + // Get a task for the always reaction and submit it to the scheduler auto always_task = always_reaction->get_task(); if (always_task) { From dade557745c2b2b60f26ffadb2672f48ff6d92e4 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:08:11 +1000 Subject: [PATCH 64/87] Add labels to reactions --- tests/dsl/SingleSync.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/dsl/SingleSync.cpp b/tests/dsl/SingleSync.cpp index 8b7b3dfad..614e349f9 100644 --- a/tests/dsl/SingleSync.cpp +++ b/tests/dsl/SingleSync.cpp @@ -36,12 +36,13 @@ class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - on, Sync>().then( - [](const Message& m) { values.push_back("Received value " + std::to_string(m.val)); }); + on, Sync>().then("SyncReaction", [](const Message& m) { + values.push_back("Received value " + std::to_string(m.val)); + }); - on, Priority::IDLE>().then([this] { powerplant.shutdown(); }); + on, Priority::IDLE>().then("ShutdownOnIdle", [this] { powerplant.shutdown(); }); - on().then([this] { + on().then("Startup", [this] { values.clear(); emit(std::make_unique(0)); emit(std::make_unique(1)); From 920ad5344ae67d7f6c6a2682ef620f8aa054c6df Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:17:03 +1000 Subject: [PATCH 65/87] Fix comment reflow --- src/dsl/word/Buffer.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/dsl/word/Buffer.hpp b/src/dsl/word/Buffer.hpp index 07a402702..470bbc2a7 100644 --- a/src/dsl/word/Buffer.hpp +++ b/src/dsl/word/Buffer.hpp @@ -31,10 +31,9 @@ namespace dsl { * * @details * @code on, Buffer>>() @endcode - * In the case above, when the subscribing reaction is triggered, should there be less than n - * existing tasks associated with this reaction (either executing or in the queue), then a new task will be - * created and scheduled. However, should n tasks already be allocated, then this new task request - * will be ignored. + * In the case above, when the subscribing reaction is triggered, should there be less than n existing + * tasks associated with this reaction (either executing or in the queue), then a new task will be created and + * scheduled. However, should n tasks already be allocated, then this new task request will be ignored. * * For best use, this word should be fused with at least one other binding DSL word. * From 78104e7d7a36ed3adc060f81b6d38f97aa70e4bb Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:38:31 +1000 Subject: [PATCH 66/87] Fix comment --- tests/dsl/MainThread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/MainThread.cpp b/tests/dsl/MainThread.cpp index 55387b1cf..b7f1d1ec6 100644 --- a/tests/dsl/MainThread.cpp +++ b/tests/dsl/MainThread.cpp @@ -33,7 +33,7 @@ class TestReactor : public NUClear::Reactor { emit(std::make_unique(1.1)); }); - // Run a task with MainTHread and ensure that it is on the main thread + // Run a task with MainThread and ensure that it is on the main thread on, MainThread>().then("MainThread reaction", [this] { // Shutdown first so the test will end even if the next check fails powerplant.shutdown(); From f3e4b44f6b61c9ecdb15605df3dc4a325d088279 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:48:58 +1000 Subject: [PATCH 67/87] Simplify diff in ChronoController --- src/extension/ChronoController.hpp | 64 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 040aaec28..71e5e21fe 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -77,50 +77,52 @@ namespace extension { // Acquire the mutex lock so we can wait on it std::unique_lock lock(mutex); - if (running) { - // If we have tasks to do - if (!tasks.empty()) { + if (!running) { + return; + } - // Make the list into a heap so we can remove the soonest ones - std::make_heap(tasks.begin(), tasks.end(), std::greater<>()); + // If we have tasks to do + if (!tasks.empty()) { - // If we are within the wait offset of the time, spinlock until we get there for greater - // accuracy - if (NUClear::clock::now() + wait_offset > tasks.front().time) { + // Make the list into a heap so we can remove the soonest ones + std::make_heap(tasks.begin(), tasks.end(), std::greater<>()); - // Spinlock! - while (NUClear::clock::now() < tasks.front().time) { - } + // If we are within the wait offset of the time, spinlock until we get there for greater + // accuracy + if (NUClear::clock::now() + wait_offset > tasks.front().time) { + + // Spinlock! + while (NUClear::clock::now() < tasks.front().time) { + } - const NUClear::clock::time_point now = NUClear::clock::now(); + const NUClear::clock::time_point now = NUClear::clock::now(); - // Move back from the end poping the heap - for (auto end = tasks.end(); end != tasks.begin() && tasks.front().time < now;) { - // Run our task and if it returns false remove it - const bool renew = tasks.front()(); + // Move back from the end poping the heap + for (auto end = tasks.end(); end != tasks.begin() && tasks.front().time < now;) { + // Run our task and if it returns false remove it + const bool renew = tasks.front()(); - // Move this to the back of the list - std::pop_heap(tasks.begin(), end, std::greater<>()); + // Move this to the back of the list + std::pop_heap(tasks.begin(), end, std::greater<>()); - if (!renew) { - end = tasks.erase(--end); - } - else { - --end; - } + if (!renew) { + end = tasks.erase(--end); + } + else { + --end; } - } - // Otherwise we wait for the next event using a wait_for (with a small offset for greater - // accuracy) Either that or until we get interrupted with a new event - else { - wait.wait_until(lock, tasks.front().time - wait_offset); } } - // Otherwise we wait for something to happen + // Otherwise we wait for the next event using a wait_for (with a small offset for greater + // accuracy) Either that or until we get interrupted with a new event else { - wait.wait(lock); + wait.wait_until(lock, tasks.front().time - wait_offset); } } + // Otherwise we wait for something to happen + else { + wait.wait(lock); + } }); } From 3884744adc917dc99fd3557e81e2aefd5c619d2f Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:50:22 +1000 Subject: [PATCH 68/87] Simplify statics for pools and groups --- src/dsl/word/Always.hpp | 2 +- src/dsl/word/Group.hpp | 21 +++++---------------- src/dsl/word/Pool.hpp | 21 +++++---------------- src/dsl/word/Sync.hpp | 20 ++++---------------- src/util/GroupDescriptor.hpp | 2 +- src/util/ThreadPoolDescriptor.hpp | 2 +- 6 files changed, 17 insertions(+), 51 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 8742c1475..fe3784f2d 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -74,7 +74,7 @@ namespace dsl { const std::lock_guard lock(mutex); if (pool_id.count(reaction.id) == 0) { - pool_id[reaction.id] = util::ThreadPoolIDSource::get_new_pool_id(); + pool_id[reaction.id] = util::ThreadPoolIDSource::get_unique_pool_id(); } return util::ThreadPoolDescriptor{pool_id[reaction.id], 1}; } diff --git a/src/dsl/word/Group.hpp b/src/dsl/word/Group.hpp index d6809335e..0ccbcc8a7 100644 --- a/src/dsl/word/Group.hpp +++ b/src/dsl/word/Group.hpp @@ -32,29 +32,18 @@ namespace dsl { template struct Group { - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::map group_id; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::mutex mutex; + static const util::GroupDescriptor group_descriptor; template static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - - const std::lock_guard lock(mutex); - if (group_id.count(typeid(GroupType)) == 0) { - group_id[typeid(GroupType)] = util::GroupDescriptor::get_new_group_id(); - } - return util::GroupDescriptor{group_id[typeid(GroupType)], 1}; + return group_descriptor; } }; - // Initialise the static map and mutex - template - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::map Group::group_id; + // Initialise the group descriptor template - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::mutex Group::mutex; + const util::GroupDescriptor Group::group_descriptor = {util::GroupDescriptor::get_unique_group_id(), + GroupType::concurrency}; } // namespace word } // namespace dsl diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index 52e600fa1..c05c0b6ca 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -31,29 +31,18 @@ namespace dsl { template struct Pool { - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::map pool_id; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::mutex mutex; + static const util::ThreadPoolDescriptor pool_descriptor; template static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - - const std::lock_guard lock(mutex); - if (pool_id.count(typeid(PoolType)) == 0) { - pool_id[typeid(PoolType)] = util::ThreadPoolIDSource::get_new_pool_id(); - } - return util::ThreadPoolDescriptor{pool_id[typeid(PoolType)], PoolType::concurrency}; + return pool_descriptor; } }; - // Initialise the static map and mutex - template - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::map Pool::pool_id; + // Initialise the thread pool descriptor template - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::mutex Pool::mutex; + const util::ThreadPoolDescriptor Pool::pool_descriptor = {util::ThreadPoolIDSource::get_unique_pool_id(), + PoolType::concurrency}; } // namespace word } // namespace dsl diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index 3f13deac9..b83e9e647 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -70,29 +70,17 @@ namespace dsl { template struct Sync { - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::map group_id; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static std::mutex mutex; + static const util::GroupDescriptor group_descriptor; template static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - - const std::lock_guard lock(mutex); - if (group_id.count(typeid(SyncGroup)) == 0) { - group_id[typeid(SyncGroup)] = util::GroupDescriptor::get_new_group_id(); - } - return util::GroupDescriptor{group_id[typeid(SyncGroup)], 1}; + return group_descriptor; } }; - // Initialise the static map and mutex - template - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::map Sync::group_id; + // Initialise the group descriptor template - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - std::mutex Sync::mutex; + const util::GroupDescriptor Sync::group_descriptor = {util::GroupDescriptor::get_unique_group_id(), 1}; } // namespace word } // namespace dsl diff --git a/src/util/GroupDescriptor.hpp b/src/util/GroupDescriptor.hpp index 159fcd95a..95ea14c99 100644 --- a/src/util/GroupDescriptor.hpp +++ b/src/util/GroupDescriptor.hpp @@ -33,7 +33,7 @@ namespace util { /// @brief The number of threads this thread pool will use. size_t thread_count{std::numeric_limits::max()}; - static uint64_t get_new_group_id() { + static uint64_t get_unique_group_id() { // Make group 0 the default group static std::atomic source{1}; return source++; diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp index 4fdc2983c..f69938f71 100644 --- a/src/util/ThreadPoolDescriptor.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -29,7 +29,7 @@ namespace util { static const uint64_t MAIN_THREAD_POOL_ID; static const uint64_t DEFAULT_THREAD_POOL_ID; - static uint64_t get_new_pool_id() { + static uint64_t get_unique_pool_id() { static std::atomic source{2}; return source++; } From d1628b986aed6e3b654edd108b9bb4ed56cf295a Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:50:50 +1000 Subject: [PATCH 69/87] RAII-ify `current_task` --- src/threading/ReactionTask.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 743025efb..58e765af5 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -104,15 +104,11 @@ namespace threading { inline void run() { // Update our current task - // TODO(Trent) RAII THIS - Task* old_task = current_task; - current_task = this; + std::unique_ptr lock(current_task, [](Task* t) { current_task = t; }); + current_task = this; // Run our callback callback(*this); - - // Reset our task back - current_task = old_task; } /// @brief the parent Reaction object which spawned this From 1d37e2084d49308884500a1a5a8e4aeaad64222f Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:51:05 +1000 Subject: [PATCH 70/87] Make sure `nullptr`s are sorted first --- src/threading/ReactionTask.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 58e765af5..73ee5d426 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -155,8 +155,8 @@ namespace threading { // nullptr is greater than anything else so it's removed from a queue first // higher priority tasks are greater than lower priority tasks // tasks created first (smaller ids) should run before tasks created later - return a == nullptr ? false - : b == nullptr ? true + return a == nullptr ? true + : b == nullptr ? false : a->priority == b->priority ? a->id < b->id : a->priority > b->priority; } From 1003343018e12379a9dc40d64cde576625dc8453 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Tue, 20 Jun 2023 16:54:29 +1000 Subject: [PATCH 71/87] Unify thread pool structs --- src/dsl/word/Always.hpp | 2 +- src/dsl/word/MainThread.hpp | 2 +- src/dsl/word/Pool.hpp | 5 +++-- src/threading/TaskScheduler.cpp | 17 +++++++++-------- src/util/GeneratedCallback.hpp | 2 +- src/util/ThreadPoolDescriptor.cpp | 4 ++-- src/util/ThreadPoolDescriptor.hpp | 16 +++++++--------- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index fe3784f2d..b261da1e2 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -74,7 +74,7 @@ namespace dsl { const std::lock_guard lock(mutex); if (pool_id.count(reaction.id) == 0) { - pool_id[reaction.id] = util::ThreadPoolIDSource::get_unique_pool_id(); + pool_id[reaction.id] = util::ThreadPoolDescriptor::get_unique_pool_id(); } return util::ThreadPoolDescriptor{pool_id[reaction.id], 1}; } diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 8857c5178..894d36128 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -40,7 +40,7 @@ namespace dsl { template static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - return util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; + return util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, 1}; } }; diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index c05c0b6ca..b7ba00a96 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -41,8 +41,9 @@ namespace dsl { // Initialise the thread pool descriptor template - const util::ThreadPoolDescriptor Pool::pool_descriptor = {util::ThreadPoolIDSource::get_unique_pool_id(), - PoolType::concurrency}; + const util::ThreadPoolDescriptor Pool::pool_descriptor = { + util::ThreadPoolDescriptor::get_unique_pool_id(), + PoolType::concurrency}; } // namespace word } // namespace dsl diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 126a2f211..fecc70d4d 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -71,16 +71,16 @@ namespace threading { TaskScheduler::TaskScheduler() { // Setup everything (thread pool, task queue, mutex, condition variable) for the main thread here - pools[util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID] = - util::ThreadPoolDescriptor{util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, 1}; - pool_map[std::this_thread::get_id()] = util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID; + pools[util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID] = + util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, 1}; + pool_map[std::this_thread::get_id()] = util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID; - queue.emplace(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID, std::vector>{}); + queue.emplace(util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, std::vector>{}); } void TaskScheduler::start_threads(const util::ThreadPoolDescriptor& pool) { // The main thread never needs to be started - if (pool.pool_id != util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID) { + if (pool.pool_id != util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID) { const std::lock_guard pool_lock(pool_mutex); for (size_t i = 0; i < pool.thread_count; ++i) { threads.push_back(std::make_unique(&TaskScheduler::pool_func, this, pool)); @@ -118,7 +118,7 @@ namespace threading { void TaskScheduler::start(const size_t& thread_count) { // Make the default pool - create_pool(util::ThreadPoolDescriptor{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, thread_count}); + create_pool(util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID, thread_count}); // The scheduler is now started started.store(true); @@ -129,7 +129,8 @@ namespace threading { } // Run main thread tasks - pool_func(pools.at(util::ThreadPoolIDSource::MAIN_THREAD_POOL_ID)); + pool_func(pools.at(util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID)); + // Now wait for all the threads to finish executing for (auto& thread : threads) { @@ -184,7 +185,7 @@ namespace threading { // If we aren't already started then just queue the task up to run when we are started if (started.load() && task->immediate) { // Map the current thread to the thread pool it belongs to - uint64_t thread_pool = util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID; + uint64_t thread_pool = util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID; /* mutex scope */ { const std::lock_guard pool_lock(pool_mutex); if (pool_map.count(std::this_thread::get_id()) > 0) { diff --git a/src/util/GeneratedCallback.hpp b/src/util/GeneratedCallback.hpp index cd0e55892..35b5c5ffa 100644 --- a/src/util/GeneratedCallback.hpp +++ b/src/util/GeneratedCallback.hpp @@ -36,7 +36,7 @@ namespace util { : priority(priority), group(group), pool(pool), callback(std::move(callback)) {} int priority{0}; GroupDescriptor group{0, std::numeric_limits::max()}; - ThreadPoolDescriptor pool{util::ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID, 0}; + ThreadPoolDescriptor pool{util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID, 0}; threading::ReactionTask::TaskFunction callback{}; operator bool() const { diff --git a/src/util/ThreadPoolDescriptor.cpp b/src/util/ThreadPoolDescriptor.cpp index 71e76c770..606804f1b 100644 --- a/src/util/ThreadPoolDescriptor.cpp +++ b/src/util/ThreadPoolDescriptor.cpp @@ -20,8 +20,8 @@ namespace NUClear { namespace util { - const uint64_t ThreadPoolIDSource::MAIN_THREAD_POOL_ID = 0; - const uint64_t ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID = 1; + const uint64_t ThreadPoolDescriptor::MAIN_THREAD_POOL_ID = 0; + const uint64_t ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID = 1; } // namespace util } // namespace NUClear diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp index f69938f71..8a9819455 100644 --- a/src/util/ThreadPoolDescriptor.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -25,7 +25,13 @@ namespace NUClear { namespace util { - struct ThreadPoolIDSource { + struct ThreadPoolDescriptor { + /// @brief Set a unique identifier for this pool + uint64_t pool_id{ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID}; + + /// @brief The number of threads this thread pool will use. + size_t thread_count{0}; + static const uint64_t MAIN_THREAD_POOL_ID; static const uint64_t DEFAULT_THREAD_POOL_ID; @@ -35,14 +41,6 @@ namespace util { } }; - struct ThreadPoolDescriptor { - /// @brief Set a unique identifier for this pool - uint64_t pool_id{ThreadPoolIDSource::DEFAULT_THREAD_POOL_ID}; - - /// @brief The number of threads this thread pool will use. - size_t thread_count{0}; - }; - } // namespace util } // namespace NUClear From 6fad9fb65913bc546b13369c0e85c47ac133eb06 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 09:42:40 +1000 Subject: [PATCH 72/87] Make CallbackGenerator constructor explicit --- src/util/CallbackGenerator.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 3ab3e9181..8047fe5c5 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -48,12 +48,8 @@ namespace util { struct CallbackGenerator { // Don't use this constructor if F is of type CallbackGenerator - template ::type>::type, - CallbackGenerator>::value, - bool>::type = true> - CallbackGenerator(F&& callback) + template + explicit CallbackGenerator(F&& callback) : callback(std::forward(callback)) , transients(std::make_shared::type>()) {} From 73653dc66bcfe18fd4982bca397d3f209ab12881 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 09:50:10 +1000 Subject: [PATCH 73/87] Revert "Make CallbackGenerator constructor explicit" This reverts commit 6fad9fb65913bc546b13369c0e85c47ac133eb06. --- src/util/CallbackGenerator.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 8047fe5c5..3ab3e9181 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -48,8 +48,12 @@ namespace util { struct CallbackGenerator { // Don't use this constructor if F is of type CallbackGenerator - template - explicit CallbackGenerator(F&& callback) + template ::type>::type, + CallbackGenerator>::value, + bool>::type = true> + CallbackGenerator(F&& callback) : callback(std::forward(callback)) , transients(std::make_shared::type>()) {} From 7ae4e7558e3aab60abb9f10fce024794458a91e1 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 10:18:01 +1000 Subject: [PATCH 74/87] Fix post-shutdown logging --- src/threading/TaskScheduler.cpp | 11 ++++------- tests/log/Log.cpp | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index fecc70d4d..12e4a01dc 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -166,9 +166,6 @@ namespace threading { // Extract the thread pool descriptor from the current task const util::ThreadPoolDescriptor current_pool = task->thread_pool_descriptor; - // We do not accept new tasks once we are shutdown - if (running.load()) { - // Make sure the pool is created create_pool(current_pool); @@ -182,8 +179,8 @@ namespace threading { } // Check to see if this task was the result of `emit` - // If we aren't already started then just queue the task up to run when we are started - if (started.load() && task->immediate) { + // Direct tasks can run after shutdown and before starting, provided they can be run immediately + if (task->immediate) { // Map the current thread to the thread pool it belongs to uint64_t thread_pool = util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID; /* mutex scope */ { @@ -201,7 +198,8 @@ namespace threading { } } - /* Mutex Scope */ { + // We do not accept new tasks once we are shutdown + if (running.load()) { const std::lock_guard queue_lock(queue_mutex); // Find where to insert the new task to maintain task order @@ -212,7 +210,6 @@ namespace threading { // Insert before the found position queue.at(current_pool.pool_id).insert(it, std::forward>(task)); - } } // Notify all threads that there is a new task to be processed diff --git a/tests/log/Log.cpp b/tests/log/Log.cpp index faece41b6..a8fa92938 100644 --- a/tests/log/Log.cpp +++ b/tests/log/Log.cpp @@ -165,6 +165,7 @@ TEST_CASE("Testing the Log<>() function", "[api][log]") { expected_count += levels.size() * (levels.size() + 1) / 2; // Direct reaction logs expected_count += levels.size() * (levels.size() + 1) / 2; // Indirect reaction logs expected_count += levels.size() * levels.size(); // Non reaction logs + expected_count += 2 + levels.size(); // Post shutdown logs REQUIRE(messages.size() == expected_count); // Test that each of the messages are correct for each log level @@ -197,4 +198,24 @@ TEST_CASE("Testing the Log<>() function", "[api][log]") { REQUIRE_FALSE(messages[i++].from_reaction); } } + + // Test post-shutdown logs + { + const std::string expected = "Post Powerplant Shutdown " + std::to_string(NUClear::FATAL); + REQUIRE(messages[i].message == expected); + REQUIRE(messages[i].level == NUClear::FATAL); + REQUIRE(messages[i++].from_reaction); + REQUIRE(messages[i].message == expected); + REQUIRE(messages[i].level == NUClear::FATAL); + REQUIRE(messages[i++].from_reaction); + } + + // Test logs from free floating functions + for (const auto& log_level : levels) { + // No filter here, free floating prints everything + const std::string expected = "Non Reaction " + std::to_string(log_level); + REQUIRE(messages[i].message == expected); + REQUIRE(messages[i].level == log_level); + REQUIRE_FALSE(messages[i++].from_reaction); + } } From 071f0749581ba5cc6b2c27be80d50531fce8ea2c Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 10:20:01 +1000 Subject: [PATCH 75/87] Emit a lot more messages for `SingleSync` test --- tests/dsl/SingleSync.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/dsl/SingleSync.cpp b/tests/dsl/SingleSync.cpp index 614e349f9..696dd37c2 100644 --- a/tests/dsl/SingleSync.cpp +++ b/tests/dsl/SingleSync.cpp @@ -44,10 +44,9 @@ class TestReactor : public NUClear::Reactor { on().then("Startup", [this] { values.clear(); - emit(std::make_unique(0)); - emit(std::make_unique(1)); - emit(std::make_unique(2)); - emit(std::make_unique(3)); + for (int i = 0; i < 1000; ++i) { + emit(std::make_unique(i)); + } emit(std::make_unique()); }); } @@ -62,8 +61,8 @@ TEST_CASE("Testing that the Sync priority queue word works correctly", "[api][sy plant.install(); plant.start(); - REQUIRE(values.size() == 4); - for (int i = 0; i < 4; ++i) { + REQUIRE(values.size() == 1000); + for (int i = 0; i < 1000; ++i) { CHECK(values[i] == "Received value " + std::to_string(i)); } } From 312d28d442a92297939aad37c5705f7b1dff7225 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 14:48:21 +1000 Subject: [PATCH 76/87] Fix up comments --- src/dsl/word/Always.hpp | 20 +++++++++-- src/dsl/word/Group.hpp | 43 +++++++++++++++++++++--- src/dsl/word/Pool.hpp | 40 +++++++++++++++++++++- src/dsl/word/Sync.hpp | 13 +++----- src/dsl/word/emit/Delay.hpp | 2 +- src/dsl/word/emit/Direct.hpp | 2 +- src/threading/ReactionTask.hpp | 16 +++++++-- src/threading/TaskScheduler.cpp | 10 ++++-- src/threading/TaskScheduler.hpp | 55 ++++++++++++++++++++++++++++--- src/util/GeneratedCallback.hpp | 10 ++++++ src/util/GroupDescriptor.hpp | 10 ++++-- src/util/ThreadPoolDescriptor.hpp | 12 +++++-- src/util/main_thread_id.cpp | 6 ---- src/util/main_thread_id.hpp | 6 ++++ 14 files changed, 206 insertions(+), 39 deletions(-) diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index b261da1e2..c8aab28a7 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -43,8 +43,8 @@ namespace dsl { * Any reactions requested using this keyword will initialise upon system start-up and execute continually * until system shut-down. * - * Note that a task spawned from this request will execute in its own unique thread rather than the threadpool. - * However, if the task is rescheduled (such as with Sync), it will then be moved into the threadpool. + * Note that a task spawned from this request will execute in its own unique thread rather than the default + * thread pool. * * @par Infinite Loops * This word should be used in place of any reactions which would contain an infinite loop. That is, @@ -59,10 +59,11 @@ namespace dsl { * * @attention * Where possible, developers should avoid using this keyword. It has been provided, but should only be - * used when there is no other way to scheduled the reaction. If a developer is tempted to use this keyword, + * used when there is no other way to schedule the reaction. If a developer is tempted to use this keyword, * it is advised to review other options, such as on before resorting to this feature. * * @par Implements + * Pool * Bind */ struct Always { @@ -81,10 +82,23 @@ namespace dsl { template static inline void bind(const std::shared_ptr& always_reaction) { + // Static map mapping reaction id (from the always reaction) to a pair of reaction pointers -- one for + // the always reaction and one for the idle reaction that we generate in this function + // The main purpose of this map is to ensure that the always reaction pointer doesn't get destroyed static std::map, std::shared_ptr>> reaction_store = {}; + // Generate a new reaction for an idle task + // The purpose of this reaction is to ensure that the always reaction is resubmitted in the event that + // the precondition fails (e.g. on> will fail the precondition if there are no X + // messages previously emitted) + // + // In the event that the precondition on the always reaction fails this idle task will run and resubmit + // both the always reaction and the idle reaction + // + // The idle reaction must have a lower priority than the always reaction and must also run in the same + // thread pool and group as the always reaction auto idle_reaction = std::make_shared( always_reaction->reactor, std::vector{always_reaction->identifier[0] + " - IDLE Task", diff --git a/src/dsl/word/Group.hpp b/src/dsl/word/Group.hpp index 0ccbcc8a7..8c2780e43 100644 --- a/src/dsl/word/Group.hpp +++ b/src/dsl/word/Group.hpp @@ -29,9 +29,43 @@ namespace NUClear { namespace dsl { namespace word { - template + /** + * @brief + * This is used to specify that only one reaction in this GroupType can run concurrently. + * + * @details + * @code on, Group>() @endcode + * When a group of tasks has been synchronised, only N task(s) from the group will execute at a given time. + * + * Should another task from this group be scheduled/requested (during execution of the current N task(s)), it + * will be sidelined into the task queue. + * + * Tasks in the queue are ordered based on their priority level, then their task id. + * + * For best use, this word should be fused with at least one other binding DSL word. + * + * @attention + * When using NUClear, developers should not make use of devices like a mutex. In the case of a mutex, threads + * will run and then block (leading to wasted resources on a number of inactive threads). By using Sync, + * NUClear will have task and thread control so that system resources can be efficiently managed. + * + * @par Implements + * Group + * + * @tparam GroupType + * the type/group to synchronize on. This needs to be a declared type within the system. It is common to + * simply use the reactors name (i.e; if the reactor is only syncing with one group). Should more than one + * group be required, the developer can declare structs within the system, to act as a group reference. + * Note that the developer is not limited to the use of a struct; any declared type will work. + * @tparam GroupConcurrency + * the number of tasks that should be allowed to run concurrently in this group. It is an error to specify a + * group concurrency less than 1. + */ + template struct Group { + static_assert(GroupConcurrency > 0, "Can not have a group with concurrency less than 1"); + static const util::GroupDescriptor group_descriptor; template @@ -41,9 +75,10 @@ namespace dsl { }; // Initialise the group descriptor - template - const util::GroupDescriptor Group::group_descriptor = {util::GroupDescriptor::get_unique_group_id(), - GroupType::concurrency}; + template + const util::GroupDescriptor Group::group_descriptor = { + util::GroupDescriptor::get_unique_group_id(), + GroupConcurrency}; } // namespace word } // namespace dsl diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp index b7ba00a96..3a56586dc 100644 --- a/src/dsl/word/Pool.hpp +++ b/src/dsl/word/Pool.hpp @@ -29,8 +29,46 @@ namespace NUClear { namespace dsl { namespace word { + /** + * @brief + * This is used to specify that this reaction should run in the designated thread pool + * + * @details + * @code on, Pool>() @endcode + * This DSL will cause the creation of a new thread pool with a specific number of threads allocated to it. + * + * All tasks for this reaction will be queued to run on threads from this thread pool. + * + * Tasks in the queue are ordered based on their priority level, then their task id. + * + * When this DSL is not specified the default thread pool will be used. For tasks that need to run on the main + * thread use MainThread. + * + * For best use, this word should be fused with at least one other binding DSL word. + * + * @attention + * This DSL should be used sparingly as having an increased number of threads running concurrently on the + * system can lead to a degradation in performance. + * + * @par Implements + * pool + * + * @tparam PoolType + * A struct that contains the details of the thread pool to create. This struct should contain a static int + * member that sets the number of threads that should be allocated to this pool. + * @code + * struct ThreadPool { + * static const int thread_count = 2; + * }; + * @endcode + * While it is valid to have a non-const static thread count in the struct, NUClear will not look at any + * changes to this value. Only the first value that NUClear sees will be used to initiate the thread pool. + */ template struct Pool { + + static_assert(PoolType::thread_count > 0, "Can not have a thread pool with less than 1 thread"); + static const util::ThreadPoolDescriptor pool_descriptor; template @@ -43,7 +81,7 @@ namespace dsl { template const util::ThreadPoolDescriptor Pool::pool_descriptor = { util::ThreadPoolDescriptor::get_unique_pool_id(), - PoolType::concurrency}; + PoolType::thread_count}; } // namespace word } // namespace dsl diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index b83e9e647..95fcae984 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -32,19 +32,16 @@ namespace dsl { /** * @brief - * This option sets the synchronisation for a group of tasks. + * This is used to specify that only one reaction in this SyncGroup can run concurrently. * * @details - * @code on, Sync>() @endcode + * @code on, Sync>() @endcode * When a group of tasks has been synchronised, only one task from the group will execute at a given time. * * Should another task from this group be scheduled/requested (during execution of the current task), it will - * be sidelined into a priority queue. + * be sidelined into the task queue. * - * Upon completion of the currently executing task, the queue will be polled to allow execution of the next - * task in this group. - * - * Tasks in the synchronization queue are ordered based on their priority level, then their emission timestamp. + * Tasks in the queue are ordered based on their priority level, then their task id. * * For best use, this word should be fused with at least one other binding DSL word. * @@ -59,7 +56,7 @@ namespace dsl { * NUClear will have task and thread control so that system resources can be efficiently managed. * * @par Implements - * Pre-condition, Post-condition + * Group * * @tparam SyncGroup * the type/group to synchronize on. This needs to be a declared type within the system. It is common to diff --git a/src/dsl/word/emit/Delay.hpp b/src/dsl/word/emit/Delay.hpp index f58ce0502..f2f4b10dc 100644 --- a/src/dsl/word/emit/Delay.hpp +++ b/src/dsl/word/emit/Delay.hpp @@ -34,7 +34,7 @@ namespace dsl { * @details * @code emit(data, delay(ticks), dataType); @endcode * Emissions under this scope will wait for the provided time delay, and then emit the object utilising a - * local emit (that is, normal threadpool distribution). + * local emit (that is, normal thread pool distribution). * * @param data * the data to emit diff --git a/src/dsl/word/emit/Direct.hpp b/src/dsl/word/emit/Direct.hpp index ae5229241..817462a4e 100644 --- a/src/dsl/word/emit/Direct.hpp +++ b/src/dsl/word/emit/Direct.hpp @@ -32,7 +32,7 @@ namespace dsl { /** * @brief * When emitting data under this scope, the tasks created as a result of this emission will bypass the - * threadpool, and be executed immediately. + * thread pool, and be executed immediately. * * @details * @code emit(data, dataType); @endcode diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 73ee5d426..6c7758b0a 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -67,9 +67,11 @@ namespace threading { /** * @brief Creates a new ReactionTask object bound with the parent Reaction object (that created it) and task. * - * @param parent the Reaction object that spawned this ReactionTask. - * @param priority the priority to use when executing this task. - * @param callback the data bound callback to be executed in the threadpool. + * @param parent the Reaction object that spawned this ReactionTask. + * @param priority the priority to use when executing this task. + * @param group_descriptor the descriptor for the group that this task should run in + * @param thread_pool_descriptor the descriptor for the thread pool that this task should be queued in + * @param callback the data bound callback to be executed in the thread pool. */ Task(ReactionType& parent, const int& priority, @@ -123,8 +125,16 @@ namespace threading { /// reaction statistics becomes false for all created tasks. This is to stop infinite loops of death. bool emit_stats; + /// @brief details about the group that this task will run in util::GroupDescriptor group_descriptor; + + /// @brief details about the thread pool that this task will run from, this will also influence what task queue + /// the tasks will be queued on util::ThreadPoolDescriptor thread_pool_descriptor; + + /// @brief if this task should run immediately in the current thread. If immediate execution of this task is not + /// possible (e.g. due to group concurrency restrictions) this task will be queued as normal. This flag will be + /// set by `emit` bool immediate{false}; /// @brief the data bound callback to be executed diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 12e4a01dc..d08145c30 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -1,6 +1,7 @@ /* * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston + * 2014-2022 Trent Houliston + * 2023 Trent Houliston , Alex Biddulph * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the @@ -74,7 +75,6 @@ namespace threading { pools[util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID] = util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, 1}; pool_map[std::this_thread::get_id()] = util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID; - queue.emplace(util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, std::vector>{}); } @@ -221,12 +221,15 @@ namespace threading { std::unique_lock queue_lock(queue_mutex); + // Keep looking for tasks while the scheduler is still running, or while there are still tasks to process while (running.load() || !queue[pool_id].empty()) { + // Iterate over all the tasks in the current thread pool queue, looking for one that we can run for (auto it = queue.at(pool_id).begin(); it != queue.at(pool_id).end(); ++it) { - // Check if we can run it + // Check if we can run the task if (is_runnable(*it, pool_id)) { + // Move the task out of the queue std::unique_ptr task = std::move(*it); @@ -242,6 +245,7 @@ namespace threading { queue_condition.wait(queue_lock); } + // No more tasks and scheduler has shutdown return nullptr; } } // namespace threading diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index cdecbf8f9..010a70c32 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -115,31 +115,76 @@ namespace threading { */ std::unique_ptr get_task(const uint64_t& pool_id); + /** + * @brief Creates a new thread pool and ensures threads and a task queue are allocated for it + * + * @param pool the descriptor for the thread pool to create + */ void create_pool(const util::ThreadPoolDescriptor& pool); + + /** + * @brief The function that each thread runs + * + * @details This function will repeatedly query the task queue for new a task to run and then execute that task + * + * @param pool the thread pool to run from and the task queue to get tasks from + */ void pool_func(const util::ThreadPoolDescriptor& pool); + + /** + * @brief Start all threads for the given thread pool + */ void start_threads(const util::ThreadPoolDescriptor& pool); + + /** + * @brief Execute the given task + * + * @details After execution of the task has completed the number of active tasks in the tasks' group is + * decremented + */ void run_task(std::unique_ptr&& task); + + /** + * @brief Determines if the given task is able to be executed + * + * @details If the current thread is able to be executed the number of active tasks in the tasks' groups is + * incremented + * + * @param task the task to inspect + * @param pool_id the pool to run the task on + * @return true if the task is currently runnable + * @return false if the task is not currently runnable + */ bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id); - /// @brief if the scheduler is running + /// @brief if the scheduler is running, and accepting new tasks. If this is false and a new, non-immediate, task + /// is submitted it will be ignored std::atomic running{true}; + /// @brief if the scheduler has been started. This is set to true after a call to start is made. Once this is + /// set to true all threads will begin executing tasks from the tasks queue std::atomic started{false}; - /// @brief our queue which sorts tasks by priority + /// @brief A task queue for each thread pool. In each queue, each task is ordered by priority and then by task + /// id std::map>> queue; - /// @brief the mutex which our threads synchronize their access to this object + /// @brief A map of group ids to the number of active tasks currently running in that group + std::map groups{}; + + /// @brief the mutex which our threads synchronize their access to the task queues and the group concurrency + /// counts std::mutex queue_mutex; /// @brief the condition object that threads wait on if they can't get a task std::condition_variable queue_condition; /// @brief A vector of the running threads in the system std::vector> threads; + /// @brief A map of pool descriptor ids to pool descriptors std::map pools{}; + /// @brief A map of thread ids to the pools they belong to std::map pool_map{}; + /// @brief the mutex which our threads synchronize their access to the thread pool maps and the threads list std::mutex pool_mutex; - - std::map groups{}; }; } // namespace threading diff --git a/src/util/GeneratedCallback.hpp b/src/util/GeneratedCallback.hpp index 35b5c5ffa..31ed39af0 100644 --- a/src/util/GeneratedCallback.hpp +++ b/src/util/GeneratedCallback.hpp @@ -27,6 +27,9 @@ namespace NUClear { namespace util { + /** + * @brief Generated callback for a task + */ struct GeneratedCallback { GeneratedCallback() = default; GeneratedCallback(const int& priority, @@ -34,11 +37,18 @@ namespace util { const ThreadPoolDescriptor& pool, threading::ReactionTask::TaskFunction callback) : priority(priority), group(group), pool(pool), callback(std::move(callback)) {} + /// @brief the priority this task should run with int priority{0}; + /// @brief the descriptor for the group the task should run in GroupDescriptor group{0, std::numeric_limits::max()}; + /// @brief the descriptor the thread pool and task queue that the should run in ThreadPoolDescriptor pool{util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID, 0}; + /// @brief the function that should be executed in order to run the task threading::ReactionTask::TaskFunction callback{}; + /** + * @return true if this represents a valid callback object + */ operator bool() const { return bool(callback); } diff --git a/src/util/GroupDescriptor.hpp b/src/util/GroupDescriptor.hpp index 95ea14c99..c625a8e7a 100644 --- a/src/util/GroupDescriptor.hpp +++ b/src/util/GroupDescriptor.hpp @@ -26,13 +26,19 @@ namespace NUClear { namespace util { + /** + * @brief A description of a group + */ struct GroupDescriptor { - /// @brief Set a unique identifier for this pool + /// @brief a unique identifier for this pool uint64_t group_id{0}; - /// @brief The number of threads this thread pool will use. + /// @brief the maximum number of threads that can run concurrently in this group size_t thread_count{std::numeric_limits::max()}; + /** + * @brief Return the next unique ID for a new group + */ static uint64_t get_unique_group_id() { // Make group 0 the default group static std::atomic source{1}; diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp index 8a9819455..4c5f508b9 100644 --- a/src/util/ThreadPoolDescriptor.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -25,16 +25,24 @@ namespace NUClear { namespace util { + /** + * @brief A description of a thread pool + */ struct ThreadPoolDescriptor { - /// @brief Set a unique identifier for this pool + /// @brief a unique identifier for this pool uint64_t pool_id{ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID}; - /// @brief The number of threads this thread pool will use. + /// @brief the number of threads this thread pool will use size_t thread_count{0}; + /// @brief the ID of the main thread pool (not to be confused with the ID of the main thread) static const uint64_t MAIN_THREAD_POOL_ID; + /// @brief the ID of the default thread pool static const uint64_t DEFAULT_THREAD_POOL_ID; + /** + * @brief Return the next unique ID for a new thread pool + */ static uint64_t get_unique_pool_id() { static std::atomic source{2}; return source++; diff --git a/src/util/main_thread_id.cpp b/src/util/main_thread_id.cpp index 32851f506..ffbecd9f7 100644 --- a/src/util/main_thread_id.cpp +++ b/src/util/main_thread_id.cpp @@ -21,12 +21,6 @@ namespace NUClear { namespace util { - /** - * @brief The thread id of the main execution thread for this process - - * @detail In order to get the main threads id, we set it as a global static variable. - * This should result in the static setup code executing on startup (in the main thread). - */ const std::thread::id main_thread_id = std::this_thread::get_id(); } // namespace util diff --git a/src/util/main_thread_id.hpp b/src/util/main_thread_id.hpp index 1adee4745..ba576dc72 100644 --- a/src/util/main_thread_id.hpp +++ b/src/util/main_thread_id.hpp @@ -24,6 +24,12 @@ namespace NUClear { namespace util { + /** + * @brief The thread id of the main execution thread for this process + + * @details In order to get the main threads id, we set it as a global static variable. + * This should result in the static setup code executing on startup (in the main thread). + */ extern const std::thread::id main_thread_id; } // namespace util From fcffd7b320a83b75e4d7e5c15651c184b58c7eaa Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 14:48:32 +1000 Subject: [PATCH 77/87] Revert RAII-ify --- src/threading/ReactionTask.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 6c7758b0a..a53c3bffc 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -106,11 +106,14 @@ namespace threading { inline void run() { // Update our current task - std::unique_ptr lock(current_task, [](Task* t) { current_task = t; }); - current_task = this; + Task* old_task = current_task; + current_task = this; // Run our callback callback(*this); + + // Reset our task back + current_task = old_task; } /// @brief the parent Reaction object which spawned this From eebe07deb182db5317a6bf54b387c11094574e52 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 14:48:53 +1000 Subject: [PATCH 78/87] Do thread poking once before joining --- src/threading/TaskScheduler.cpp | 75 ++++++++++++++++----------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index d08145c30..f38bc9305 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -131,19 +131,16 @@ namespace threading { // Run main thread tasks pool_func(pools.at(util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID)); + // Poke all of the threads to make sure they are awake + /* mutex scope */ { + const std::lock_guard queue_lock(queue_mutex); + queue_condition.notify_all(); + } // Now wait for all the threads to finish executing for (auto& thread : threads) { try { if (thread->joinable()) { - // Poke the thread pool that this thread belongs to to make sure it has woken up - /* mutex scope */ { - const std::lock_guard pool_lock(pool_mutex); - if (pool_map.count(thread->get_id()) > 0) { - const std::lock_guard queue_lock(queue_mutex); - queue_condition.notify_all(); - } - } thread->join(); } } @@ -166,50 +163,50 @@ namespace threading { // Extract the thread pool descriptor from the current task const util::ThreadPoolDescriptor current_pool = task->thread_pool_descriptor; - // Make sure the pool is created - create_pool(current_pool); + // Make sure the pool is created + create_pool(current_pool); - // Make sure we know about this group - /* mutex scope */ { - const std::lock_guard group_lock(queue_mutex); - const uint64_t group_id = task->group_descriptor.group_id; - if (groups.count(group_id) == 0) { - groups.emplace(group_id, 0); - } + // Make sure we know about this group + /* mutex scope */ { + const std::lock_guard group_lock(queue_mutex); + const uint64_t group_id = task->group_descriptor.group_id; + if (groups.count(group_id) == 0) { + groups.emplace(group_id, 0); } + } - // Check to see if this task was the result of `emit` + // Check to see if this task was the result of `emit` // Direct tasks can run after shutdown and before starting, provided they can be run immediately if (task->immediate) { - // Map the current thread to the thread pool it belongs to - uint64_t thread_pool = util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID; - /* mutex scope */ { - const std::lock_guard pool_lock(pool_mutex); - if (pool_map.count(std::this_thread::get_id()) > 0) { - thread_pool = pool_map.at(std::this_thread::get_id()); - } + // Map the current thread to the thread pool it belongs to + uint64_t thread_pool = util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID; + /* mutex scope */ { + const std::lock_guard pool_lock(pool_mutex); + if (pool_map.count(std::this_thread::get_id()) > 0) { + thread_pool = pool_map.at(std::this_thread::get_id()); } + } - // Check to see if this task is runnable in the current thread - // If it isn't we can just queue it up with all of the other non-immediate task - if (is_runnable(task, thread_pool)) { - run_task(std::move(task)); - return; - } + // Check to see if this task is runnable in the current thread + // If it isn't we can just queue it up with all of the other non-immediate task + if (is_runnable(task, thread_pool)) { + run_task(std::move(task)); + return; } + } // We do not accept new tasks once we are shutdown if (running.load()) { - const std::lock_guard queue_lock(queue_mutex); + const std::lock_guard queue_lock(queue_mutex); - // Find where to insert the new task to maintain task order - auto it = std::lower_bound(queue.at(current_pool.pool_id).begin(), - queue.at(current_pool.pool_id).end(), - task, - std::less<>()); + // Find where to insert the new task to maintain task order + auto it = std::lower_bound(queue.at(current_pool.pool_id).begin(), + queue.at(current_pool.pool_id).end(), + task, + std::less<>()); - // Insert before the found position - queue.at(current_pool.pool_id).insert(it, std::forward>(task)); + // Insert before the found position + queue.at(current_pool.pool_id).insert(it, std::forward>(task)); } // Notify all threads that there is a new task to be processed From 6281e95b9e3d124c2ae9ac563d4ee440bb8edda2 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 15:16:53 +1000 Subject: [PATCH 79/87] Base `Sync` on `Group` --- src/dsl/word/Sync.hpp | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index 95fcae984..cf22a8ad3 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -25,6 +25,7 @@ #include "../../threading/ReactionTask.hpp" #include "../../util/GroupDescriptor.hpp" +#include "Group.hpp" namespace NUClear { namespace dsl { @@ -65,19 +66,7 @@ namespace dsl { * Note that the developer is not limited to the use of a struct; any declared type will work. */ template - struct Sync { - - static const util::GroupDescriptor group_descriptor; - - template - static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - return group_descriptor; - } - }; - - // Initialise the group descriptor - template - const util::GroupDescriptor Sync::group_descriptor = {util::GroupDescriptor::get_unique_group_id(), 1}; + struct Sync : Group {}; } // namespace word } // namespace dsl From 839556943036e3d0bd87de8a201b2e8d46917026 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 15:18:05 +1000 Subject: [PATCH 80/87] Include `Group` in `Reactor` --- src/Reactor.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Reactor.hpp b/src/Reactor.hpp index 2e495e1c5..2ae49195f 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -97,6 +97,9 @@ namespace dsl { template struct Pool; + template + struct Group; + namespace emit { template struct Local; @@ -412,6 +415,7 @@ class Reactor { #include "dsl/word/Always.hpp" #include "dsl/word/Buffer.hpp" #include "dsl/word/Every.hpp" +#include "dsl/word/Group.hpp" #include "dsl/word/IO.hpp" #include "dsl/word/Last.hpp" #include "dsl/word/MainThread.hpp" From fdd146974517f51532f6de7c14cb1f8fb00d778b Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 15:45:34 +1000 Subject: [PATCH 81/87] Remove valgrind and sanitiser stuff --- CMakeLists.txt | 34 ---------------------------------- src/PowerPlant.cpp | 32 -------------------------------- src/PowerPlant.hpp | 20 +------------------- 3 files changed, 1 insertion(+), 85 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d8a14530..a5b3686b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,40 +48,6 @@ if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MASTER_PROJECT ON) endif() -option(USE_VALGRIND "Add extra annotations for better valgrind handling" OFF) -if(USE_VALGRIND) - add_compile_options(-DUSE_VALGRIND) -endif(USE_VALGRIND) - -# Enable address sanitizer -option(USE_ASAN "Enable address sanitization" OFF) -if(USE_ASAN) - add_compile_options(-fsanitize=address -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) - add_link_options(-fsanitize=address) - link_libraries(asan) -endif(USE_ASAN) - -option(USE_LSAN "Enable leak sanitization" OFF) -if(USE_LSAN) - add_compile_options(-fsanitize=leak -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) - add_link_options(-fsanitize=leak) - link_libraries(lsan) -endif(USE_LSAN) - -option(USE_TSAN "Enable thread sanitization" OFF) -if(USE_TSAN) - add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) - add_link_options(-fsanitize=thread) - link_libraries(tsan) -endif(USE_TSAN) - -option(USE_UBSAN "Enable undefined behaviour sanitization" OFF) -if(USE_UBSAN) - add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) - add_link_options(-fsanitize=undefined) - link_libraries(ubsan) -endif(USE_UBSAN) - # If this option is set we are building using continous integration option(CI_BUILD "Enable build options for building in the CI server" OFF) diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index 382f1ecfc..abab9a850 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -18,38 +18,6 @@ #include "PowerPlant.hpp" -// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 -#if defined(USE_VALGRIND) && !defined(NDEBUG) -// NOLINTBEGIN -namespace std { -extern "C" { -static void* execute_native_thread_routine(void* __p) { - thread::_State_ptr __t{static_cast(__p)}; - __t->_M_run(); - return nullptr; -} -void thread::_M_start_thread(_State_ptr state, void (*depend)()) { - // Make sure it's not optimized out, not even with LTO. - asm("" : : "rm"(depend)); - - if (!__gthread_active_p()) { - #if __cpp_exceptions - throw system_error(make_error_code(errc::operation_not_permitted), "Enable multithreading to use std::thread"); - #else - __builtin_abort(); - #endif - } - - const int err = __gthread_create(&_M_id._M_thread, &execute_native_thread_routine, state.get()); - if (err) - __throw_system_error(err); - state.release(); -} -} -} // namespace std -// NOLINTEND -#endif // defined(USE_VALGRIND) && !defined(NDEBUG) - namespace NUClear { PowerPlant* PowerPlant::powerplant = nullptr; // NOLINT diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 13c3682b0..2de776bd4 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -19,15 +19,6 @@ #ifndef NUCLEAR_POWERPLANT_HPP #define NUCLEAR_POWERPLANT_HPP -// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 -#if defined(USE_VALGRIND) && !defined(NDEBUG) - #include - #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE - #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER - #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) // NOLINT - #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) // NOLINT -#endif // defined(USE_VALGRIND) && !defined(NDEBUG) - #include #include #include @@ -37,19 +28,10 @@ #include #include #include +#include #include #include -// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 -#if defined(USE_VALGRIND) && !defined(NDEBUG) - // NOLINTNEXTLINE - #define _GLIBCXX_THREAD_IMPL 1 -#endif // defined(USE_VALGRIND) && !defined(NDEBUG) -#include -#if defined(USE_VALGRIND) && !defined(NDEBUG) - #undef _GLIBCXX_THREAD_IMPL -#endif // defined(USE_VALGRIND) && !defined(NDEBUG) - // Utilities #include "LogLevel.hpp" #include "message/LogMessage.hpp" From bbb3abcea7b0081bb06d46fe3196436ab26af1d4 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 15:43:59 +1000 Subject: [PATCH 82/87] Add options for valgrind, sanitisers, and profiling --- CMakeLists.txt | 58 ++++++++++++++++++++++++++++++++++ src/PowerPlant.hpp | 6 ++++ src/util/valgrind/valgrind.cpp | 54 +++++++++++++++++++++++++++++++ src/util/valgrind/valgrind.hpp | 25 +++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 src/util/valgrind/valgrind.cpp create mode 100644 src/util/valgrind/valgrind.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a5b3686b0..0d4554522 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,64 @@ if(CI_BUILD) add_compile_options(-Werror) endif(CI_BUILD) +# When using valgrind data race detector or helgrind (both are for debugging issues with threading and data races) +# certain symbols need to be overridden to enable better stack traces. This option enables those extra things +# See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 for more details +# +# This is disabled by default and will only be enabled when this option is enabled AND NDEBUG is NOT defined (so compiling in debug mode) +option(USE_VALGRIND "Add extra annotations for better valgrind handling" OFF) +if(USE_VALGRIND) + add_compile_options(-DUSE_VALGRIND) +endif(USE_VALGRIND) + +############################################################################### +### Options for enabling different sanitisers. ### +### ### +### User beware: ### +### Not all sanitisers can be enabled at the same time. ### +############################################################################### +option(USE_ASAN "Enable address sanitization" OFF) +if(USE_ASAN) + add_compile_options(-fsanitize=address -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=address) + link_libraries(asan) +endif(USE_ASAN) + +option(USE_LSAN "Enable leak sanitization" OFF) +if(USE_LSAN) + add_compile_options(-fsanitize=leak -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=leak) + link_libraries(lsan) +endif(USE_LSAN) + +option(USE_TSAN "Enable thread sanitization" OFF) +if(USE_TSAN) + add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=thread) + link_libraries(tsan) +endif(USE_TSAN) + +option(USE_UBSAN "Enable undefined behaviour sanitization" OFF) +if(USE_UBSAN) + add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer -U_FORTIFY_SOURCE -fno-common) + add_link_options(-fsanitize=undefined) + link_libraries(ubsan) +endif(USE_UBSAN) + +# Option for enabling code profiling. Disabled by default +option(ENABLE_PROFILING "Compile with profiling support enabled.") +if(ENABLE_PROFILING) + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + message( + WARNING + "Profiling is enabled but no debugging symbols will be kept in the compiled binaries. This may cause fine-grained profilling data to be lost." + ) + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg -fprofile-arcs") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg -fprofile-arcs") + set(CMAKE_LINKER "${CMAKE_LINKER_FLAGS} -pg -fprofile-arcs") +endif(ENABLE_PROFILING) + # Make the compiler display colours always (even when we build with ninja) if(CMAKE_CXX_COMPILER_ID MATCHES GNU) add_compile_options(-fdiagnostics-color=always) diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 2de776bd4..9b19ab47a 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -19,6 +19,12 @@ #ifndef NUCLEAR_POWERPLANT_HPP #define NUCLEAR_POWERPLANT_HPP +// This file needs to be included in a spot that means it will be included everywhere +// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 +#if defined(USE_VALGRIND) && !defined(NDEBUG) + #include "util/valgrind/valgrind.hpp" +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) + #include #include #include diff --git a/src/util/valgrind/valgrind.cpp b/src/util/valgrind/valgrind.cpp new file mode 100644 index 000000000..54c2ad91c --- /dev/null +++ b/src/util/valgrind/valgrind.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#if defined(USE_VALGRIND) && !defined(NDEBUG) + // NOLINTBEGIN + // _State_ptr and State are private members od std::thread, this define allows them to not be private + #define _GLIBCXX_THREAD_IMPL 1 + #include + #undef _GLIBCXX_THREAD_IMPL + + #include + +namespace std { +extern "C" { +static void* execute_native_thread_routine(void* __p) { + thread::_State_ptr __t{static_cast(__p)}; + __t->_M_run(); + return nullptr; +} +void thread::_M_start_thread(_State_ptr state, void (*depend)()) { + // Make sure it's not optimized out, not even with LTO. + asm("" : : "rm"(depend)); + + if (!__gthread_active_p()) { + #if __cpp_exceptions + throw system_error(make_error_code(errc::operation_not_permitted), "Enable multithreading to use std::thread"); + #else + __builtin_abort(); + #endif + } + + const int err = __gthread_create(&_M_id._M_thread, &execute_native_thread_routine, state.get()); + if (err) + __throw_system_error(err); + state.release(); +} +} +} // namespace std +// NOLINTEND +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) diff --git a/src/util/valgrind/valgrind.hpp b/src/util/valgrind/valgrind.hpp new file mode 100644 index 000000000..a9613b5df --- /dev/null +++ b/src/util/valgrind/valgrind.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2023 Alex Biddulph + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +// See https://valgrind.org/docs/manual/drd-manual.html#drd-manual.CXX11 +#if defined(USE_VALGRIND) && !defined(NDEBUG) + #include + #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE + #undef _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER + #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(addr) ANNOTATE_HAPPENS_BEFORE(addr) // NOLINT + #define _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(addr) ANNOTATE_HAPPENS_AFTER(addr) // NOLINT +#endif // defined(USE_VALGRIND) && !defined(NDEBUG) From cbb3134ce4e19bb4daeee6dc42f892e16b044cd1 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 15:47:58 +1000 Subject: [PATCH 83/87] Fix up cmake formatting --- .cmake-format | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.cmake-format b/.cmake-format index 242171f55..ff3057140 100644 --- a/.cmake-format +++ b/.cmake-format @@ -26,14 +26,11 @@ command_case: lower # Format keywords consistently as 'lower' or 'upper' case keyword_case: upper +# enable comment markup parsing and reflow +enable_markup: False + # Setup all the additional commands additional_commands: - header_library: - kwargs: - NAME: "*" - HEADER: "*" - PATH_SUFFIX: "*" - URL: "*" header_library: kwargs: NAME: "*" From 9ea6f63605411803364af2a85ee65b9dd574d987 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Wed, 21 Jun 2023 15:56:22 +1000 Subject: [PATCH 84/87] clang-tidy --- src/util/GroupDescriptor.hpp | 2 +- src/util/ThreadPoolDescriptor.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/GroupDescriptor.hpp b/src/util/GroupDescriptor.hpp index c625a8e7a..0d70b4d9f 100644 --- a/src/util/GroupDescriptor.hpp +++ b/src/util/GroupDescriptor.hpp @@ -39,7 +39,7 @@ namespace util { /** * @brief Return the next unique ID for a new group */ - static uint64_t get_unique_group_id() { + static uint64_t get_unique_group_id() noexcept { // Make group 0 the default group static std::atomic source{1}; return source++; diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp index 4c5f508b9..daa0271a6 100644 --- a/src/util/ThreadPoolDescriptor.hpp +++ b/src/util/ThreadPoolDescriptor.hpp @@ -43,7 +43,7 @@ namespace util { /** * @brief Return the next unique ID for a new thread pool */ - static uint64_t get_unique_pool_id() { + static uint64_t get_unique_pool_id() noexcept { static std::atomic source{2}; return source++; } From 68da88a3388427c726e9ff4a70859a42898573f2 Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 23 Jun 2023 15:05:32 +1000 Subject: [PATCH 85/87] Rebase on main --- src/PowerPlant.cpp | 59 ++++++- src/Reactor.hpp | 10 +- src/dsl/Fusion.hpp | 6 +- src/dsl/Parse.hpp | 11 +- src/dsl/fusion/NoOp.hpp | 21 +-- src/dsl/word/Always.hpp | 124 ++----------- src/dsl/word/Buffer.hpp | 6 +- src/dsl/word/Every.hpp | 1 - src/dsl/word/IO.hpp | 1 - src/dsl/word/Last.hpp | 1 - src/dsl/word/MainThread.hpp | 25 ++- src/dsl/word/Network.hpp | 1 - src/dsl/word/Once.hpp | 1 - src/dsl/word/Optional.hpp | 1 - src/dsl/word/Sync.hpp | 82 +++++++-- src/dsl/word/TCP.hpp | 1 - src/dsl/word/UDP.hpp | 1 - src/dsl/word/Watchdog.hpp | 1 - src/dsl/word/emit/Delay.hpp | 2 +- src/dsl/word/emit/Direct.hpp | 7 +- src/dsl/word/emit/Initialise.hpp | 8 +- src/extension/ChronoController.hpp | 14 +- src/extension/IOController_Posix.hpp | 8 +- src/extension/IOController_Windows.hpp | 6 +- src/threading/Reaction.hpp | 15 +- src/threading/ReactionTask.hpp | 49 ++--- src/threading/TaskScheduler.cpp | 236 ++++--------------------- src/threading/TaskScheduler.hpp | 136 +++----------- src/util/CallbackGenerator.hpp | 99 +++++------ 29 files changed, 324 insertions(+), 609 deletions(-) diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index abab9a850..974e29931 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -18,6 +18,8 @@ #include "PowerPlant.hpp" +#include "threading/ThreadPoolTask.hpp" + namespace NUClear { PowerPlant* PowerPlant::powerplant = nullptr; // NOLINT @@ -28,36 +30,83 @@ PowerPlant::~PowerPlant() { powerplant = nullptr; } +void PowerPlant::on_startup(std::function&& func) { + if (is_running) { + throw std::runtime_error("Unable to do on_startup as the PowerPlant has already started"); + } + startup_tasks.push_back(func); +} + +void PowerPlant::add_thread_task(std::function&& task) { + tasks.push_back(task); +} + void PowerPlant::start() { // We are now running - is_running.store(true); + is_running = true; + + // Run all our Initialise scope tasks + for (auto&& func : startup_tasks) { + func(); + } + startup_tasks.clear(); // Direct emit startup event emit(std::make_unique()); - // Start all of the threads - scheduler.start(configuration.thread_count); + // Start all our threads + for (size_t i = 0; i < configuration.thread_count; ++i) { + tasks.push_back(threading::make_thread_pool_task(scheduler)); + } + + // Start all our tasks + for (auto& task : tasks) { + threads.push_back(std::make_unique(task)); + } + + // Start our main thread using our main task scheduler + threading::make_thread_pool_task(main_thread_scheduler)(); + + // Now wait for all the threads to finish executing + for (auto& thread : threads) { + try { + if (thread->joinable()) { + thread->join(); + } + } + // This gets thrown some time if between checking if joinable and joining + // the thread is no longer joinable + catch (const std::system_error&) { + } + } } void PowerPlant::submit(std::unique_ptr&& task) { scheduler.submit(std::move(task)); } +void PowerPlant::submit_main(std::unique_ptr&& task) { + main_thread_scheduler.submit(std::move(task)); +} + void PowerPlant::shutdown() { // Stop running before we emit the Shutdown event // Some things such as on depend on this flag and it's possible to miss it - is_running.store(false); + is_running = false; // Emit our shutdown event emit(std::make_unique()); // Shutdown the scheduler scheduler.shutdown(); + + // Shutdown the main threads scheduler + main_thread_scheduler.shutdown(); } bool PowerPlant::running() const { - return is_running.load(); + return is_running; } } // namespace NUClear diff --git a/src/Reactor.hpp b/src/Reactor.hpp index 2ae49195f..4d3855bdf 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -94,12 +94,6 @@ namespace dsl { template struct Sync; - template - struct Pool; - - template - struct Group; - namespace emit { template struct Local; @@ -298,7 +292,7 @@ class Reactor { template auto then(const std::string& label, Function&& callback, const util::Sequence& /*s*/) { - // Generate the identifier + // Generate the identifer std::vector identifier = {label, reactor.reactor_name, util::demangle(typeid(DSL).name()), @@ -415,14 +409,12 @@ class Reactor { #include "dsl/word/Always.hpp" #include "dsl/word/Buffer.hpp" #include "dsl/word/Every.hpp" -#include "dsl/word/Group.hpp" #include "dsl/word/IO.hpp" #include "dsl/word/Last.hpp" #include "dsl/word/MainThread.hpp" #include "dsl/word/Network.hpp" #include "dsl/word/Once.hpp" #include "dsl/word/Optional.hpp" -#include "dsl/word/Pool.hpp" #include "dsl/word/Priority.hpp" #include "dsl/word/Shutdown.hpp" #include "dsl/word/Single.hpp" diff --git a/src/dsl/Fusion.hpp b/src/dsl/Fusion.hpp index 0b7608e2e..19c993673 100644 --- a/src/dsl/Fusion.hpp +++ b/src/dsl/Fusion.hpp @@ -22,11 +22,10 @@ #include "../threading/ReactionHandle.hpp" #include "fusion/BindFusion.hpp" #include "fusion/GetFusion.hpp" -#include "fusion/GroupFusion.hpp" -#include "fusion/PoolFusion.hpp" #include "fusion/PostconditionFusion.hpp" #include "fusion/PreconditionFusion.hpp" #include "fusion/PriorityFusion.hpp" +#include "fusion/RescheduleFusion.hpp" namespace NUClear { namespace dsl { @@ -38,8 +37,7 @@ namespace dsl { , public fusion::GetFusion , public fusion::PreconditionFusion , public fusion::PriorityFusion - , public fusion::GroupFusion - , public fusion::PoolFusion + , public fusion::RescheduleFusion , public fusion::PostconditionFusion {}; } // namespace dsl diff --git a/src/dsl/Parse.hpp b/src/dsl/Parse.hpp index 239f5d66a..259a4e031 100644 --- a/src/dsl/Parse.hpp +++ b/src/dsl/Parse.hpp @@ -53,14 +53,9 @@ namespace dsl { Parse>(r); } - static inline util::GroupDescriptor group(threading::Reaction& r) { - return std::conditional_t::value, DSL, fusion::NoOp>::template group< - Parse>(r); - } - - static inline util::ThreadPoolDescriptor pool(threading::Reaction& r) { - return std::conditional_t::value, DSL, fusion::NoOp>::template pool< - Parse>(r); + static std::unique_ptr reschedule(std::unique_ptr&& task) { + return std::conditional_t::value, DSL, fusion::NoOp>::template reschedule( + std::move(task)); } static inline void postcondition(threading::ReactionTask& r) { diff --git a/src/dsl/fusion/NoOp.hpp b/src/dsl/fusion/NoOp.hpp index 70372e93e..7489fd0f3 100644 --- a/src/dsl/fusion/NoOp.hpp +++ b/src/dsl/fusion/NoOp.hpp @@ -19,12 +19,6 @@ #ifndef NUCLEAR_DSL_FUSION_NOOP_HPP #define NUCLEAR_DSL_FUSION_NOOP_HPP -#include - -#include "../../threading/Reaction.hpp" -#include "../../threading/ReactionTask.hpp" -#include "../../util/GroupDescriptor.hpp" -#include "../../util/ThreadPoolDescriptor.hpp" #include "../word/Priority.hpp" namespace NUClear { @@ -56,13 +50,9 @@ namespace dsl { } template - static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - return util::GroupDescriptor{}; - } - - template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - return util::ThreadPoolDescriptor{}; + static inline std::unique_ptr reschedule( + std::unique_ptr&& task) { + return std::move(task); } template @@ -84,9 +74,8 @@ namespace dsl { static inline int priority(threading::Reaction&); - static inline util::GroupDescriptor group(threading::Reaction&); - - static inline util::ThreadPoolDescriptor pool(threading::Reaction&); + static inline std::unique_ptr reschedule( + std::unique_ptr&& task); static inline void postcondition(threading::ReactionTask&); }; diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index c8aab28a7..fba5525e0 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -1,7 +1,6 @@ /* * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2022 Trent Houliston - * 2023 Trent Houliston , Alex Biddulph + * 2014-2017 Trent Houliston * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the @@ -20,13 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_ALWAYS_HPP #define NUCLEAR_DSL_WORD_ALWAYS_HPP -#include -#include -#include - -#include "../../threading/ReactionTask.hpp" -#include "../../util/ThreadPoolDescriptor.hpp" - namespace NUClear { namespace dsl { namespace word { @@ -43,8 +35,8 @@ namespace dsl { * Any reactions requested using this keyword will initialise upon system start-up and execute continually * until system shut-down. * - * Note that a task spawned from this request will execute in its own unique thread rather than the default - * thread pool. + * Note that a task spawned from this request will execute in its own unique thread rather than the threadpool. + * However, if the task is rescheduled (such as with Sync), it will then be moved into the threadpool. * * @par Infinite Loops * This word should be used in place of any reactions which would contain an infinite loop. That is, @@ -59,113 +51,35 @@ namespace dsl { * * @attention * Where possible, developers should avoid using this keyword. It has been provided, but should only be - * used when there is no other way to schedule the reaction. If a developer is tempted to use this keyword, + * used when there is no other way to scheduled the reaction. If a developer is tempted to use this keyword, * it is advised to review other options, such as on before resorting to this feature. * * @par Implements - * Pool * Bind */ struct Always { template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& reaction) { - static std::map pool_id; - static std::mutex mutex; + static inline void bind(const std::shared_ptr& reaction) { - const std::lock_guard lock(mutex); - if (pool_id.count(reaction.id) == 0) { - pool_id[reaction.id] = util::ThreadPoolDescriptor::get_unique_pool_id(); - } - return util::ThreadPoolDescriptor{pool_id[reaction.id], 1}; - } + reaction->unbinders.push_back([](threading::Reaction& r) { r.enabled = false; }); - template - static inline void bind(const std::shared_ptr& always_reaction) { - // Static map mapping reaction id (from the always reaction) to a pair of reaction pointers -- one for - // the always reaction and one for the idle reaction that we generate in this function - // The main purpose of this map is to ensure that the always reaction pointer doesn't get destroyed - static std::map, std::shared_ptr>> - reaction_store = {}; + // This is our function that runs forever until the powerplant exits + reaction->reactor.powerplant.add_thread_task([reaction] { + while (reaction->reactor.powerplant.running()) { + try { + // Get a task + auto task = reaction->get_task(); - // Generate a new reaction for an idle task - // The purpose of this reaction is to ensure that the always reaction is resubmitted in the event that - // the precondition fails (e.g. on> will fail the precondition if there are no X - // messages previously emitted) - // - // In the event that the precondition on the always reaction fails this idle task will run and resubmit - // both the always reaction and the idle reaction - // - // The idle reaction must have a lower priority than the always reaction and must also run in the same - // thread pool and group as the always reaction - auto idle_reaction = std::make_shared( - always_reaction->reactor, - std::vector{always_reaction->identifier[0] + " - IDLE Task", - always_reaction->identifier[1], - always_reaction->identifier[2], - always_reaction->identifier[3]}, - [always_reaction](threading::Reaction& idle_reaction) -> util::GeneratedCallback { - auto callback = [&idle_reaction, always_reaction](threading::ReactionTask& /*task*/) { - // Get a task for the always reaction and submit it to the scheduler - auto always_task = always_reaction->get_task(); - if (always_task) { - always_reaction->reactor.powerplant.submit(std::move(always_task)); + // If we got a real task back + if (task) { + task = task->run(std::move(task)); } - - // Get a task for the idle reaction and submit it to the scheduler - auto idle_task = idle_reaction.get_task(); - if (idle_task) { - // Set the thread pool on the task - idle_task->thread_pool_descriptor = DSL::pool(*always_reaction); - - // Make sure that idle reaction always has lower priority than the always reaction - idle_task->priority = DSL::priority(*always_reaction) - 1; - - // Submit the task to be run - idle_reaction.reactor.powerplant.submit(std::move(idle_task)); - } - }; - - // Make sure that idle reaction always has lower priority than the always reaction - return {DSL::priority(*always_reaction) - 1, - DSL::group(*always_reaction), - DSL::pool(*always_reaction), - callback}; - }); - - // Don't emit stats for the idle reaction - idle_reaction->emit_stats = false; - - // Keep this reaction handy so it doesn't go out of scope - reaction_store[always_reaction->id] = {always_reaction, idle_reaction}; - - // Create an unbinder for the always reaction - always_reaction->unbinders.push_back([](threading::Reaction& r) { - r.enabled = false; - reaction_store.erase(r.id); + } + catch (...) { + } + } }); - - // Get a task for the always reaction and submit it to the scheduler - auto always_task = always_reaction->get_task(); - if (always_task) { - always_reaction->reactor.powerplant.submit(std::move(always_task)); - } - - // Get a task for the idle reaction and submit it to the scheduler - auto idle_task = idle_reaction->get_task(); - if (idle_task) { - idle_reaction->reactor.powerplant.submit(std::move(idle_task)); - } - } - - template - static inline void postcondition(threading::ReactionTask& task) { - // Get a task for the always reaction and submit it to the scheduler - auto new_task = task.parent.get_task(); - if (new_task) { - task.parent.reactor.powerplant.submit(std::move(new_task)); - } } }; diff --git a/src/dsl/word/Buffer.hpp b/src/dsl/word/Buffer.hpp index 470bbc2a7..bc64afe21 100644 --- a/src/dsl/word/Buffer.hpp +++ b/src/dsl/word/Buffer.hpp @@ -19,8 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_BUFFER_HPP #define NUCLEAR_DSL_WORD_BUFFER_HPP -#include "../../threading/Reaction.hpp" - namespace NUClear { namespace dsl { namespace word { @@ -32,8 +30,8 @@ namespace dsl { * @details * @code on, Buffer>>() @endcode * In the case above, when the subscribing reaction is triggered, should there be less than n existing - * tasks associated with this reaction (either executing or in the queue), then a new task will be created and - * scheduled. However, should n tasks already be allocated, then this new task request will be ignored. + * tasks associated with this reaction (either executing or in the queue), then a new task will be created and + * scheduled. However, should n tasks already be allocated, then this new task request will be ignored. * * For best use, this word should be fused with at least one other binding DSL word. * diff --git a/src/dsl/word/Every.hpp b/src/dsl/word/Every.hpp index f670029c3..cdea390fa 100644 --- a/src/dsl/word/Every.hpp +++ b/src/dsl/word/Every.hpp @@ -21,7 +21,6 @@ #include -#include "../../threading/Reaction.hpp" #include "../operation/ChronoTask.hpp" #include "../operation/Unbind.hpp" #include "emit/Direct.hpp" diff --git a/src/dsl/word/IO.hpp b/src/dsl/word/IO.hpp index 3b5f83ce9..39c73136e 100644 --- a/src/dsl/word/IO.hpp +++ b/src/dsl/word/IO.hpp @@ -19,7 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_IO_HPP #define NUCLEAR_DSL_WORD_IO_HPP -#include "../../threading/Reaction.hpp" #include "../../util/platform.hpp" #include "../operation/Unbind.hpp" #include "../store/ThreadStore.hpp" diff --git a/src/dsl/word/Last.hpp b/src/dsl/word/Last.hpp index 25bb9ed79..ea2de272e 100644 --- a/src/dsl/word/Last.hpp +++ b/src/dsl/word/Last.hpp @@ -22,7 +22,6 @@ #include #include -#include "../../threading/Reaction.hpp" #include "../../util/MergeTransient.hpp" namespace NUClear { diff --git a/src/dsl/word/MainThread.hpp b/src/dsl/word/MainThread.hpp index 894d36128..817dc634e 100644 --- a/src/dsl/word/MainThread.hpp +++ b/src/dsl/word/MainThread.hpp @@ -19,8 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_MAINTHREAD_HPP #define NUCLEAR_DSL_WORD_MAINTHREAD_HPP -#include "../../threading/ReactionTask.hpp" -#include "../../util/ThreadPoolDescriptor.hpp" +#include "../../util/main_thread_id.hpp" namespace NUClear { namespace dsl { @@ -35,12 +34,30 @@ namespace dsl { * This will most likely be used with graphics related tasks. * * For best use, this word should be fused with at least one other binding DSL word. + * + * @par Implements + * Pre-condition */ struct MainThread { + using task_ptr = std::unique_ptr; + template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - return util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, 1}; + static inline std::unique_ptr reschedule( + std::unique_ptr&& task) { + + // If we are not the main thread, move us to the main thread + if (std::this_thread::get_id() != util::main_thread_id) { + + // Submit to the main thread scheduler + task->parent.reactor.powerplant.submit_main(std::move(task)); + + // We took the task away so return null + return nullptr; + } + + // Otherwise run! + return std::move(task); } }; diff --git a/src/dsl/word/Network.hpp b/src/dsl/word/Network.hpp index 7d02c0148..31219fcae 100644 --- a/src/dsl/word/Network.hpp +++ b/src/dsl/word/Network.hpp @@ -19,7 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_NETWORK_HPP #define NUCLEAR_DSL_WORD_NETWORK_HPP -#include "../../threading/Reaction.hpp" #include "../../util/network/sock_t.hpp" #include "../../util/serialise/Serialise.hpp" #include "../store/ThreadStore.hpp" diff --git a/src/dsl/word/Once.hpp b/src/dsl/word/Once.hpp index 99b59a4ed..f557c2762 100644 --- a/src/dsl/word/Once.hpp +++ b/src/dsl/word/Once.hpp @@ -19,7 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_ONCE_HPP #define NUCLEAR_DSL_WORD_ONCE_HPP -#include "../../threading/ReactionTask.hpp" #include "Single.hpp" namespace NUClear { diff --git a/src/dsl/word/Optional.hpp b/src/dsl/word/Optional.hpp index 54dfdcedf..f182a9b51 100644 --- a/src/dsl/word/Optional.hpp +++ b/src/dsl/word/Optional.hpp @@ -19,7 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_OPTIONAL_HPP #define NUCLEAR_DSL_WORD_OPTIONAL_HPP -#include "../../threading/Reaction.hpp" namespace NUClear { namespace dsl { namespace word { diff --git a/src/dsl/word/Sync.hpp b/src/dsl/word/Sync.hpp index cf22a8ad3..b4484f9fa 100644 --- a/src/dsl/word/Sync.hpp +++ b/src/dsl/word/Sync.hpp @@ -19,30 +19,25 @@ #ifndef NUCLEAR_DSL_WORD_SYNC_HPP #define NUCLEAR_DSL_WORD_SYNC_HPP -#include -#include -#include - -#include "../../threading/ReactionTask.hpp" -#include "../../util/GroupDescriptor.hpp" -#include "Group.hpp" - namespace NUClear { namespace dsl { namespace word { /** * @brief - * This is used to specify that only one reaction in this SyncGroup can run concurrently. + * This option sets the synchronisation for a group of tasks. * * @details - * @code on, Sync>() @endcode + * @code on, Sync>() @endcode * When a group of tasks has been synchronised, only one task from the group will execute at a given time. * * Should another task from this group be scheduled/requested (during execution of the current task), it will - * be sidelined into the task queue. + * be sidelined into a priority queue. + * + * Upon completion of the currently executing task, the queue will be polled to allow execution of the next + * task in this group. * - * Tasks in the queue are ordered based on their priority level, then their task id. + * Tasks in the synchronization queue are ordered based on their priority level, then their emission timestamp. * * For best use, this word should be fused with at least one other binding DSL word. * @@ -57,7 +52,7 @@ namespace dsl { * NUClear will have task and thread control so that system resources can be efficiently managed. * * @par Implements - * Group + * Pre-condition, Post-condition * * @tparam SyncGroup * the type/group to synchronize on. This needs to be a declared type within the system. It is common to @@ -66,7 +61,66 @@ namespace dsl { * Note that the developer is not limited to the use of a struct; any declared type will work. */ template - struct Sync : Group {}; + struct Sync { + + using task_ptr = std::unique_ptr; + + /// @brief our queue which sorts tasks by priority + static std::priority_queue queue; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + /// @brief how many tasks are currently running + static volatile bool running; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + /// @brief a mutex to ensure data consistency + static std::mutex mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + + template + static inline std::unique_ptr reschedule( + std::unique_ptr&& task) { + + // Lock our mutex + const std::lock_guard lock(mutex); + + // If we are already running then queue, otherwise return and set running + if (running) { + queue.push(std::move(task)); + return nullptr; + } + + running = true; + return std::move(task); + } + + template + static void postcondition(threading::ReactionTask& task) { + + // Lock our mutex + const std::lock_guard lock(mutex); + + // We are finished running + running = false; + + // If we have another task, add it + if (!queue.empty()) { + std::unique_ptr next_task( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + std::move(const_cast&>(queue.top()))); + queue.pop(); + + // Resubmit this task to the reaction queue + task.parent.reactor.powerplant.submit(std::move(next_task)); + } + } + }; + + template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,cert-err58-cpp) + std::priority_queue::task_ptr> Sync::queue; + + template + volatile bool Sync::running = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + + template + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + std::mutex Sync::mutex; } // namespace word } // namespace dsl diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index 2b5a569ad..34a13e3fb 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -22,7 +22,6 @@ #include #include "../../PowerPlant.hpp" -#include "../../threading/Reaction.hpp" #include "../../util/FileDescriptor.hpp" #include "../../util/platform.hpp" #include "IO.hpp" diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index d3b7720ab..83cd39442 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -22,7 +22,6 @@ #include #include "../../PowerPlant.hpp" -#include "../../threading/Reaction.hpp" #include "../../util/FileDescriptor.hpp" #include "../../util/network/get_interfaces.hpp" #include "../../util/platform.hpp" diff --git a/src/dsl/word/Watchdog.hpp b/src/dsl/word/Watchdog.hpp index c01c21cc5..66047fb85 100644 --- a/src/dsl/word/Watchdog.hpp +++ b/src/dsl/word/Watchdog.hpp @@ -19,7 +19,6 @@ #ifndef NUCLEAR_DSL_WORD_WATCHDOG_HPP #define NUCLEAR_DSL_WORD_WATCHDOG_HPP -#include "../../threading/Reaction.hpp" #include "../../util/demangle.hpp" #include "../operation/Unbind.hpp" #include "../store/DataStore.hpp" diff --git a/src/dsl/word/emit/Delay.hpp b/src/dsl/word/emit/Delay.hpp index f2f4b10dc..f58ce0502 100644 --- a/src/dsl/word/emit/Delay.hpp +++ b/src/dsl/word/emit/Delay.hpp @@ -34,7 +34,7 @@ namespace dsl { * @details * @code emit(data, delay(ticks), dataType); @endcode * Emissions under this scope will wait for the provided time delay, and then emit the object utilising a - * local emit (that is, normal thread pool distribution). + * local emit (that is, normal threadpool distribution). * * @param data * the data to emit diff --git a/src/dsl/word/emit/Direct.hpp b/src/dsl/word/emit/Direct.hpp index 817462a4e..7cc6be073 100644 --- a/src/dsl/word/emit/Direct.hpp +++ b/src/dsl/word/emit/Direct.hpp @@ -32,7 +32,7 @@ namespace dsl { /** * @brief * When emitting data under this scope, the tasks created as a result of this emission will bypass the - * thread pool, and be executed immediately. + * threadpool, and be executed immediately. * * @details * @code emit(data, dataType); @endcode @@ -52,7 +52,7 @@ namespace dsl { template struct Direct { - static void emit(PowerPlant& powerplant, std::shared_ptr data) { + static void emit(PowerPlant& /*powerplant*/, std::shared_ptr data) { // Run all our reactions that are interested for (auto& reaction : store::TypeCallbackStore::get()) { @@ -63,8 +63,7 @@ namespace dsl { auto task = reaction->get_task(); if (task) { - task->immediate = true; - powerplant.submit(std::move(task)); + task = task->run(std::move(task)); } } catch (const std::exception& ex) { diff --git a/src/dsl/word/emit/Initialise.hpp b/src/dsl/word/emit/Initialise.hpp index 4e8a3a437..4b0912bcb 100644 --- a/src/dsl/word/emit/Initialise.hpp +++ b/src/dsl/word/emit/Initialise.hpp @@ -19,7 +19,7 @@ #ifndef NUCLEAR_DSL_WORD_EMIT_INITIALISE_HPP #define NUCLEAR_DSL_WORD_EMIT_INITIALISE_HPP -#include "Delay.hpp" +#include "Direct.hpp" namespace NUClear { namespace dsl { @@ -53,9 +53,9 @@ namespace dsl { static void emit(PowerPlant& powerplant, std::shared_ptr data) { - // Delay the emit by 0 seconds, this will delay the emit until the chrono controller starts, which - // will be when the system starts - Delay::emit(powerplant, data, std::chrono::seconds(0)); + auto task = [&powerplant, data] { emit::Direct::emit(powerplant, data); }; + + powerplant.on_startup(task); } }; diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 71e5e21fe..764c364bc 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -37,10 +37,8 @@ namespace extension { // Lock the mutex while we're doing stuff const std::lock_guard lock(mutex); - // Add our new task to the heap if we are still running - if (running) { - tasks.push_back(*task); - } + // Add our new task to the heap + tasks.push_back(*task); // Poke the system wait.notify_all(); @@ -57,7 +55,7 @@ namespace extension { return task.id == unbind.id; }); - // Remove if it exists + // Remove if if it exists if (it != tasks.end()) { tasks.erase(it); } @@ -69,7 +67,6 @@ namespace extension { // When we shutdown we notify so we quit now on().then("Shutdown Chrono Controller", [this] { const std::lock_guard lock(mutex); - running = false; wait.notify_all(); }); @@ -77,10 +74,6 @@ namespace extension { // Acquire the mutex lock so we can wait on it std::unique_lock lock(mutex); - if (!running) { - return; - } - // If we have tasks to do if (!tasks.empty()) { @@ -130,7 +123,6 @@ namespace extension { std::vector tasks; std::mutex mutex; std::condition_variable wait; - bool running{true}; NUClear::clock::duration wait_offset; }; diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 4b2df200b..9ef2eace8 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -118,7 +118,7 @@ namespace extension { on().then("Shutdown IO Controller", [this] { // Set shutdown to true so it won't try to poll again - shutdown.store(true); + shutdown = true; // A byte to send down the pipe char val = 0; @@ -133,7 +133,7 @@ namespace extension { on().then("IO Controller", [this] { // To make sure we don't get caught in a weird loop // shutdown keeps us out here - if (!shutdown.load()) { + if (!shutdown) { // TODO(trent): check for dirty here @@ -261,8 +261,8 @@ namespace extension { fd_t notify_recv{-1}; fd_t notify_send{-1}; - std::atomic shutdown{false}; - bool dirty = true; + bool shutdown = false; + bool dirty = true; std::mutex reaction_mutex; std::vector fds{}; std::vector reactions{}; diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index d3e1402c6..7a8f3e243 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -125,7 +125,7 @@ namespace extension { on().then("Shutdown IO Controller", [this] { // Set shutdown to true - shutdown.store(true); + shutdown = true; // Signal the notifier event to return from WSAWaitForMultipleEvents() and shutdown if (!WSASetEvent(notifier)) { @@ -136,7 +136,7 @@ namespace extension { }); on().then("IO Controller", [this] { - if (!shutdown.load()) { + if (!shutdown) { // Wait for events auto event_index = WSAWaitForMultipleEvents(static_cast(events.size()), events.data(), @@ -247,7 +247,7 @@ namespace extension { WSAEVENT notifier; - std::atomic shutdown{false}; + bool shutdown = false; bool reactions_list_dirty = false; std::mutex reaction_mutex; diff --git a/src/threading/Reaction.hpp b/src/threading/Reaction.hpp index 0c076b589..08d64eb34 100644 --- a/src/threading/Reaction.hpp +++ b/src/threading/Reaction.hpp @@ -25,7 +25,6 @@ #include #include -#include "../util/GeneratedCallback.hpp" #include "ReactionTask.hpp" namespace NUClear { @@ -51,7 +50,7 @@ namespace threading { public: // The type of the generator that is used to create functions for ReactionTask objects - using TaskGenerator = std::function; + using TaskGenerator = std::function(Reaction&)>; /** * @brief Constructs a new Reaction with the passed callback generator and options @@ -75,15 +74,13 @@ namespace threading { } // Run our generator to get a functor we can run - auto callback = generator(*this); + int priority = 0; + std::function(std::unique_ptr &&)> func; + std::tie(priority, func) = generator(*this); // If our generator returns a valid function - if (callback) { - return std::make_unique(*this, - callback.priority, - callback.group, - callback.pool, - std::move(callback.callback)); + if (func) { + return std::make_unique(*this, priority, std::move(func)); } // Otherwise we return a null pointer diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index a53c3bffc..cceb1dd11 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -26,8 +26,6 @@ #include #include "../message/ReactionStatistics.hpp" -#include "../util/GroupDescriptor.hpp" -#include "../util/ThreadPoolDescriptor.hpp" #include "../util/platform.hpp" namespace NUClear { @@ -53,7 +51,7 @@ namespace threading { public: /// Type of the functions that ReactionTasks execute - using TaskFunction = std::function&)>; + using TaskFunction = std::function>(std::unique_ptr>&&)>; /** * @brief Gets the current executing task, or nullptr if there isn't one. @@ -67,17 +65,11 @@ namespace threading { /** * @brief Creates a new ReactionTask object bound with the parent Reaction object (that created it) and task. * - * @param parent the Reaction object that spawned this ReactionTask. - * @param priority the priority to use when executing this task. - * @param group_descriptor the descriptor for the group that this task should run in - * @param thread_pool_descriptor the descriptor for the thread pool that this task should be queued in - * @param callback the data bound callback to be executed in the thread pool. + * @param parent the Reaction object that spawned this ReactionTask. + * @param priority the priority to use when executing this task. + * @param callback the data bound callback to be executed in the threadpool. */ - Task(ReactionType& parent, - const int& priority, - const util::GroupDescriptor& group_descriptor, - const util::ThreadPoolDescriptor& thread_pool_descriptor, - TaskFunction&& callback) + Task(ReactionType& parent, int priority, TaskFunction&& callback) : parent(parent) , id(++task_id_source) , priority(priority) @@ -91,8 +83,6 @@ namespace threading { clock::time_point(std::chrono::seconds(0)), nullptr)) , emit_stats(parent.emit_stats && (current_task != nullptr ? current_task->emit_stats : true)) - , group_descriptor(group_descriptor) - , thread_pool_descriptor(thread_pool_descriptor) , callback(callback) {} @@ -103,17 +93,20 @@ namespace threading { * This runs the internal data bound task and times how long the execution takes. These figures can then be * used in a debugging context to calculate how long callbacks are taking to run. */ - inline void run() { + inline std::unique_ptr> run(std::unique_ptr>&& us) { // Update our current task Task* old_task = current_task; current_task = this; - // Run our callback - callback(*this); + // Run our callback at catch the returned task (to see if it rescheduled itself) + us = callback(std::move(us)); // Reset our task back current_task = old_task; + + // Return our original task + return std::move(us); } /// @brief the parent Reaction object which spawned this @@ -128,18 +121,6 @@ namespace threading { /// reaction statistics becomes false for all created tasks. This is to stop infinite loops of death. bool emit_stats; - /// @brief details about the group that this task will run in - util::GroupDescriptor group_descriptor; - - /// @brief details about the thread pool that this task will run from, this will also influence what task queue - /// the tasks will be queued on - util::ThreadPoolDescriptor thread_pool_descriptor; - - /// @brief if this task should run immediately in the current thread. If immediate execution of this task is not - /// possible (e.g. due to group concurrency restrictions) this task will be queued as normal. This flag will be - /// set by `emit` - bool immediate{false}; - /// @brief the data bound callback to be executed /// @attention note this must be last in the list as the this pointer is passed to the callback generator TaskFunction callback; @@ -168,10 +149,10 @@ namespace threading { // nullptr is greater than anything else so it's removed from a queue first // higher priority tasks are greater than lower priority tasks // tasks created first (smaller ids) should run before tasks created later - return a == nullptr ? true - : b == nullptr ? false - : a->priority == b->priority ? a->id < b->id - : a->priority > b->priority; + return a == nullptr ? false + : b == nullptr ? true + : a->priority == b->priority ? a->id > b->id + : a->priority < b->priority; } // Alias the templated Task so that public API remains intact diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index f38bc9305..e8c822aca 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -1,7 +1,6 @@ /* * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2022 Trent Houliston - * 2023 Trent Houliston , Alex Biddulph + * 2014-2017 Trent Houliston * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the @@ -19,231 +18,62 @@ #include "TaskScheduler.hpp" -#include -#include -#include -#include -#include -#include -#include - -#include "../dsl/word/MainThread.hpp" -#include "../util/update_current_thread_priority.hpp" - namespace NUClear { namespace threading { - bool TaskScheduler::is_runnable(const std::unique_ptr& task, const uint64_t& pool_id) { - - // Task can run if it is meant to run on the current thread pool - const bool correct_pool = pool_id == task->thread_pool_descriptor.pool_id; - - // Task can run if the group it belongs to has spare threads - if (groups.at(task->group_descriptor.group_id) < task->group_descriptor.thread_count && correct_pool) { - // This task is about to run in this group, increase the number of active tasks in the group - groups.at(task->group_descriptor.group_id)++; - return true; - } - - return false; - } - - void TaskScheduler::run_task(std::unique_ptr&& task) { - if (task) { - task->run(); - - // This task is no longer running, decrease the number of active tasks in the group - /* mutex scope */ { - const std::lock_guard group_lock(queue_mutex); - groups.at(task->group_descriptor.group_id)--; - } - } - } - - void TaskScheduler::pool_func(const util::ThreadPoolDescriptor& pool) { - while (running.load() || !queue.at(pool.pool_id).empty()) { - // Wait at a high (but not realtime) priority to reduce latency - // for picking up a new task - update_current_thread_priority(1000); - - run_task(get_task(pool.pool_id)); - } - } - - TaskScheduler::TaskScheduler() { - // Setup everything (thread pool, task queue, mutex, condition variable) for the main thread here - pools[util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID] = - util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, 1}; - pool_map[std::this_thread::get_id()] = util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID; - queue.emplace(util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID, std::vector>{}); - } - - void TaskScheduler::start_threads(const util::ThreadPoolDescriptor& pool) { - // The main thread never needs to be started - if (pool.pool_id != util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID) { - const std::lock_guard pool_lock(pool_mutex); - for (size_t i = 0; i < pool.thread_count; ++i) { - threads.push_back(std::make_unique(&TaskScheduler::pool_func, this, pool)); - pool_map[threads.back()->get_id()] = pool.pool_id; - } - } - } - - void TaskScheduler::create_pool(const util::ThreadPoolDescriptor& pool) { - // Pool already exists - /* mutex scope */ { - const std::lock_guard pool_lock(pool_mutex); - if (pools.count(pool.pool_id) > 0 && pools.at(pool.pool_id).thread_count > 0) { - return; - } - - // Make a copy of the pool descriptor - pools[pool.pool_id] = util::ThreadPoolDescriptor{pool.pool_id, pool.thread_count}; - } - - // Make sure the task queue is created for this pool - /* mutex scope */ { - const std::lock_guard queue_lock(queue_mutex); - if (queue.count(pool.pool_id) == 0) { - queue.emplace(pool.pool_id, std::vector>{}); - } - } - - // If the scheduler has not yet started then don't start the threads for this pool yet - if (started.load()) { - start_threads(pool); - } - } - - void TaskScheduler::start(const size_t& thread_count) { - - // Make the default pool - create_pool(util::ThreadPoolDescriptor{util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID, thread_count}); - - // The scheduler is now started - started.store(true); - - // Start all our threads - for (const auto& pool : pools) { - start_threads(pool.second); - } - - // Run main thread tasks - pool_func(pools.at(util::ThreadPoolDescriptor::MAIN_THREAD_POOL_ID)); - - // Poke all of the threads to make sure they are awake - /* mutex scope */ { - const std::lock_guard queue_lock(queue_mutex); - queue_condition.notify_all(); - } - - // Now wait for all the threads to finish executing - for (auto& thread : threads) { - try { - if (thread->joinable()) { - thread->join(); - } - } - // This gets thrown some time if between checking if joinable and joining - // the thread is no longer joinable - catch (const std::system_error&) { - } - } - } - void TaskScheduler::shutdown() { - started.store(false); - running.store(false); - const std::lock_guard queue_lock(queue_mutex); - queue_condition.notify_all(); + { + const std::lock_guard lock(mutex); + running = false; + } + condition.notify_all(); } void TaskScheduler::submit(std::unique_ptr&& task) { - // Extract the thread pool descriptor from the current task - const util::ThreadPoolDescriptor current_pool = task->thread_pool_descriptor; - - // Make sure the pool is created - create_pool(current_pool); - - // Make sure we know about this group - /* mutex scope */ { - const std::lock_guard group_lock(queue_mutex); - const uint64_t group_id = task->group_descriptor.group_id; - if (groups.count(group_id) == 0) { - groups.emplace(group_id, 0); - } - } - - // Check to see if this task was the result of `emit` - // Direct tasks can run after shutdown and before starting, provided they can be run immediately - if (task->immediate) { - // Map the current thread to the thread pool it belongs to - uint64_t thread_pool = util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID; - /* mutex scope */ { - const std::lock_guard pool_lock(pool_mutex); - if (pool_map.count(std::this_thread::get_id()) > 0) { - thread_pool = pool_map.at(std::this_thread::get_id()); - } - } - - // Check to see if this task is runnable in the current thread - // If it isn't we can just queue it up with all of the other non-immediate task - if (is_runnable(task, thread_pool)) { - run_task(std::move(task)); - return; - } - } - // We do not accept new tasks once we are shutdown - if (running.load()) { - const std::lock_guard queue_lock(queue_mutex); + if (running) { - // Find where to insert the new task to maintain task order - auto it = std::lower_bound(queue.at(current_pool.pool_id).begin(), - queue.at(current_pool.pool_id).end(), - task, - std::less<>()); - - // Insert before the found position - queue.at(current_pool.pool_id).insert(it, std::forward>(task)); + /* Mutex Scope */ { + const std::lock_guard lock(mutex); + queue.push(std::forward>(task)); + } } - // Notify all threads that there is a new task to be processed - const std::lock_guard queue_lock(queue_mutex); - queue_condition.notify_all(); + // Notify a thread that it can proceed + condition.notify_one(); } - std::unique_ptr TaskScheduler::get_task(const uint64_t& pool_id) { + std::unique_ptr TaskScheduler::get_task() { - std::unique_lock queue_lock(queue_mutex); + // Obtain the lock + std::unique_lock lock(mutex); - // Keep looking for tasks while the scheduler is still running, or while there are still tasks to process - while (running.load() || !queue[pool_id].empty()) { + // While our queue is empty + while (queue.empty()) { - // Iterate over all the tasks in the current thread pool queue, looking for one that we can run - for (auto it = queue.at(pool_id).begin(); it != queue.at(pool_id).end(); ++it) { + // If the queue is empty we either wait or shutdown + if (!running) { - // Check if we can run the task - if (is_runnable(*it, pool_id)) { + // Notify any other threads that might be waiting on this condition + condition.notify_all(); - // Move the task out of the queue - std::unique_ptr task = std::move(*it); - - // Erase the old position in the queue - queue.at(pool_id).erase(it); - - // Return the task - return task; - } + // Return a nullptr to signify there is nothing on the queue + return nullptr; } // Wait for something to happen! - queue_condition.wait(queue_lock); + condition.wait(lock); } - // No more tasks and scheduler has shutdown - return nullptr; + // Return the type + // If you're wondering why all the ridiculousness, it's because priority queue is not as feature complete as it + // should be its 'top' method returns a const reference (which we can't use to move a unique pointer) + std::unique_ptr task( + std::move(const_cast&>(queue.top()))); // NOLINT + queue.pop(); + + return task; } } // namespace threading } // namespace NUClear diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 010a70c32..ca1b9f385 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -19,70 +19,52 @@ #ifndef NUCLEAR_THREADING_TASKSCHEDULER_HPP #define NUCLEAR_THREADING_TASKSCHEDULER_HPP +#include #include #include #include #include #include -#include +#include +#include #include -#include "../util/GroupDescriptor.hpp" -#include "../util/ThreadPoolDescriptor.hpp" #include "Reaction.hpp" -#include "ReactionTask.hpp" namespace NUClear { namespace threading { /** - * @brief This class is responsible for scheduling tasks and distributing them amongst threads. + * @brief This class is responsible for scheduling tasks and distributing them amoungst threads. * * @details - * PRIORITY - * what priority this task should run with - * tasks are ordered by priority -> creation order - * POOL - * which thread pool this task should execute in - * 0 being execute on the main thread - * 1 being the default pool - * Work out how to create other pools later (fold in always into this?) - * GROUP - * which grouping this task belongs to for concurrency (default to the 0 group) - * CONCURRENCY - * only run if there are less than this many tasks running in this group - * INLINE - * if the submitter of this task should wait until this task is finished before returning (for DIRECT - * emits) + * This task scheduler uses the options from each of the tasks to decide when to execute them in a thread. The + * rules + * are applied to the tasks in the following order. * * @em Priority * @code Priority

@endcode - * When a priority is encountered, the task will be scheduled to execute based on this. If one of the three - * normal options are specified (HIGH, DEFAULT and LOW), then within the specified Sync group, it will run - * before, normally or after other reactions. + * When a priority is encountered, the task will be scheduled to execute based on this. If one of the three normal + * options are specified (HIGH, DEFAULT and LOW), then within the specified Sync group, it will run before, + * normally + * or after other reactions. * @attention Note that if Priority is specified, the Sync type is ignored (Single is not). * * @em Sync * @code Sync @endcode - * When a Sync type is encountered, the system uses this as a compile time mutex flag. It will not allow two - * callbacks with the same Sync type to execute at the same time. It will effectively ensure that all of the - * callbacks with this type run in sequence with each other, rather then in parallel. It is also important to - * note again, that if the priority of a task is realtime, it will ignore Sync groups. + * When a Sync type is encounterd, the system uses this as a compile time mutex flag. It will not allow two + * callbacks + * with the same Sync type to execute at the same time. It will effectivly ensure that all of the callbacks with + * this type run in sequence with eachother, rather then in parallell. It is also important to note again, that if + * the priority of a task is realtime, it will ignore Sync groups. * * @em Single * @code Single @endcode - * If single is encountered while processing the function, and a Task object for this Reaction is already - * running in a thread, or waiting in the Queue, then this task is ignored and dropped from the system. + * If single is encountered while processing the function, and a Task object for this Reaction is already running + * in a thread, or waiting in the Queue, then this task is ignored and dropped from the system. */ class TaskScheduler { public: - /** - * @brief Constructs a new TaskScheduler instance, and builds the nullptr sync queue. - */ - TaskScheduler(); - - void start(const size_t& thread_count); - /** * @brief * Shuts down the scheduler, all waiting threads are woken, and any attempt to get a task results in an @@ -102,89 +84,27 @@ namespace threading { */ void submit(std::unique_ptr&& task); - private: /** * @brief Get a task object to be executed by a thread. * * @details * This method will get a task object to be executed from the queue. It will block until such a time as a - * task is available to be executed. For example, if a task with a particular sync type was out, then this + * task is available to be executed. For example, if a task with a paticular sync type was out, then this * thread would block until that sync type was no longer out, and then it would take a task. * * @return the task which has been given to be executed */ - std::unique_ptr get_task(const uint64_t& pool_id); - - /** - * @brief Creates a new thread pool and ensures threads and a task queue are allocated for it - * - * @param pool the descriptor for the thread pool to create - */ - void create_pool(const util::ThreadPoolDescriptor& pool); - - /** - * @brief The function that each thread runs - * - * @details This function will repeatedly query the task queue for new a task to run and then execute that task - * - * @param pool the thread pool to run from and the task queue to get tasks from - */ - void pool_func(const util::ThreadPoolDescriptor& pool); + std::unique_ptr get_task(); - /** - * @brief Start all threads for the given thread pool - */ - void start_threads(const util::ThreadPoolDescriptor& pool); - - /** - * @brief Execute the given task - * - * @details After execution of the task has completed the number of active tasks in the tasks' group is - * decremented - */ - void run_task(std::unique_ptr&& task); - - /** - * @brief Determines if the given task is able to be executed - * - * @details If the current thread is able to be executed the number of active tasks in the tasks' groups is - * incremented - * - * @param task the task to inspect - * @param pool_id the pool to run the task on - * @return true if the task is currently runnable - * @return false if the task is not currently runnable - */ - bool is_runnable(const std::unique_ptr& task, const uint64_t& pool_id); - - /// @brief if the scheduler is running, and accepting new tasks. If this is false and a new, non-immediate, task - /// is submitted it will be ignored - std::atomic running{true}; - /// @brief if the scheduler has been started. This is set to true after a call to start is made. Once this is - /// set to true all threads will begin executing tasks from the tasks queue - std::atomic started{false}; - - /// @brief A task queue for each thread pool. In each queue, each task is ordered by priority and then by task - /// id - std::map>> queue; - - /// @brief A map of group ids to the number of active tasks currently running in that group - std::map groups{}; - - /// @brief the mutex which our threads synchronize their access to the task queues and the group concurrency - /// counts - std::mutex queue_mutex; + private: + /// @brief if the scheduler is running or is shut down + volatile bool running{true}; + /// @brief our queue which sorts tasks by priority + std::priority_queue> queue{}; + /// @brief the mutex which our threads synchronize their access to this object + std::mutex mutex; /// @brief the condition object that threads wait on if they can't get a task - std::condition_variable queue_condition; - - /// @brief A vector of the running threads in the system - std::vector> threads; - /// @brief A map of pool descriptor ids to pool descriptors - std::map pools{}; - /// @brief A map of thread ids to the pools they belong to - std::map pool_map{}; - /// @brief the mutex which our threads synchronize their access to the thread pool maps and the threads list - std::mutex pool_mutex; + std::condition_variable condition; }; } // namespace threading diff --git a/src/util/CallbackGenerator.hpp b/src/util/CallbackGenerator.hpp index 3ab3e9181..b3ed6b6b6 100644 --- a/src/util/CallbackGenerator.hpp +++ b/src/util/CallbackGenerator.hpp @@ -19,11 +19,8 @@ #ifndef NUCLEAR_UTIL_CALLBACKGENERATOR_HPP #define NUCLEAR_UTIL_CALLBACKGENERATOR_HPP -#include - #include "../dsl/trait/is_transient.hpp" #include "../dsl/word/emit/Direct.hpp" -#include "../util/GeneratedCallback.hpp" #include "../util/MergeTransient.hpp" #include "../util/TransientDataElements.hpp" #include "../util/apply.hpp" @@ -47,15 +44,9 @@ namespace util { template struct CallbackGenerator { - // Don't use this constructor if F is of type CallbackGenerator - template ::type>::type, - CallbackGenerator>::value, - bool>::type = true> - CallbackGenerator(F&& callback) - : callback(std::forward(callback)) - , transients(std::make_shared::type>()) {} + CallbackGenerator(Function&& callback) + : callback(std::forward(callback)) + , transients(std::make_shared::type>()){}; template void merge_transients(std::tuple& data, @@ -68,7 +59,8 @@ namespace util { std::get(data))...); } - GeneratedCallback operator()(threading::Reaction& r) { + + std::pair operator()(threading::Reaction& r) { // Add one to our active tasks ++r.active_tasks; @@ -79,7 +71,7 @@ namespace util { --r.active_tasks; // We cancel our execution by returning an empty function - return {}; + return {0, threading::ReactionTask::TaskFunction()}; } // Bind our data to a variable (this will run in the dispatching thread) @@ -96,46 +88,53 @@ namespace util { --r.active_tasks; // We cancel our execution by returning an empty function - return {}; + return {0, threading::ReactionTask::TaskFunction()}; } // We have to make a copy of the callback because the "this" variable can go out of scope auto c = callback; - return GeneratedCallback(DSL::priority(r), - DSL::group(r), - DSL::pool(r), - [c, data](threading::ReactionTask& task) { - // Update our thread's priority to the correct level - update_current_thread_priority(task.priority); - - // Record our start time - task.stats->started = clock::now(); - - // We have to catch any exceptions - try { - // We call with only the relevant arguments to the passed function - util::apply_relevant(c, std::move(data)); - } - catch (...) { - - // Catch our exception if it happens - task.stats->exception = std::current_exception(); - } - - // Our finish time - task.stats->finished = clock::now(); - - // Run our postconditions - DSL::postcondition(task); - - // Take one from our active tasks - --task.parent.active_tasks; - - // Emit our reaction statistics if it wouldn't cause a loop - if (task.emit_stats) { - PowerPlant::powerplant->emit_shared(task.stats); - } - }); + return std::make_pair(DSL::priority(r), [c, data](std::unique_ptr&& task) { + // Check if we are going to reschedule + task = DSL::reschedule(std::move(task)); + + // If we still control our task + if (task) { + + // Update our thread's priority to the correct level + update_current_thread_priority(task->priority); + + // Record our start time + task->stats->started = clock::now(); + + // We have to catch any exceptions + try { + // We call with only the relevant arguments to the passed function + util::apply_relevant(c, std::move(data)); + } + catch (...) { + + // Catch our exception if it happens + task->stats->exception = std::current_exception(); + } + + // Our finish time + task->stats->finished = clock::now(); + + // Run our postconditions + DSL::postcondition(*task); + + // Take one from our active tasks + --task->parent.active_tasks; + + // Emit our reaction statistics if it wouldn't cause a loop + if (task->emit_stats) { + PowerPlant::powerplant->emit_shared(task->stats); + } + } + + // Return our task + return std::move(task); + }); } Function callback; From 21cfbed41fa27bf9dffaf7d149f2395cb6b9ddee Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 23 Jun 2023 15:10:57 +1000 Subject: [PATCH 86/87] Finish rebase --- src/PowerPlant.hpp | 33 +++++- src/dsl/fusion/GroupFusion.hpp | 98 ---------------- src/dsl/fusion/PoolFusion.hpp | 98 ---------------- src/dsl/fusion/RescheduleFusion.hpp | 109 ++++++++++++++++++ src/dsl/fusion/has_pool.hpp | 53 --------- .../{has_group.hpp => has_reschedule.hpp} | 21 ++-- src/dsl/word/Group.hpp | 87 -------------- src/dsl/word/Pool.hpp | 90 --------------- src/util/GeneratedCallback.hpp | 60 ---------- src/util/GroupDescriptor.hpp | 52 --------- src/util/ThreadPoolDescriptor.cpp | 27 ----- src/util/ThreadPoolDescriptor.hpp | 55 --------- src/util/main_thread_id.cpp | 6 + src/util/main_thread_id.hpp | 6 - tests/dsl/Always.cpp | 26 +---- tests/dsl/BlockNoData.cpp | 2 - tests/dsl/FlagMessage.cpp | 4 +- tests/dsl/IO.cpp | 2 +- tests/dsl/Last.cpp | 2 +- tests/dsl/MainThread.cpp | 11 +- tests/dsl/Optional.cpp | 3 + tests/dsl/Single.cpp | 8 +- tests/dsl/SingleSync.cpp | 68 ----------- tests/dsl/With.cpp | 3 +- 24 files changed, 180 insertions(+), 744 deletions(-) delete mode 100644 src/dsl/fusion/GroupFusion.hpp delete mode 100644 src/dsl/fusion/PoolFusion.hpp create mode 100644 src/dsl/fusion/RescheduleFusion.hpp delete mode 100644 src/dsl/fusion/has_pool.hpp rename src/dsl/fusion/{has_group.hpp => has_reschedule.hpp} (70%) delete mode 100644 src/dsl/word/Group.hpp delete mode 100644 src/dsl/word/Pool.hpp delete mode 100644 src/util/GeneratedCallback.hpp delete mode 100644 src/util/GroupDescriptor.hpp delete mode 100644 src/util/ThreadPoolDescriptor.cpp delete mode 100644 src/util/ThreadPoolDescriptor.hpp delete mode 100644 tests/dsl/SingleSync.cpp diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index 9b19ab47a..0e42ad624 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -41,11 +41,9 @@ // Utilities #include "LogLevel.hpp" #include "message/LogMessage.hpp" -#include "threading/ReactionTask.hpp" #include "threading/TaskScheduler.hpp" #include "util/FunctionFusion.hpp" #include "util/demangle.hpp" -#include "util/main_thread_id.hpp" #include "util/unpack.hpp" namespace NUClear { @@ -133,6 +131,22 @@ class PowerPlant { */ bool running() const; + /** + * @brief Adds a function to the set of startup tasks. + * + * @param func the task being added to the set of startup tasks. + * + * @throws std::runtime_error if the PowerPlant is already running/has already started. + */ + void on_startup(std::function&& func); + + /** + * @brief Adds a function to the set of tasks to be run when the PowerPlant starts up + * + * @param task The function to add to the task list + */ + void add_thread_task(std::function&& task); + /** * @brief Installs a reactor of a particular type to the system. * @@ -154,6 +168,13 @@ class PowerPlant { */ void submit(std::unique_ptr&& task); + /** + * @brief Submits a new task to the main threads thread pool to be queued and then executed. + * + * @param task The Reaction task to be executed in the thread pool + */ + void submit_main(std::unique_ptr&& task); + /** * @brief Log a message through NUClear's system. * @@ -238,12 +259,18 @@ class PowerPlant { private: /// @brief A list of tasks that must be run when the powerplant starts up std::vector> tasks; + /// @brief A vector of the running threads in the system + std::vector> threads; /// @brief Our TaskScheduler that handles distributing task to the pool threads threading::TaskScheduler scheduler; + /// @brief Our TaskScheduler that handles distributing tasks to the main thread + threading::TaskScheduler main_thread_scheduler; /// @brief Our vector of Reactors, will get destructed when this vector is std::vector> reactors; + /// @brief Tasks that will be run during the startup process + std::vector> startup_tasks; /// @brief True if the powerplant is running - std::atomic is_running{false}; + volatile bool is_running = false; }; /** diff --git a/src/dsl/fusion/GroupFusion.hpp b/src/dsl/fusion/GroupFusion.hpp deleted file mode 100644 index 552d0b25e..000000000 --- a/src/dsl/fusion/GroupFusion.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_FUSION_GROUPFUSION_HPP -#define NUCLEAR_DSL_FUSION_GROUPFUSION_HPP - -#include - -#include "../../threading/Reaction.hpp" -#include "../operation/DSLProxy.hpp" -#include "has_group.hpp" - -namespace NUClear { -namespace dsl { - namespace fusion { - - /// Type that redirects types without a group function to their proxy type - template - struct Group { - using type = std::conditional_t::value, Word, operation::DSLProxy>; - }; - - template > - struct GroupWords; - - /** - * @brief Metafunction that extracts all of the Words with a group function - * - * @tparam Word1 The word we are looking at - * @tparam WordN The words we have yet to look at - * @tparam FoundWords The words we have found with group functions - */ - template - struct GroupWords, std::tuple> - : public std::conditional_t< - has_group::type>::value, - /*T*/ GroupWords, std::tuple::type>>, - /*F*/ GroupWords, std::tuple>> {}; - - /** - * @brief Termination case for the GroupWords metafunction - * - * @tparam FoundWords The words we have found with group functions - */ - template - struct GroupWords, std::tuple> { - using type = std::tuple; - }; - - - // Default case where there are no group words - template - struct GroupFuser {}; - - // Case where there is only a single word remaining - template - struct GroupFuser> { - - template - static inline util::GroupDescriptor group(threading::Reaction& reaction) { - - // Return our group - return Word::template group(reaction); - } - }; - - // Case where there are 2 or more words remaining - template - struct GroupFuser> { - - template - static inline void group(threading::Reaction& /*reaction*/) { - throw std::runtime_error("Can not be a member of more than one group"); - } - }; - - template - struct GroupFusion : public GroupFuser>::type> {}; - - } // namespace fusion -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_FUSION_GROUPFUSION_HPP diff --git a/src/dsl/fusion/PoolFusion.hpp b/src/dsl/fusion/PoolFusion.hpp deleted file mode 100644 index 8c4c67221..000000000 --- a/src/dsl/fusion/PoolFusion.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_FUSION_POOLFUSION_HPP -#define NUCLEAR_DSL_FUSION_POOLFUSION_HPP - -#include - -#include "../../threading/Reaction.hpp" -#include "../operation/DSLProxy.hpp" -#include "has_pool.hpp" - -namespace NUClear { -namespace dsl { - namespace fusion { - - /// Type that redirects types without a pool function to their proxy type - template - struct Pool { - using type = std::conditional_t::value, Word, operation::DSLProxy>; - }; - - template > - struct PoolWords; - - /** - * @brief Metafunction that extracts all of the Words with a pool function - * - * @tparam Word1 The word we are looking at - * @tparam WordN The words we have yet to look at - * @tparam FoundWords The words we have found with pool functions - */ - template - struct PoolWords, std::tuple> - : public std::conditional_t< - has_pool::type>::value, - /*T*/ PoolWords, std::tuple::type>>, - /*F*/ PoolWords, std::tuple>> {}; - - /** - * @brief Termination case for the PoolWords metafunction - * - * @tparam FoundWords The words we have found with pool functions - */ - template - struct PoolWords, std::tuple> { - using type = std::tuple; - }; - - - // Default case where there are no pool words - template - struct PoolFuser {}; - - // Case where there is only a single word remaining - template - struct PoolFuser> { - - template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& reaction) { - - // Return our pool - return Word::template pool(reaction); - } - }; - - // Case where there are 2 or more words remaining - template - struct PoolFuser> { - - template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - throw std::runtime_error("Can not be a member of more than one pool"); - } - }; - - template - struct PoolFusion : public PoolFuser>::type> {}; - - } // namespace fusion -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_FUSION_POOLFUSION_HPP diff --git a/src/dsl/fusion/RescheduleFusion.hpp b/src/dsl/fusion/RescheduleFusion.hpp new file mode 100644 index 000000000..9ff5c7717 --- /dev/null +++ b/src/dsl/fusion/RescheduleFusion.hpp @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_DSL_FUSION_RESCHEDULEFUSION_HPP +#define NUCLEAR_DSL_FUSION_RESCHEDULEFUSION_HPP + +#include "../../threading/ReactionTask.hpp" +#include "../operation/DSLProxy.hpp" +#include "has_reschedule.hpp" + +namespace NUClear { +namespace dsl { + namespace fusion { + + /// Type that redirects types without a reschedule function to their proxy type + template + struct Reschedule { + using type = std::conditional_t::value, Word, operation::DSLProxy>; + }; + + template > + struct RescheduleWords; + + /** + * @brief Metafunction that extracts all of the Words with a reschedule function + * + * @tparam Word1 The word we are looking at + * @tparam WordN The words we have yet to look at + * @tparam FoundWords The words we have found with reschedule functions + */ + template + struct RescheduleWords, std::tuple> + : public std::conditional_t< + has_reschedule::type>::value, + /*T*/ + RescheduleWords, std::tuple::type>>, + /*F*/ RescheduleWords, std::tuple>> {}; + + /** + * @brief Termination case for the RescheduleWords metafunction + * + * @tparam FoundWords The words we have found with reschedule functions + */ + template + struct RescheduleWords, std::tuple> { + using type = std::tuple; + }; + + + // Default case where there are no reschedule words + template + struct RescheduleFuser {}; + + // Case where there is only a single word remaining + template + struct RescheduleFuser> { + + template + static inline std::unique_ptr reschedule( + std::unique_ptr&& task) { + + // Pass our task to see if it gets rescheduled and return the result + return Word::template reschedule(std::move(task)); + } + }; + + // Case where there is more 2 more more words remaining + template + struct RescheduleFuser> { + + template + static inline std::unique_ptr reschedule( + std::unique_ptr&& task) { + + // Pass our task to see if it gets rescheduled + auto ptr = Word1::template reschedule(std::move(task)); + + // If it was not rescheduled pass to the next rescheduler + if (ptr) { + return RescheduleFuser>::template reschedule(std::move(ptr)); + } + return ptr; + } + }; + + template + struct RescheduleFusion + : public RescheduleFuser>::type> {}; + + } // namespace fusion +} // namespace dsl +} // namespace NUClear + +#endif // NUCLEAR_DSL_FUSION_RESCHEDULEFUSION_HPP diff --git a/src/dsl/fusion/has_pool.hpp b/src/dsl/fusion/has_pool.hpp deleted file mode 100644 index 160916972..000000000 --- a/src/dsl/fusion/has_pool.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_FUSION_HAS_POOL_HPP -#define NUCLEAR_DSL_FUSION_HAS_POOL_HPP - -#include "../../threading/Reaction.hpp" -#include "NoOp.hpp" - -namespace NUClear { -namespace dsl { - namespace fusion { - - /** - * @brief SFINAE struct to test if the passed class has a pool function that conforms to the NUClear DSL - * - * @tparam T the class to check - */ - template - struct has_pool { - private: - using yes = std::true_type; - using no = std::false_type; - - template - static auto test(int) - -> decltype(U::template pool(std::declval()), yes()); - template - static no test(...); - - public: - static constexpr bool value = std::is_same(0)), yes>::value; - }; - - } // namespace fusion -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_FUSION_HAS_POOL_HPP diff --git a/src/dsl/fusion/has_group.hpp b/src/dsl/fusion/has_reschedule.hpp similarity index 70% rename from src/dsl/fusion/has_group.hpp rename to src/dsl/fusion/has_reschedule.hpp index 6451259a0..6bef8958c 100644 --- a/src/dsl/fusion/has_group.hpp +++ b/src/dsl/fusion/has_reschedule.hpp @@ -1,5 +1,6 @@ /* - * Copyright (C) 2023 Alex Biddulph + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the @@ -15,10 +16,10 @@ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef NUCLEAR_DSL_FUSION_HAS_GROUP_HPP -#define NUCLEAR_DSL_FUSION_HAS_GROUP_HPP +#ifndef NUCLEAR_DSL_FUSION_HAS_RESCHEDULE_HPP +#define NUCLEAR_DSL_FUSION_HAS_RESCHEDULE_HPP -#include "../../threading/Reaction.hpp" +#include "../../threading/ReactionTask.hpp" #include "NoOp.hpp" namespace NUClear { @@ -26,19 +27,21 @@ namespace dsl { namespace fusion { /** - * @brief SFINAE struct to test if the passed class has a group function that conforms to the NUClear DSL + * @brief SFINAE struct to test if the passed class has a reschedule function that conforms to the + * NUClear DSL * * @tparam T the class to check */ template - struct has_group { + struct has_reschedule { private: using yes = std::true_type; using no = std::false_type; template - static auto test(int) - -> decltype(U::template group(std::declval()), yes()); + static auto test(int) -> decltype(U::template reschedule( + std::declval>()), + yes()); template static no test(...); @@ -50,4 +53,4 @@ namespace dsl { } // namespace dsl } // namespace NUClear -#endif // NUCLEAR_DSL_FUSION_HAS_GROUP_HPP +#endif // NUCLEAR_DSL_FUSION_HAS_RESCHEDULE_HPP diff --git a/src/dsl/word/Group.hpp b/src/dsl/word/Group.hpp deleted file mode 100644 index 8c2780e43..000000000 --- a/src/dsl/word/Group.hpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_WORD_GROUP_HPP -#define NUCLEAR_DSL_WORD_GROUP_HPP - -#include -#include -#include - -#include "../../threading/ReactionTask.hpp" -#include "../../util/GroupDescriptor.hpp" - -namespace NUClear { -namespace dsl { - namespace word { - - /** - * @brief - * This is used to specify that only one reaction in this GroupType can run concurrently. - * - * @details - * @code on, Group>() @endcode - * When a group of tasks has been synchronised, only N task(s) from the group will execute at a given time. - * - * Should another task from this group be scheduled/requested (during execution of the current N task(s)), it - * will be sidelined into the task queue. - * - * Tasks in the queue are ordered based on their priority level, then their task id. - * - * For best use, this word should be fused with at least one other binding DSL word. - * - * @attention - * When using NUClear, developers should not make use of devices like a mutex. In the case of a mutex, threads - * will run and then block (leading to wasted resources on a number of inactive threads). By using Sync, - * NUClear will have task and thread control so that system resources can be efficiently managed. - * - * @par Implements - * Group - * - * @tparam GroupType - * the type/group to synchronize on. This needs to be a declared type within the system. It is common to - * simply use the reactors name (i.e; if the reactor is only syncing with one group). Should more than one - * group be required, the developer can declare structs within the system, to act as a group reference. - * Note that the developer is not limited to the use of a struct; any declared type will work. - * @tparam GroupConcurrency - * the number of tasks that should be allowed to run concurrently in this group. It is an error to specify a - * group concurrency less than 1. - */ - template - struct Group { - - static_assert(GroupConcurrency > 0, "Can not have a group with concurrency less than 1"); - - static const util::GroupDescriptor group_descriptor; - - template - static inline util::GroupDescriptor group(threading::Reaction& /*reaction*/) { - return group_descriptor; - } - }; - - // Initialise the group descriptor - template - const util::GroupDescriptor Group::group_descriptor = { - util::GroupDescriptor::get_unique_group_id(), - GroupConcurrency}; - - } // namespace word -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_WORD_GROUP_HPP diff --git a/src/dsl/word/Pool.hpp b/src/dsl/word/Pool.hpp deleted file mode 100644 index 3a56586dc..000000000 --- a/src/dsl/word/Pool.hpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_DSL_WORD_POOL_HPP -#define NUCLEAR_DSL_WORD_POOL_HPP - -#include -#include -#include - -#include "../../threading/ReactionTask.hpp" -#include "../../util/ThreadPoolDescriptor.hpp" - -namespace NUClear { -namespace dsl { - namespace word { - - /** - * @brief - * This is used to specify that this reaction should run in the designated thread pool - * - * @details - * @code on, Pool>() @endcode - * This DSL will cause the creation of a new thread pool with a specific number of threads allocated to it. - * - * All tasks for this reaction will be queued to run on threads from this thread pool. - * - * Tasks in the queue are ordered based on their priority level, then their task id. - * - * When this DSL is not specified the default thread pool will be used. For tasks that need to run on the main - * thread use MainThread. - * - * For best use, this word should be fused with at least one other binding DSL word. - * - * @attention - * This DSL should be used sparingly as having an increased number of threads running concurrently on the - * system can lead to a degradation in performance. - * - * @par Implements - * pool - * - * @tparam PoolType - * A struct that contains the details of the thread pool to create. This struct should contain a static int - * member that sets the number of threads that should be allocated to this pool. - * @code - * struct ThreadPool { - * static const int thread_count = 2; - * }; - * @endcode - * While it is valid to have a non-const static thread count in the struct, NUClear will not look at any - * changes to this value. Only the first value that NUClear sees will be used to initiate the thread pool. - */ - template - struct Pool { - - static_assert(PoolType::thread_count > 0, "Can not have a thread pool with less than 1 thread"); - - static const util::ThreadPoolDescriptor pool_descriptor; - - template - static inline util::ThreadPoolDescriptor pool(threading::Reaction& /*reaction*/) { - return pool_descriptor; - } - }; - - // Initialise the thread pool descriptor - template - const util::ThreadPoolDescriptor Pool::pool_descriptor = { - util::ThreadPoolDescriptor::get_unique_pool_id(), - PoolType::thread_count}; - - } // namespace word -} // namespace dsl -} // namespace NUClear - -#endif // NUCLEAR_DSL_WORD_POOL_HPP diff --git a/src/util/GeneratedCallback.hpp b/src/util/GeneratedCallback.hpp deleted file mode 100644 index 31ed39af0..000000000 --- a/src/util/GeneratedCallback.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_UTIL_GENERATEDCALLBACK_HPP -#define NUCLEAR_UTIL_GENERATEDCALLBACK_HPP - -#include - -#include "../threading/ReactionTask.hpp" -#include "GroupDescriptor.hpp" -#include "ThreadPoolDescriptor.hpp" - -namespace NUClear { -namespace util { - - /** - * @brief Generated callback for a task - */ - struct GeneratedCallback { - GeneratedCallback() = default; - GeneratedCallback(const int& priority, - const GroupDescriptor& group, - const ThreadPoolDescriptor& pool, - threading::ReactionTask::TaskFunction callback) - : priority(priority), group(group), pool(pool), callback(std::move(callback)) {} - /// @brief the priority this task should run with - int priority{0}; - /// @brief the descriptor for the group the task should run in - GroupDescriptor group{0, std::numeric_limits::max()}; - /// @brief the descriptor the thread pool and task queue that the should run in - ThreadPoolDescriptor pool{util::ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID, 0}; - /// @brief the function that should be executed in order to run the task - threading::ReactionTask::TaskFunction callback{}; - - /** - * @return true if this represents a valid callback object - */ - operator bool() const { - return bool(callback); - } - }; - -} // namespace util -} // namespace NUClear - -#endif // NUCLEAR_UTIL_GENERATEDCALLBACK_HPP diff --git a/src/util/GroupDescriptor.hpp b/src/util/GroupDescriptor.hpp deleted file mode 100644 index 0d70b4d9f..000000000 --- a/src/util/GroupDescriptor.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_UTIL_GROUPDESCRIPTOR_HPP -#define NUCLEAR_UTIL_GROUPDESCRIPTOR_HPP - -#include -#include -#include -#include - -namespace NUClear { -namespace util { - - /** - * @brief A description of a group - */ - struct GroupDescriptor { - /// @brief a unique identifier for this pool - uint64_t group_id{0}; - - /// @brief the maximum number of threads that can run concurrently in this group - size_t thread_count{std::numeric_limits::max()}; - - /** - * @brief Return the next unique ID for a new group - */ - static uint64_t get_unique_group_id() noexcept { - // Make group 0 the default group - static std::atomic source{1}; - return source++; - } - }; - -} // namespace util -} // namespace NUClear - -#endif // NUCLEAR_UTIL_GROUPDESCRIPTOR_HPP diff --git a/src/util/ThreadPoolDescriptor.cpp b/src/util/ThreadPoolDescriptor.cpp deleted file mode 100644 index 606804f1b..000000000 --- a/src/util/ThreadPoolDescriptor.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "ThreadPoolDescriptor.hpp" - -namespace NUClear { -namespace util { - - const uint64_t ThreadPoolDescriptor::MAIN_THREAD_POOL_ID = 0; - const uint64_t ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID = 1; - -} // namespace util -} // namespace NUClear diff --git a/src/util/ThreadPoolDescriptor.hpp b/src/util/ThreadPoolDescriptor.hpp deleted file mode 100644 index daa0271a6..000000000 --- a/src/util/ThreadPoolDescriptor.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2023 Alex Biddulph - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_UTIL_THREADPOOL_HPP -#define NUCLEAR_UTIL_THREADPOOL_HPP - -#include -#include -#include - -namespace NUClear { -namespace util { - - /** - * @brief A description of a thread pool - */ - struct ThreadPoolDescriptor { - /// @brief a unique identifier for this pool - uint64_t pool_id{ThreadPoolDescriptor::DEFAULT_THREAD_POOL_ID}; - - /// @brief the number of threads this thread pool will use - size_t thread_count{0}; - - /// @brief the ID of the main thread pool (not to be confused with the ID of the main thread) - static const uint64_t MAIN_THREAD_POOL_ID; - /// @brief the ID of the default thread pool - static const uint64_t DEFAULT_THREAD_POOL_ID; - - /** - * @brief Return the next unique ID for a new thread pool - */ - static uint64_t get_unique_pool_id() noexcept { - static std::atomic source{2}; - return source++; - } - }; - -} // namespace util -} // namespace NUClear - -#endif // NUCLEAR_UTIL_THREADPOOL_HPP diff --git a/src/util/main_thread_id.cpp b/src/util/main_thread_id.cpp index ffbecd9f7..32851f506 100644 --- a/src/util/main_thread_id.cpp +++ b/src/util/main_thread_id.cpp @@ -21,6 +21,12 @@ namespace NUClear { namespace util { + /** + * @brief The thread id of the main execution thread for this process + + * @detail In order to get the main threads id, we set it as a global static variable. + * This should result in the static setup code executing on startup (in the main thread). + */ const std::thread::id main_thread_id = std::this_thread::get_id(); } // namespace util diff --git a/src/util/main_thread_id.hpp b/src/util/main_thread_id.hpp index ba576dc72..1adee4745 100644 --- a/src/util/main_thread_id.hpp +++ b/src/util/main_thread_id.hpp @@ -24,12 +24,6 @@ namespace NUClear { namespace util { - /** - * @brief The thread id of the main execution thread for this process - - * @details In order to get the main threads id, we set it as a global static variable. - * This should result in the static setup code executing on startup (in the main thread). - */ extern const std::thread::id main_thread_id; } // namespace util diff --git a/tests/dsl/Always.cpp b/tests/dsl/Always.cpp index 293195ae1..d48e52e59 100644 --- a/tests/dsl/Always.cpp +++ b/tests/dsl/Always.cpp @@ -20,31 +20,18 @@ #include namespace { -struct BlankMessage {}; -int i = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool emitted_message = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +int i = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { on().then([this] { - // Run until it's 11 then emit the blank message - if (i > 10) { - if (!emitted_message) { - emitted_message = true; - emit(std::make_unique()); - } - } - else { - ++i; - } - }); + ++i; - on>().then([this] { - if (i == 11) { - ++i; + // Run until it's 11 then shutdown + if (i > 10) { powerplant.shutdown(); } }); @@ -52,7 +39,7 @@ class TestReactor : public NUClear::Reactor { }; } // namespace -TEST_CASE("Testing on functionality (permanent run)", "[api][always]") { +TEST_CASE("Testing on functionality (permanant run)", "[api][always]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; @@ -65,6 +52,5 @@ TEST_CASE("Testing on functionality (permanent run)", "[api][always]") { plant.start(); - REQUIRE(emitted_message); - REQUIRE(i == 12); + REQUIRE(i == 11); } diff --git a/tests/dsl/BlockNoData.cpp b/tests/dsl/BlockNoData.cpp index 59b86401b..b88318c10 100644 --- a/tests/dsl/BlockNoData.cpp +++ b/tests/dsl/BlockNoData.cpp @@ -62,6 +62,4 @@ TEST_CASE("Testing that when a trigger does not have it's data satisfied it does plant.emit(message); plant.start(); - - REQUIRE(a != nullptr); } diff --git a/tests/dsl/FlagMessage.cpp b/tests/dsl/FlagMessage.cpp index db333ce87..4ba9686bd 100644 --- a/tests/dsl/FlagMessage.cpp +++ b/tests/dsl/FlagMessage.cpp @@ -62,7 +62,9 @@ class TestReactor : public NUClear::Reactor { // We make this high priority to ensure it runs first (will check for more errors) on, With, Priority::HIGH>().then([](const MessageA&, const MessageB&) { - FAIL("A was never emitted after B so this should not be possible"); + // Check A and B have been emitted + REQUIRE(a != nullptr); + REQUIRE(b != nullptr); }); } }; diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 165b16c8f..a5153df7a 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -30,7 +30,7 @@ namespace { class TestReactor : public NUClear::Reactor { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)), in(0), out(0) { std::array fds{-1, -1}; diff --git a/tests/dsl/Last.cpp b/tests/dsl/Last.cpp index 927bfaf9c..1ad91c7b7 100644 --- a/tests/dsl/Last.cpp +++ b/tests/dsl/Last.cpp @@ -54,7 +54,7 @@ class TestReactor : public NUClear::Reactor { REQUIRE(int(messages.size()) == messages.back()->value); } - // Check that our numbers are increasing + // Check that our numbers are decreasing int i = messages.front()->value; for (auto& m : messages) { REQUIRE(m->value == i); diff --git a/tests/dsl/MainThread.cpp b/tests/dsl/MainThread.cpp index b7f1d1ec6..8fa2c5921 100644 --- a/tests/dsl/MainThread.cpp +++ b/tests/dsl/MainThread.cpp @@ -26,20 +26,19 @@ class TestReactor : public NUClear::Reactor { TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { // Run a task without MainThread to make sure it isn't on the main thread - on>().then("Non-MainThread reaction", [this] { + on>().then([this] { // We shouldn't be on the main thread REQUIRE(NUClear::util::main_thread_id != std::this_thread::get_id()); emit(std::make_unique(1.1)); }); - // Run a task with MainThread and ensure that it is on the main thread - on, MainThread>().then("MainThread reaction", [this] { - // Shutdown first so the test will end even if the next check fails - powerplant.shutdown(); - + // Run a task with MainTHread and ensure that it is on the main thread + on, MainThread>().then([this] { // We should be on the main thread REQUIRE(NUClear::util::main_thread_id == std::this_thread::get_id()); + + powerplant.shutdown(); }); on().then([this]() { diff --git a/tests/dsl/Optional.cpp b/tests/dsl/Optional.cpp index c5ce9fa85..535792268 100644 --- a/tests/dsl/Optional.cpp +++ b/tests/dsl/Optional.cpp @@ -27,6 +27,9 @@ int trigger3 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) int trigger4 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) struct MessageA {}; + +template +auto resolve_function_type(R (*)(A...)) -> R (*)(A...); struct MessageB {}; class TestReactor : public NUClear::Reactor { diff --git a/tests/dsl/Single.cpp b/tests/dsl/Single.cpp index 17522e1e0..d7262919f 100644 --- a/tests/dsl/Single.cpp +++ b/tests/dsl/Single.cpp @@ -47,7 +47,7 @@ class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - on, Single>().then("SimpleMessage1", [this](const SimpleMessage1&) { + on, Single>().then([this](const SimpleMessage1&) { // Increment our run count ++message_count.message1; @@ -67,14 +67,12 @@ class TestReactor : public NUClear::Reactor { powerplant.shutdown(); }); - on, Single>().then("SimpleMessage2", - [](const SimpleMessage2&) { ++message_count.message2; }); + on, Single>().then([](const SimpleMessage2&) { ++message_count.message2; }); on, With, Single>().then( - "SimpleMessage2 With SimpleMessage3", [](const SimpleMessage2&, const SimpleMessage3&) { ++message_count.message3; }); - on().then("Startup", [this]() { + on().then([this]() { // Emit two events, only one should run emit(std::make_unique()); emit(std::make_unique()); diff --git a/tests/dsl/SingleSync.cpp b/tests/dsl/SingleSync.cpp deleted file mode 100644 index 696dd37c2..000000000 --- a/tests/dsl/SingleSync.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include - -namespace { - -struct Message { - int val; - Message(int val) : val(val){}; -}; - -struct ShutdownOnIdle {}; - -std::vector values; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -class TestReactor : public NUClear::Reactor { -public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - - on, Sync>().then("SyncReaction", [](const Message& m) { - values.push_back("Received value " + std::to_string(m.val)); - }); - - on, Priority::IDLE>().then("ShutdownOnIdle", [this] { powerplant.shutdown(); }); - - on().then("Startup", [this] { - values.clear(); - for (int i = 0; i < 1000; ++i) { - emit(std::make_unique(i)); - } - emit(std::make_unique()); - }); - } -}; -} // namespace - -TEST_CASE("Testing that the Sync priority queue word works correctly", "[api][sync][priority]") { - NUClear::PowerPlant::Configuration config; - config.thread_count = 2; - NUClear::PowerPlant plant(config); - - plant.install(); - plant.start(); - - REQUIRE(values.size() == 1000); - for (int i = 0; i < 1000; ++i) { - CHECK(values[i] == "Received value " + std::to_string(i)); - } -} diff --git a/tests/dsl/With.cpp b/tests/dsl/With.cpp index d51b4077f..d7be3dafb 100644 --- a/tests/dsl/With.cpp +++ b/tests/dsl/With.cpp @@ -58,6 +58,5 @@ TEST_CASE("Testing poorly ordered on arguments", "[api][with]") { plant.emit(std::make_unique()); plant.emit(std::make_unique()); - REQUIRE_NOTHROW(plant.start()); - REQUIRE_FALSE(plant.running()); + plant.start(); } From 61eca93075cebba3bd1467cdb83f0de6434f33fd Mon Sep 17 00:00:00 2001 From: Alex Biddulph Date: Fri, 23 Jun 2023 15:12:26 +1000 Subject: [PATCH 87/87] Finish finish rebase --- .github/workflows/main.yaml | 1 - src/threading/ThreadPoolTask.hpp | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 src/threading/ThreadPoolTask.hpp diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3672ec5b0..c7227e7d1 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -28,7 +28,6 @@ jobs: # The type of runner that the job will run on runs-on: ubuntu-latest - continue-on-error: true # Use the container for this specific version of gcc container: ${{ matrix.container }} diff --git a/src/threading/ThreadPoolTask.hpp b/src/threading/ThreadPoolTask.hpp new file mode 100644 index 000000000..773d266b9 --- /dev/null +++ b/src/threading/ThreadPoolTask.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_THREADING_THREADPOOLTASK_HPP +#define NUCLEAR_THREADING_THREADPOOLTASK_HPP + +#include "../PowerPlant.hpp" +#include "../util/update_current_thread_priority.hpp" +#include "TaskScheduler.hpp" + +namespace NUClear { +namespace threading { + + inline std::function make_thread_pool_task(TaskScheduler& scheduler) { + return [&scheduler] { + // Wait at a high (but not realtime) priority to reduce latency + // for picking up a new task + update_current_thread_priority(1000); + + // Run while our scheduler gives us tasks + for (std::unique_ptr task(scheduler.get_task()); task; task = scheduler.get_task()) { + + // Run the task + task = task->run(std::move(task)); + + // Back up to realtime while waiting + update_current_thread_priority(1000); + } + }; + } + +} // namespace threading +} // namespace NUClear + +#endif // NUCLEAR_THREADING_THREADPOOLTASK_HPP