From ee999ccbdf8c68175195b604ef5988ddda7b4956 Mon Sep 17 00:00:00 2001 From: Pauli Date: Sat, 9 Jun 2018 12:02:42 +0300 Subject: [PATCH 1/5] Implement runtime debug print filtering The runtime debug print filtering support dynamic debug print selection. Tis patch only implements basic core support for filtering. The commands to change the runtime filtering settings will be added in a following patch. But even with only this one can change filtering settings by editing memory using a debugger. It can even be automated by using gdb break point commands. --- library/CMakeLists.txt | 4 + library/Debug.cpp | 186 ++++++++ library/include/Debug.h | 369 ++++++++++++++++ library/include/DebugManager.h | 111 +++++ library/include/Signal.hpp | 782 +++++++++++++++++++++++++++++++++ 5 files changed, 1452 insertions(+) create mode 100644 library/Debug.cpp create mode 100644 library/include/Debug.h create mode 100644 library/include/DebugManager.h create mode 100644 library/include/Signal.hpp diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 390efdcfe..416e42c41 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -28,6 +28,8 @@ include/Core.h include/ColorText.h include/DataDefs.h include/DataIdentity.h +include/Debug.h +include/DebugManager.h include/VTableInterpose.h include/LuaWrapper.h include/LuaTools.h @@ -38,6 +40,7 @@ include/MiscUtils.h include/Module.h include/Pragma.h include/MemAccess.h +include/Signal.hpp include/TileTypes.h include/Types.h include/VersionInfo.h @@ -55,6 +58,7 @@ SET(MAIN_SOURCES Core.cpp ColorText.cpp DataDefs.cpp +Debug.cpp Error.cpp VTableInterpose.cpp LuaWrapper.cpp diff --git a/library/Debug.cpp b/library/Debug.cpp new file mode 100644 index 000000000..aa0cf8cdf --- /dev/null +++ b/library/Debug.cpp @@ -0,0 +1,186 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ +#define _POSIX_C_SOURCE 200809L +#include "Core.h" + +#include "Debug.h" +#include "DebugManager.h" + +#include +#include +#include + +#ifdef _MSC_VER +static tm* localtime_r(const time_t* time, tm* result) +{ + localtime_s(result, time); + return result; +} +#endif + +namespace DFHack { +DBG_DECLARE(core,debug); + +void DebugManager::registerCategory(DebugCategory& cat) +{ + DEBUG(debug) << "register DebugCategory '" << cat.category() + << "' from '" << cat.plugin() + << "' allowed " << cat.allowed() << std::endl; + std::lock_guard guard(access_mutex_); + push_back(&cat); + categorySignal(CAT_ADD, cat); +} + +void DebugManager::unregisterCategory(DebugCategory& cat) +{ + DEBUG(debug) << "unregister DebugCategory '" << cat.category() + << "' from '" << cat.plugin() + << "' allowed " << cat.allowed() << std::endl; + std::lock_guard guard(access_mutex_); + auto iter = std::find(begin(), end(), &cat); + std::swap(*iter, back()); + pop_back(); + categorySignal(CAT_REMOVE, cat); +} + +DebugRegisterBase::DebugRegisterBase(DebugCategory* cat) +{ + // Make sure Core lives at least as long any DebugCategory to + // allow debug prints until all Debugcategories has been destructed + Core::getInstance(); + DebugManager::getInstance().registerCategory(*cat); +} + +void DebugRegisterBase::unregister(DebugCategory* cat) +{ + DebugManager::getInstance().unregisterCategory(*cat); +} + +static color_value selectColor(const DebugCategory::level msgLevel) +{ + switch(msgLevel) { + case DebugCategory::LTRACE: + return COLOR_GREY; + case DebugCategory::LDEBUG: + return COLOR_LIGHTBLUE; + case DebugCategory::LINFO: + return COLOR_CYAN; + case DebugCategory::LWARNING: + return COLOR_YELLOW; + case DebugCategory::LERROR: + return COLOR_LIGHTRED; + } + return COLOR_WHITE; +} + +#if __GNUC__ +// Allow gcc to optimize tls access. It also makes sure initialized is done as +// early as possible. The early initialization helps to give threads same ids as +// gdb shows. +#define EXEC_ATTR __attribute__((tls_model("initial-exec"))) +#else +#define EXEC_ATTR +#endif + +namespace { +static std::atomic nextId{0}; +static EXEC_ATTR thread_local uint32_t thread_id{nextId.fetch_add(1)+1}; +} + +DebugCategory::ostream_proxy_prefix::ostream_proxy_prefix( + const DebugCategory& cat, + color_ostream& target, + const DebugCategory::level msgLevel) : + color_ostream_proxy(target) +{ + color(selectColor(msgLevel)); + auto now = std::chrono::system_clock::now(); + tm local{}; + //! \todo c++ 2020 will have std::chrono::to_stream(fmt, system_clock::now()) + //! but none implements it yet. + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + auto ms = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + // Output time in format %02H:%02M:%02S.%03ms +#if __GNUC__ < 5 + // Fallback for gcc 4 + char buffer[32]; + size_t sz = strftime(buffer, sizeof(buffer)/sizeof(buffer[0]), + "%T.", localtime_r(&now_c, &local)); + *this << (sz > 0 ? buffer : "HH:MM:SS.") +#else + *this << std::put_time(localtime_r(&now_c, &local),"%T.") +#endif + << std::setfill('0') << std::setw(3) << ms.count() + // Thread id is allocated in the thread creation order to a thread_local + // variable + << ":t" << thread_id + // Output plugin and category names to make it easier to locate where + // the message is coming. It would be easy replaces these with __FILE__ + // and __LINE__ passed from the macro if that would be preferred prefix. + << ':' << cat.plugin() << ':' << cat.category() << ": "; +} + + +DebugCategory::level DebugCategory::allowed() const noexcept +{ + return allowed_.load(std::memory_order_relaxed); +} + +void DebugCategory::allowed(DebugCategory::level value) noexcept +{ + level old = allowed_.exchange(value, std::memory_order_relaxed); + if (old == value) + return; + TRACE(debug) << "modify DebugCategory '" << category() + << "' from '" << plugin() + << "' allowed " << value << std::endl; + auto& manager = DebugManager::getInstance(); + manager.categorySignal(DebugManager::CAT_MODIFIED, *this); +} + +DebugCategory::cstring_ref DebugCategory::category() const noexcept +{ + return category_; +} + +DebugCategory::cstring_ref DebugCategory::plugin() const noexcept +{ + return plugin_; +} + +#if __cplusplus < 201703L && __cpp_lib_atomic_is_always_lock_free < 201603 +//! C++17 has std::atomic::is_always_lock_free for static_assert. Older +//! standards only provide runtime checks if an atomic type is lock free +struct failIfEnumAtomicIsNotLockFree { + failIfEnumAtomicIsNotLockFree() { + std::atomic test; + if (test.is_lock_free()) + return; + std::cerr << __FILE__ << ':' << __LINE__ + << ": error: std::atomic should be lock free. Your compiler reports the atomic requires runtime locks. Either you are using a very old CPU or we need to change code to use integer atomic type." << std::endl; + std::abort(); + } +} failIfEnumAtomicIsNotLockFree; +#endif + +} diff --git a/library/include/Debug.h b/library/include/Debug.h new file mode 100644 index 000000000..976172c8d --- /dev/null +++ b/library/include/Debug.h @@ -0,0 +1,369 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ + +#pragma once + +#include "ColorText.h" + +#include +#include "Core.h" + +namespace DFHack { + +/*! \file Debug.h + * Light weight wrappers to runtime debug output filtering. Idea is to add as + * little as possible code compared to debug output without filtering. The + * effect is archived using #TRACE, #DEBUG, #INFO, #WARN and #ERR macros. They + * "return" color_ostream object or reference that can be then used normally for + * either printf or stream style debug output. + * + * Internally macros do inline filtering check which allows compiler to have a + * fast path without debug output only checking unlikely condition. But if + * output is enabled then runtime code will jump to debug printing function + * calls. The macro setup code will also print standardized leading part of + * debug string including time stamp, plugin name and debug category name. + * + * \code{.cpp} + * #include "Debug.h" + * DBG_DECLARE(myplugin,init); + * + * DFhackCExport command_result plugin_init ( color_ostream &out, std::vector &commands) + * { + * command_result rv = CR_OK; + * DEBUG(init, out).print("initializing\n") + * if ((rv = initWork()) != CR_OK) { + * ERR(init, out) << "initWork failed with " + * << rv << " error code" << std::endl; + * return rv; + * } + * return rv + * } + * \endcode + * + * The debug print filtering levels can be changed using debugger. Following + * gdb example would automatically setup core/init and core/render to trace + * level when SDL_init is called. + * + * \code{.unparsed} + * break SDL_init + * commands + * silent + * p DFHack::debug::core::debug_init.allowed_ = 0 + * p DFHack::debug::core::debug_render.allowed_ = 0 + * c + * end + * \endcode + * + */ + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + +/*! + * \defgroup debug_branch_prediction Branch prediction helper macros + * Helper macro tells compiler that debug output branch is unlikely and should + * be optimized to cold section of the function. + * \{ + */ +#if __cplusplus >= 202000L || __has_cpp_attribute(likely) +// c++20 will have standard branch prediction hint attributes +#define likely(x) (x) [[likely]] +#define unlikely(x) (x) [[unlikely]] +#elif defined(__GNUC__) +// gcc has builtin functions that give hints to the branch prediction +#define likely(x) (__builtin_expect(!!(x), 1)) +#define unlikely(x) (__builtin_expect(!!(x), 0)) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif +//! \} + +#ifdef NDEBUG +//! Reduce minimum compiled in debug levels if NDEBUG is defined +#define DBG_FILTER DFHack::DebugCategory::LINFO +#else +//! Set default compiled in debug levels to include all prints +#define DBG_FILTER DFHack::DebugCategory::LTRACE +#endif + +/*! + * DebugCategory is used to enable and disable debug messages in runtime. + * Declaration and definition are handled by #DBG_DECLARE and #DBG_DEFINE + * macros. Runtime filtering support is handled by #TRACE, #DEBUG, #INFO, #WARN + * and #ERR macros. + */ +class DFHACK_EXPORT DebugCategory final { +public: + //! type helper to maybe make it easier to convert to std::string_view when + //! c++17 can be required. + using cstring = const char*; + using cstring_ref = const char*; + /*! + * Debug level enum for message filtering + */ + enum level { + LTRACE = 0, + LDEBUG = 1, + LINFO = 2, + LWARNING = 3, + LERROR = 4, + }; + + /*! + * \param plugin the name of plugin the category belongs to + * \param category the name of category + * \param defaultLevel optional default filtering level for the category + */ + constexpr DebugCategory(cstring_ref plugin, + cstring_ref category, + level defaultLevel = LWARNING) noexcept : + plugin_{plugin}, + category_{category}, + allowed_{defaultLevel} + {} + + + DebugCategory(const DebugCategory&) = delete; + DebugCategory(DebugCategory&&) = delete; + DebugCategory& operator=(DebugCategory) = delete; + DebugCategory& operator=(DebugCategory&&) = delete; + + /*! + * Used by debug macros to check if message should be printed. + * + * It is defined in the header to allow compiler inline it and make disabled + * state a fast path without function calls. + * + * \param msgLevel the debug message level the following print belongs to + * \return boolean with true indicating that message should be printed + */ + bool isEnabled(const level msgLevel) const noexcept { + const uint32_t intLevel = static_cast(msgLevel); + // Compile time filtering to allow compiling out debug checks prints + // from binary. + return static_cast(DBG_FILTER) <= intLevel && + // Runtime filtering for debug messages + static_cast(allowed_.load(std::memory_order_relaxed)) <= intLevel; + } + + struct DFHACK_EXPORT ostream_proxy_prefix : public color_ostream_proxy { + ostream_proxy_prefix(const DebugCategory& cat, + color_ostream& target, + DebugCategory::level level); + ~ostream_proxy_prefix() + { + flush(); + } + }; + + /*! + * Fetch a steam object proxy object for output. It also adds standard + * message components like time and plugin and category names to the line. + * + * User must make sure that the line is terminated with a line end. + * + * \param msgLevel Specifies the level which next debug message belongs + * \return color_ostream_proxy that can be used to print the message + * \sa DFHack::Core::getConsole() + */ + ostream_proxy_prefix getStream(const level msgLevel) const + { + return {*this,Core::getInstance().getConsole(),msgLevel}; + } + /*! + * Add standard message components to existing output stream object to begin + * a new message line to an shared buffered object. + * + * \param msgLevel Specifies the level which next debug message belongs + * \param target An output stream object where a debug output is printed + * \return color_ostream reference that was passed as second parameter + */ + ostream_proxy_prefix getStream(const level msgLevel, color_ostream& target) const + { + return {*this,target,msgLevel}; + } + + /*! + * \brief Allow management code to set a new filtering level + * Caller must have locked DebugManager::access_mutex_. + */ + void allowed(level value) noexcept; + //! Query current filtering level + level allowed() const noexcept; + //! Query plugin name + cstring_ref plugin() const noexcept; + //! Query category name + cstring_ref category() const noexcept; +private: + + cstring plugin_; + cstring category_; + std::atomic allowed_; +#if __cplusplus >= 201703L || __cpp_lib_atomic_is_always_lock_free >= 201603 + static_assert(std::atomic::is_always_lock_free, + "std::atomic should be lock free. You are using a very old CPU or code needs to use std::atomic"); +#endif +}; + +/** + * Handle actual registering wrong template parameter generated pointer + * calculation. + */ +class DFHACK_EXPORT DebugRegisterBase { +protected: + DebugRegisterBase(DebugCategory* category); + void unregister(DebugCategory* category); +}; + +/** + * Register DebugCategory to DebugManager + */ +template +class DebugRegister final : public DebugRegisterBase { +public: + DebugRegister() : + DebugRegisterBase{category} + {} + ~DebugRegister() { + unregister(category); + } +}; + +#define DBG_NAME(category) debug_ ## category + + +/*! + * Declares a debug category. There must be only a declaration per category. + * Declaration should be in same plugin where it is used. If same category name + * is used in core and multiple plugins they all are changed with same command + * unless user specifies explicitly plugin name. + * + * Must be used in one translation unit only. + * + * \param plugin the name of plugin where debug category is used + * \param category the name of category + * \param level the initial DebugCategory::level filtering level. + */ +#define DBG_DECLARE(plugin,category, ...) \ + namespace debug { namespace plugin { \ + DebugCategory DBG_NAME(category){#plugin,#category,__VA_ARGS__}; \ + DebugRegister<&DBG_NAME(category)> register_ ## category; \ + } } \ + using debug::plugin::DBG_NAME(category) + +/*! + * Can be used to access a shared DBG_DECLARE category. But may not be used from + * static initializer because translation unit order is undefined. + * + * Can be used in shared headers to gain access to one definition from + * DBG_DECLARE. + * \param plugin The plugin name that must match DBG_DECLARE + * \param category The category name that must matnch DBG_DECLARE + */ +#define DBG_EXTERN(plugin,category) \ + namespace debug { namespace plugin { \ + extern DFHack::DebugCategory DBG_NAME(category); \ + } } \ + using debug::plugin::DBG_NAME(category) + +#define DBG_PRINT(category,pred,level,...) \ + if pred(!DFHack::DBG_NAME(category).isEnabled(level)) \ + ; /* nop fast path when debug category is disabled */ \ + else /* else to allow macro use in if-else branches */ \ + DFHack::DBG_NAME(category).getStream(level, ## __VA_ARGS__) \ +/* end of DBG_PRINT */ + +/*! + * Open a line for trace level debug output if enabled + * + * Preferred category for inside loop debug messages or callbacks/methods called + * multiple times per second. Good example would be render or onUpdate methods. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define TRACE(category, ...) DBG_PRINT(category, likely, \ + DFHack::DebugCategory::LTRACE, ## __VA_ARGS__) + +/*! + * Open a line for debug level debug output if enabled + * + * Preferred place to use it would be commonly called functions that don't fall + * into trace category. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define DEBUG(category, ...) DBG_PRINT(category, likely, \ + DFHack::DebugCategory::LDEBUG, ## __VA_ARGS__) + +/*! + * Open a line for error level debug output if enabled + * + * Important debug messages when some rarely changed state changes. Example + * would be when a debug category filtering level changes. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define INFO(category, ...) DBG_PRINT(category, likely, \ + DFHack::DebugCategory::LINFO, ## __VA_ARGS__) + +/*! + * Open a line for warning level debug output if enabled + * + * Warning category is for recoverable errors. This generally signals that + * something unusual happened but there is code handling the error which should + * allow df continue running without issues. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define WARN(category, ...) DBG_PRINT(category, unlikely, \ + DFHack::DebugCategory::LWARNING, ## __VA_ARGS__) + +/*! + * Open a line for error level error output if enabled + * + * Errors should be printed only for cases where plugin or dfhack can't recover + * from reported error and it requires manual handling from the user. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define ERR(category, ...) DBG_PRINT(category, unlikely, \ + DFHack::DebugCategory::LERROR, ## __VA_ARGS__) + +} diff --git a/library/include/DebugManager.h b/library/include/DebugManager.h new file mode 100644 index 000000000..fb4c635fb --- /dev/null +++ b/library/include/DebugManager.h @@ -0,0 +1,111 @@ +/** + Copyright © 2018 Pauli + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any + damages arising from the use of this software. + + Permission is granted to anyone to use this software for any + purpose, including commercial applications, and to alter it and + redistribute it freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. + */ + +#pragma once + +#include "Export.h" +#include "Signal.hpp" + +#include +#include + +namespace DFHack { + +/*! \file DebugManager.h + * Expose an simple interface to runtime debug output filtering. The management + * interface is separated from output interface because output is required in + * many places while management is expected to be required only in a few places. + */ + +class DebugCategory; + +/*! + * \brief Container holding all registered runtime debug categories + * Singleton DebugManager is a minor extension to std::vector that allows signal + * callbacks to be attached from ui code that manages. + * + * To avoid parallel plugin unload causing issues access to DebugManager must be + * protected by mutex. The access mutex will be taken when + * DFHack::DebugCategory::~DebugCategory performs unregister calls to + * DFHack::DebugManager. The mutex will protect from memory disappearing while + * ui code is accessing or changing the runtime state. + * + * Signal emitting happens from a locked contexts. Taking the + * DFHack::DebugManager::access_mutex_ in a signal callback will results to a + * deadlock. + * + * The interface is extremely simple but enough to implement persistent filter + * states and runtime configuration code in a plugin. + */ +class DFHACK_EXPORT DebugManager : public std::vector { +public: + friend class DebugRegisterBase; + + //! access_mutex_ protects all readers and writers to DFHack::DebugManager + std::mutex access_mutex_; + + //! Different signals that all will be routed to + //! DebugManager::categorySignal + enum signalType { + CAT_ADD, + CAT_REMOVE, + CAT_MODIFIED, + }; + + //! type to help access signal features like Connection and BlockGuard + using categorySignal_t = Signal; + + /*! + * Signal object where callbacks can be connected. Connecting to a class + * method can use a lambda wrapper to the capture object pointer and correctly + * call required method. + * + * Signal is internally serialized allowing multiple threads call it + * freely. + */ + categorySignal_t categorySignal; + + //! Get the singleton object + static DebugManager& getInstance() { + static DebugManager instance; + return instance; + } + + //! Prevent copies + DebugManager(const DebugManager&) = delete; + //! Prevent copies + DebugManager(DebugManager&&) = delete; + //! Prevent copies + DebugManager& operator=(DebugManager) = delete; + //! Prevent copies + DebugManager& operator=(DebugManager&&) = delete; +protected: + DebugManager() = default; + + //! Helper for automatic category registering and signaling + void registerCategory(DebugCategory &); + //! Helper for automatic category unregistering and signaling + void unregisterCategory(DebugCategory &); +private: +}; +} diff --git a/library/include/Signal.hpp b/library/include/Signal.hpp new file mode 100644 index 000000000..acc8f4a0a --- /dev/null +++ b/library/include/Signal.hpp @@ -0,0 +1,782 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#ifdef __SSE__ +#include +#endif + +namespace DFHack { + +/*! + * Select inline implementation for Signal members + * This requires careful destruction order where all connection has been + * disconnected before Signal::~Signal() + */ +class signal_inline_tag; +/*! + * Select share_ptr managed implementation for Signal members. + * + * If Connection holding object may be deleted without full serialization + * between disconnect and signal emit the holding object must be managed by + * shared_ptr and derive from ConnectedBase. It will also have to pass the + * std::shared_ptr to connect. + + * It uses two way std::weak_ptr reference to guarantee destruction of either + * object doesn't happen when call is made to them. + * + * It is still possible to get a callback call after manual disconnect from + * outside destructor. But without destruction risk the disconnect race can be + * handled by slot implementation side. + */ +class signal_shared_tag; + +/** + * Used for signal_shared_tag holders that may race with destructor triggered + * disconnect and emit from Signal. + */ +class ConnectedBase { +}; + +template +class Signal; + +namespace details { + +template +struct SignalImpl; + +template +struct selectImpl; + +//! Manage callback states in thread safe manner +template +class CallbackHolderImpl; + +template +struct CallbackHolderBase { + using Callback = std::function; + + CallbackHolderBase(const Callback& cb) : + cb_{cb}, + state_{} + {} + + //! Block the connection + void block() noexcept + { + state_ += blocked; + } + + //! Unblock the connection + void unblock() noexcept + { + state_ -= blocked; + } + + //! Check if connection is deleted + bool erased() const noexcept + { + return state_ & deleted; + } + + //! Check if connection is still active (not blocked or erased) + operator bool() const noexcept + { + return !(state_ & ~inCall); + } + +protected: + //! Immutable callback object + const Callback cb_; + using state_t = unsigned; + //! Single shared state as a bitfield to simplify synchronization + //! between state changes. + std::atomic state_; + static constexpr state_t deleted = 0x1 << (sizeof(state_t)*CHAR_BIT - 1); + static constexpr state_t inCall = deleted >> (sizeof(state_t)*CHAR_BIT/2); + static constexpr state_t blocked = 0x1; + static constexpr state_t blockedMask = inCall - 1; + static constexpr state_t inCallMask = (deleted - 1) ^ blockedMask; +}; + +template +class CallbackHolderImpl : + public CallbackHolderBase { + using parent_t = CallbackHolderBase; +public: + using Callback = typename parent_t::Callback; +private: + using state_t = typename parent_t::state_t; + //! Make sure callback pointed object doesn't disappear under us + //! while we call it. + struct CallGuard { + + //! Prevent copies but allow copy elision + CallGuard(const CallGuard&); + + //! Allow implicit conversion to callback for simply syntax + const Callback& operator*() const noexcept + { + return holder_->cb_; + } + + operator bool() const noexcept + { + return *holder_; + } + + //! Mark call not to be called any more + ~CallGuard() { + holder_->state_ -= parent_t::inCall; + } + private: + //! Reference to the connection + CallbackHolderImpl* holder_; + + //! Mark call to be in process + CallGuard(CallbackHolderImpl* holder) : + holder_{holder} + { + holder_->state_ += parent_t::inCall; + } + //! Only allow construction from the CallbackHolderImpl::prepareCall + friend class CallbackHolderImpl; + }; +public: + //! Construct the callback state for a callback + CallbackHolderImpl(const Callback& cb) : + parent_t{cb} + {} + + /*! + * Data race free disconnection for the connection. It spins until + * no more callers to wait. Spinning should be problem as callbacks + * are expected to be simple and fast to execute. + * + * Must not be called from withing callback! + * + * \todo Maybe use monitor instruction to avoid busy wait and call + * std::thread::yield() if wait is longer than expected. + */ + void erase() noexcept + { + state_t oldstate; + state_t newstate; + /** Spin until no callers to this callback */ +spin: + while ((oldstate = parent_t::state_) & parent_t::inCallMask) { + // pause would be portable to all old processors but there + // isn't portable way to generate it without SSE header. +#ifdef __SSE__ + _mm_pause(); +#endif + } + do { + if (oldstate & parent_t::inCallMask) + goto spin; + newstate = oldstate | parent_t::deleted; + } while(!parent_t::state_.compare_exchange_weak(oldstate, newstate)); + } + + //! Return RAII CallGuard to protect race between callback and + //! disconnect. + CallGuard prepareCall() + { + return {this}; + } +}; + +template +class CallbackHolderImpl : + public CallbackHolderBase { + using parent_t = CallbackHolderBase; +public: + using Callback = typename parent_t::Callback; +private: + using state_t = typename parent_t::state_t; + //! Make sure callback pointed object doesn't disappear under us + //! while we call it. + struct CallGuard { + + //! Prevent copies but allow copy elision + CallGuard(const CallGuard&); + + //! Allow implicit conversion to callback for simply syntax + const Callback& operator*() const noexcept + { + return holder_->cb_; + } + + operator bool() const noexcept + { + // If this is not marked erased then weak_ref->lock succeeded or + // the slot isn't managed by shared_ptr + return *holder_; + } + + private: + //! Reference to the connection + CallbackHolderImpl* holder_; + std::shared_ptr strong_ref_; + + //! Mark call to be in process + CallGuard(CallbackHolderImpl* holder) : + holder_{holder}, + strong_ref_{holder->weak_ref_.lock()} + { + } + //! Only allow construction from the CallbackHolderImpl::prepareCall + friend class CallbackHolderImpl; + }; + + std::weak_ptr weak_ref_; + friend CallGuard; +public: + //! Construct the callback state for an automatically synchronized object + CallbackHolderImpl(const Callback& cb, + std::shared_ptr& ref) : + parent_t{cb}, + weak_ref_{ref} + {} + + //! Construct the callback state for an externally synchronized object + CallbackHolderImpl(const Callback& cb) : + parent_t{cb}, + weak_ref_{} + {} + + /*! + * erase from destructor can't happen while we are in call because + */ + void erase() noexcept + { + parent_t::state_ |= parent_t::deleted; + } + + //! Return RAII CallGuard to protect race between callback and + //! disconnect. + CallGuard prepareCall() + { + return {this}; + } +}; + +template +struct SignalImpl : public selectImpl::parent_t { +protected: + using select_t = selectImpl; + using parent_t = typename select_t::parent_t; +public: + using CallbackHolder = CallbackHolderImpl; + using Callback = typename CallbackHolder::Callback; + + //! The container type used to store callbacks + using CallbackContainer = std::list; + struct BlockGuard; + + //! Simple connection class that is required to disconnect from the + //! signal. + struct Connection { + //! Construct a default Connection object but using it will result + //! to undefined behavior unless proper connection is assigned to it + Connection() = default; + + Connection(Connection&& o) : + iter_{o.iter_}, + signal_{} + { + std::swap(signal_, o.signal_); + } + + Connection& operator=(Connection&& o) + { + disconnect(); + iter_ = o.iter_; + std::swap(signal_, o.signal_); + return *this; + } + + Connection(const Connection&) = delete; + Connection& operator=(const Connection&) = delete; + + //! Disconnect from signal + void disconnect() + { + auto s = select_t::lock(signal_); + if (!s) + return; + + s->disconnect(*this); + } + + ~Connection() + { + disconnect(); + } + private: + //! Block the connection temporary + void block() + { + auto s = select_t::lock(signal_); + if (!s) + return; + iter_->block(); + } + + //! Restore blocked connection + void unblock() + { + auto s = select_t::lock(signal_); + if (!s) + return; + iter_->unblock(); + } + + //! Construct connection object + Connection(const typename CallbackContainer::iterator &iter, + typename select_t::weak_ptr ptr) : + iter_{iter}, + signal_{ptr} + {} + + //! std::list iterator that is used to access the callback and allow + //! removal from the list. + typename CallbackContainer::iterator iter_; + //! Reference to signal object + typename select_t::weak_ptr signal_; + friend SignalImpl; + friend BlockGuard; + }; + + /*! + * BlockGuard allows temporary RAII guard managed blocking of a + * connection object. + */ + struct BlockGuard { + /*! + * Block a connection that belongs to signal + * \param connection The connection that will be temporary blocked + */ + BlockGuard(Connection& connection) : + blocked_{&connection} + { + connection.block(); + } + + /*! + * Unblock the temporary blocked connection + */ + ~BlockGuard() + { + blocked_->unblock(); + } + + //! Prevent copies but allow copy elision + BlockGuard(const BlockGuard&); + private: + Connection* blocked_; + }; + + Connection connect(const Callback& f) + { + std::lock_guard lock(access_); + auto iter = callbacks_.emplace(callbacks_.begin(), f); + return {iter, parent_t::shared_from_this()}; + } + + Connection connect(std::shared_ptr c, const Callback& f) + { + std::lock_guard lock(access_); + auto iter = callbacks_.emplace(callbacks_.begin(), f, c); + return {iter, parent_t::shared_from_this()}; + } + + void disconnect(Connection& connection) { + std::lock_guard lock(access_); + if (recursion_) { + deleted_ = true; + connection.iter_->erase(); + } else { + callbacks_.erase(connection.iter_); + } + select_t::reset(connection.signal_); + } + + template + void operator()(Combiner &combiner, Args&&... arg) + { + std::unique_lock lock(access_); + struct RecursionGuard { + SignalImpl* signal_; + std::unique_lock* lock_; + //! Increment access count to make sure disconnect doesn't erase + RecursionGuard(SignalImpl *signal, std::unique_lock* lock) : + signal_{signal}, + lock_{lock} + { + ++signal_->recursion_; + } + + /*! + * Clean up deleted functions in data race free and exception + * safe manner. + */ + ~RecursionGuard() + { + lock_->lock(); + if (--signal_->recursion_ == 0 && signal_->deleted_) { + for (auto iter = signal_->callbacks_.begin(); iter != signal_->callbacks_.end();) { + if (iter->erased()) + iter = signal_->callbacks_.erase(iter); + else + ++iter; + } + signal_->deleted_ = false; + } + } + + } guard{this, &lock}; + // Call begin in locked context to allow data race free iteration + // even if there is parallel inserts to the begin after unlocking. + auto iter = callbacks_.begin(); + lock.unlock(); + for (; iter != callbacks_.end(); ++iter) { + // Quickly skip blocked calls without memory writes + if (!*iter) + continue; + // Protect connection from deletion while we are about to call + // it. + auto cb = iter->prepareCall(); + if (cb) + combiner(*cb, std::forward(arg)...); + } + } + + void operator()(Args&&... arg) + { + auto combiner = [](const Callback& cb, Args&&... arg2) + { + cb(std::forward(arg2)...); + }; + (*this)(combiner,std::forward(arg)...); + } + + ~SignalImpl() { + // Check that callbacks are empty. If this triggers then signal may + // have to be extended to allow automatic disconnection of active + // connections in the destructor. + if (std::is_same::value) + assert(callbacks_.empty() && "It is very likely that this signal should use signal_shared_tag"); + } + + //! Simplify access to pimpl when it is inline + SignalImpl* operator->() { + return this; + } + SignalImpl& operator*() { + return *this; + } + + SignalImpl() = default; +private: + SignalImpl(const SignalImpl&) : + SignalImpl{} + {} + std::mutex access_; + CallbackContainer callbacks_; + int recursion_; + bool deleted_; + friend Signal; +}; + +template +struct selectImpl { + using impl_t = SignalImpl; + using interface_t = Signal; + using type = impl_t; + using weak_ptr = impl_t*; + struct ptr_from_this { + weak_ptr shared_from_this() + { + return static_cast(this); + } + }; + using parent_t = ptr_from_this; + + selectImpl() = default; + + // Disallow copies for inline version. + selectImpl(const selectImpl&) = delete; + selectImpl(selectImpl&&) = delete; + selectImpl& operator=(const selectImpl&) = delete; + selectImpl& operator=(selectImpl&&) = delete; + + + static type make() { + return {}; + } + + static void reset(weak_ptr& ptr) { + ptr = nullptr; + } + + static weak_ptr lock(weak_ptr& ptr) { + return ptr; + } + + static weak_ptr get(interface_t& signal) { + return &signal.pimpl; + } +}; + +template +struct selectImpl { + using impl_t = SignalImpl; + using interface_t = Signal; + using type = std::shared_ptr; + using weak_ptr = std::weak_ptr; + using parent_t = std::enable_shared_from_this; + + // Allow copies for shared version + + static type make() { + return std::make_shared>(); + } + + static void reset(weak_ptr& ptr) { + ptr.reset(); + } + + static type lock(weak_ptr& ptr) { + return ptr.lock(); + } + + static weak_ptr get(interface_t& signal) { + return signal.pimpl; + } +}; +} + +/*! + * As I couldn't figure out which signal library would be a good. Too bad all + * signal libraries seem to be either heavy with unnecessary features or written + * before C++11/14 have become useable targets. That seems to indicate everyone + * is now building signal system with standard components. + * + * Implementation and interface is build around std::function holding delegates + * to a function pointer or a functor. One can put there example lambda function + * that captures this pointer from connect side. The lambda function then calls + * the slot method of object correctly. + * + * It is fairly simple to change the signal signature to directly call methods + * but internally that std::function becomes more complex. The pointer to + * member function is problematic because multiple inheritance requires + * adjustments to this. The lambda capture approach should be easy to use while + * letting compiler optimize method call in the callee side. + * + * DFHack::Signal::Connection is an connection handle. The handle can be used to + * disconnect and block a callback. Connection destructor will automatically + * disconnect from the signal. + * + * DFHack::Signal::BlockGuard is an automatic blocked callback guard object. It + * prevents any signals from calling the slot as long the BlockGuard object is + * alive. Internally it replaces the callback with an empty callback and stores + * the real callback in a member variable. Destructor then puts back the real + * callback. This allows easily recursive BlockGuard work correctly because only + * the first BlockGuard has the real callback. + * + * signal_inline_tag requires careful destruction order where all connection are + * disconnected before signal destruction. The implementation is specifically + * targeting places like static and singleton variables and widget hierarchies. + * It provides data race free connect, disconnect and emit operations. + * + * signal_shared_tag allows a bit more freedom when destroying the Signal. It + * adds data race safety between Connection, BlockGuard and destructor. If + * multiple callers need access to Signal with potential of destruction of + * original owner then callers can use Signal copy constructor to take a strong + * reference managed by shared_ptr or weak_ptr with Signal::weak_from_this(). + * weak_from_this returns an object that forwards call directly to + * implementation when the shared_ptr is created using Signal::lock + * + * \param RT return type is derived from a single signature template argument + * \param Args Variable argument type list that is derived from a signature + * template argument. + * \param tag The tag type which selects between shared_ptr managed pimpl and + * inline member variables. + */ +template +class Signal : protected details::selectImpl { +public: + //! Type of callable that can be connected to the signal. + using Callback = std::function; + +protected: + using select_t = details::selectImpl; + using CallbackContainer = typename select_t::impl_t::CallbackContainer; +public: + using weak_ptr = typename select_t::weak_ptr; + + /*! + * Simple connection class that is required to disconnect from the + * signal. + * \sa SignalImpl::Connection + */ + using Connection = typename select_t::impl_t::Connection; + /*! + * BlockGuard allows temporary RAII guard managed blocking of a + * connection object. + * \sa SignalImpl::BlockGuard + */ + using BlockGuard = typename select_t::impl_t::BlockGuard; + + /*! + * Connect a callback function to the signal + * + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + * + * \param f callable that will connected to the signal + * \return connection handle that can be used to disconnect it + */ + Connection connect(const Callback& f) + { + return pimpl->connect(f); + } + + /*! + * Thread safe connect variant connection and Connected object destruction + * can't race with emit from different threads. + * + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + */ + Connection connect(std::shared_ptr c, const Callback& f) + { + static_assert(std::is_same::value, + "Race free destruction is only possible with signal_shared_tag"); + return pimpl->connect(c, f); + } + + /*! + * Disconnection a callback from slots + * + * signal_inline_tag: + * This may not be called if the callback has been called in same + * thread. If callback should trigger destruction an object then + * deletion must use deferred. This rule prevents issues if other thread + * are trying to call the callback when disconnecting. + * + * signal_shared_tag: + * disconnect can be freely called from anywhere as long as caller holds a + * strong reference to the Signal. Strong reference can be obtained by using + * Connection::disconnect, Signal copy constructor to have a copy of signal + * or weak_ptr from weak_from_this() passed to Signal::lock(). + * + * \param connection the object returned from DFHack::Signal::connect + */ + void disconnect(Connection& connection) + { + pimpl->disconnect(connection); + } + + /*! + * Call all connected callbacks using passed arguments. + * + * signal_inline_tag: + * Must not call operator() from callbacks. + * Must not disconnect called callback from inside callback. Solution often + * is to set just atomic state variables in callback and do actual + * processing including deletion in update handler or logic vmethod. + * + * signal_shared_tag: + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + * Safe to disconnect any connection from callbacks. + * + * \param combiner that calls callbacks and processes return values + * \param arg arguments list defined by template parameter signature. + */ + template + void operator()(Combiner &combiner, Args&&... arg) + { + (*pimpl)(combiner, std::forward(arg)...); + } + + /*! + * Call all connected callbacks using passed arguments. + * + * signal_inline_tag: + * Must not call operator() from callbacks. + * Must not disconnect called callback from inside callback. Solution often + * is to set just atomic state variables in callback and do actual + * processing including deletion in update handler or logic vmethod. + * + * signal_shared_tag: + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + * Safe to disconnect any connection from callbacks. + * + * \param arg arguments list defined by template parameter signature. + */ + void operator()(Args&&... arg) + { + (*pimpl)(std::forward(arg)...); + } + + /*! + * Helper to lock the weak_ptr + */ + static typename select_t::type lock(weak_ptr& ptr) + { + return select_t::lock(ptr); + } + + /*! + * Helper to create a weak reference to pimpl which can be used to access + * pimpl directly. If the tag is signal_shared_tag then it provides race + * free access to Signal when using Signal::lock and checking returned + * shared_ptr. + */ + weak_ptr weak_from_this() noexcept + { + return select_t::get(*this); + } + + Signal() : + pimpl{select_t::make()} + {} +private: + typename select_t::type pimpl; + friend select_t; +}; +} From 8a3a05de242fdede47d832b4e453d7bec8a734fe Mon Sep 17 00:00:00 2001 From: Pauli Date: Sun, 10 Jun 2018 10:17:23 +0300 Subject: [PATCH 2/5] Allow unloading plugins that use std::regex --- library/CMakeLists.txt | 1 + library/CompilerWorkAround.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 library/CompilerWorkAround.cpp diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 416e42c41..d3bff72df 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -57,6 +57,7 @@ include/wdirent.h SET(MAIN_SOURCES Core.cpp ColorText.cpp +CompilerWorkAround.cpp DataDefs.cpp Debug.cpp Error.cpp diff --git a/library/CompilerWorkAround.cpp b/library/CompilerWorkAround.cpp new file mode 100644 index 000000000..402553bd8 --- /dev/null +++ b/library/CompilerWorkAround.cpp @@ -0,0 +1,32 @@ +#include + +namespace DFHack { +namespace neverCalled { + +/** + * gcc/linstdc++ seems to generate code that links libstdc++ back to first + * shared object using std::regex. To allow plugins unload with std::regex in + * the code we need the std::regex functions inside libdfhack.so. + * + * If your plugin decides to use any overloads that aren't listed here it may + * stay in memory after dlclose. + */ +std::regex stdRegexPluginUnloadWorkaround() +{ + std::regex fake("foo"); + std::string haystack("bar is foo in the world"); + std::regex fake2(std::string("bar")); + if (std::regex_match(haystack, fake)) + std::swap(fake, fake2); + if (std::regex_search(haystack, fake)) + std::swap(fake, fake2); + const char* haystack2 = "foo"; + if (std::regex_match(haystack2, fake)) + std::swap(fake, fake2); + if (std::regex_search(haystack2, fake)) + std::swap(fake, fake2); + return fake; +} + +} +} From 9cfb07f4760f7d99cc95109c06c44be8669bb731 Mon Sep 17 00:00:00 2001 From: Pauli Date: Sun, 10 Jun 2018 10:18:39 +0300 Subject: [PATCH 3/5] Add debug plugin to manage runtime debug filters --- plugins/CMakeLists.txt | 1 + plugins/debug.cpp | 1139 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1140 insertions(+) create mode 100644 plugins/debug.cpp diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index fbf458c4f..a05d680ea 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -107,6 +107,7 @@ if (BUILD_SUPPORTED) DFHACK_PLUGIN(cursecheck cursecheck.cpp) DFHACK_PLUGIN(cxxrandom cxxrandom.cpp LINK_LIBRARIES lua) DFHACK_PLUGIN(deramp deramp.cpp) + DFHACK_PLUGIN(debug debug.cpp LINK_LIBRARIES jsoncpp_lib_static) DFHACK_PLUGIN(dig dig.cpp) DFHACK_PLUGIN(digFlood digFlood.cpp) add_subdirectory(diggingInvaders) diff --git a/plugins/debug.cpp b/plugins/debug.cpp new file mode 100644 index 000000000..9464305dd --- /dev/null +++ b/plugins/debug.cpp @@ -0,0 +1,1139 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ + +#include "Core.h" +#include "PluginManager.h" +#include "DebugManager.h" +#include "Debug.h" +#include "modules/Filesystem.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +DFHACK_PLUGIN("debug"); + +namespace DFHack { +DBG_DECLARE(debug,filter); +DBG_DECLARE(debug,init); +DBG_DECLARE(debug,command); +DBG_DECLARE(debug,ui); +} + +namespace serialization { + +template +struct nvp : public std::pair { + using parent_t = std::pair; + nvp(const char* name, T& value) : + parent_t{name, &value} + {} +}; + +template +nvp make_nvp(const char* name, T& value) { + return {name, value}; +} + +} + +#define NVP(variable) serialization::make_nvp(#variable, variable) + +namespace Json { +template +typename std::enable_if::value, ET>::type +get(Json::Value& ar, const std::string &key, const ET& default_) +{ + return static_cast(as(ar.get(key, static_cast(default_)))); +} +} + +namespace DFHack { namespace debugPlugin { + +using JsonArchive = Json::Value; + +//! Write a named and type value to Json::Value. enable_if makes sure this is +//! only available for types that Json::Value supports directly. +template::value, int>::type = 0> +JsonArchive& operator<<(JsonArchive& ar, const serialization::nvp& target) +{ + ar[target.first] = *target.second; + return ar; +} + +//! Read a named and typed value from Json::Value +template +JsonArchive& operator>>(JsonArchive& ar, const serialization::nvp& target) +{ + *target.second = Json::get(ar, target.first, T{}); + return ar; +} + +/*! + * Default regex flags optimized for matching speed because objects are stored + * long time in memory. + */ +static constexpr auto defaultRegex = + std::regex::optimize | std::regex::nosubs | std::regex::collate; + +static const char* const commandHelp = + " Manage runtime debug print filters.\n" + "\n" + " debugfilter category [ []]\n" + " List categories matching regular expressions.\n" + " debugfilter filter []\n" + " List active filters or show detailed information for a filter.\n" + " debugfilter set [persistent] [ []]\n" + " Set a filter level to categories matching regular expressions.\n" + " debugfilter unset [ ...]\n" + " Unset filters matching space separated list of ids from 'filter'.\n" + " debugfilter disable [ ...]\n" + " Disable filters matching space separated list of ids from 'filter'.\n" + " debugfilter enable [ ...]\n" + " Enable filters matching space separated list of ids from 'filter'.\n" + " debugfilter help []\n" + " Show detailed help for a command or this help.\n"; +static const char* const commandCategory = + " category [ []]\n" + " List categories with optional filters. Parameters are passed to\n" + " std::regex to limit which once are shown. The first regular\n" + " expression is used to match category and the second is used match\n" + " plugin name.\n"; +static const char* const commandSet = + " set [persistent] [ []]\n" + " Set filtering level for matching categories. 'level' must be one of\n" + " trace, debug, info, warning and error. The 'level' parameter sets\n" + " the lowest message level that will be shown. The command doesn't\n" + " allow filters to disable any error messages.\n" + " Default filter life time is until Dwarf Fortress process exists or\n" + " plugin is unloaded. Passing 'persistent' as second parameter tells\n" + " the plugin to store the filter to dfhack-config. Stored filters\n" + " will be active until always when the plugin is loaded. 'unset'\n" + " command can be used to remove persistent filters.\n" + " Filters are applied FIFO order. The latest filter will override any\n" + " older filter that also matches.\n"; +static const char* const commandFilters = + " filter []\n" + " Show the list of active filters. The first column is 'id' which can\n" + " be used to deactivate filters using 'unset' command.\n" + " Filters are printed in same order as applied - the oldest first.\n"; +static const char* const commandUnset = + " unset [ ...]\n" + " 'unset' takes space separated list of filter ids from 'filter'.\n" + " It will reset any matching category back to the default 'warning'\n" + " level or any other still active matching filter level.\n"; +static const char* const commandDisable = + " disable [ ...]\n" + " 'disable' takes space separated list of filter ids from 'filter'.\n" + " It will reset any matching category back to the default 'warning'\n" + " level or any other still active matching filter level.\n" + " 'disable' will print red filters that were already disabled.\n"; +static const char* const commandEnable = + " enable [ ...]\n" + " 'enable' takes space separated list of filter ids from 'filter'.\n" + " It will reset any matching category back to the default 'warning'\n" + " level or any other still active matching filter level.\n" + " 'enable' will print red filters that were already enabled.\n"; +static const char* const commandHelpDetails = + " help []\n" + " Show help for any of subcommands. Without any parameters it shows\n" + " short help for all subcommands.\n"; + +//! Helper type to hold static dispatch table for subcommands +struct CommandDispatch { + //! Store handler function pointer and help message for commands + struct Command { + using handler_t = command_result(*)(color_ostream&,std::vector&); + Command(handler_t handler, const char* help) : + handler_(handler), + help_(help) + {} + + command_result operator()(color_ostream& out, + std::vector& parameters) const noexcept + { + return handler_(out, parameters); + } + + handler_t handler() const noexcept + { return handler_; } + + const char* help() const noexcept + { return help_; } + private: + handler_t handler_; + const char* help_; + }; + using dispatch_t = const std::map; + //! Name to handler function and help message mapping + static dispatch_t dispatch; +}; + +struct LevelName { + static constexpr auto regex_opt = std::regex::icase | std::regex::optimize | std::regex::nosubs; + LevelName(const std::string& name) : + name_{name}, + match_{name, regex_opt} + {} + + bool match(const std::string& value) const + { + return std::regex_match(value, match_); + } + + operator const std::string&() const noexcept + { + return name_; + } + + const std::string& str() const noexcept + { + return name_; + } + + template + std::string operator+(const T& v) const + { + return name_ + v; + } +private: + std::string name_; + std::regex match_; +}; + +std::string operator+(const std::string& a, const LevelName& b) +{ + return a + static_cast(b); +} + +//! List of DebugCategory::level's in human readable form +static const std::array levelNames{ + LevelName{"Trace"}, + LevelName{"Debug"}, + LevelName{"Info"}, + LevelName{"Warning"}, + LevelName{"Error"}, +}; + +/*! + * Filter applies a runtime filter to matching DFHack::DebugCategory 's. + * Filters are stored in DFHack::debugPlugin::FilterManager which applies + * filters to dynamically added categories. The manager also stores and loads + * persistent Filters from the configuration file. + */ +struct Filter { + explicit Filter(DebugCategory::level level, + const std::string& categoryText, + const std::regex& category, + const std::string& pluginText, + const std::regex& plugin, + bool persistent = true, + bool enabled = true) noexcept; + + explicit Filter(DebugCategory::level level, + const std::string& categoryText, + const std::string& pluginText, + bool persistent = true, + bool enabled = true); + + explicit Filter() = default; + + //! Apply the filter to a category if regex's match + void apply(DFHack::DebugCategory& cat) noexcept; + //! Apply the filter to a category that has been already matched + bool applyAgain(DFHack::DebugCategory& cat) const noexcept; + //! Remove the category from matching count if regex's match + //! \return true if regex's matched + bool remove(const DFHack::DebugCategory& cat) noexcept; + + //! Query if filter is enabled + bool enabled() const noexcept {return enabled_;} + //! Set the enable status of filter + void enabled(bool enable) noexcept; + //! Query if filter is persistent + bool persistent() const noexcept {return persistent_;} + //! Query if the filter level + bool level() const noexcept {return level_;} + //! Query number of matching categories + size_t matches() const noexcept {return matches_;} + //! Add matches count for the initial category filter matching + void addMatch() noexcept {++matches_;} + //! Return the category filter text + const std::string& categoryText() const noexcept {return categoryText_;} + //! Return the plugin filter text + const std::string& pluginText() const noexcept {return pluginText_;} + + //! Load Filter from configuration file. Second parameter would be version + //! number if format changes in future to include more fields. Then new + //! fields would have to be loaded conditionally + void load(JsonArchive& ar, const unsigned int) + { + ar >> NVP(categoryText_) + >> NVP(pluginText_) + >> NVP(enabled_) + >> NVP(level_); + TRACE(filter) << "Loading filter cat: " << categoryText_ + << " plug: " << pluginText_ + << " ena " << enabled_ + << " level: " << level_ + << std::endl; + persistent_ = true; + matches_ = 0; + category_ = std::regex{categoryText_}; + plugin_ = std::regex{pluginText_}; + } + + //! Save the Filter to json configuration file + void save(JsonArchive& ar, const unsigned int) const + { + ar << NVP(categoryText_) + << NVP(pluginText_) + << NVP(enabled_) + << NVP(level_); + } + +private: + std::regex category_; + std::regex plugin_; + DebugCategory::level level_; + size_t matches_; + bool persistent_; + bool enabled_; + std::string categoryText_; + std::string pluginText_; +}; + +Filter::Filter(DebugCategory::level level, + const std::string& categoryText, + const std::regex& category, + const std::string& pluginText, + const std::regex& plugin, + bool persistent, + bool enabled) noexcept : + category_{category}, + plugin_{plugin}, + level_{level}, + matches_{0}, + persistent_{persistent}, + enabled_{enabled}, + categoryText_{categoryText}, + pluginText_{pluginText} +{} + +Filter::Filter(DebugCategory::level level, + const std::string& categoryText, + const std::string& pluginText, + bool persistent, + bool enabled) : + Filter{ + level, + categoryText, + std::regex{categoryText, defaultRegex}, + pluginText, + std::regex{pluginText, defaultRegex}, + persistent, + enabled, + } +{} + +void Filter::enabled(bool enable) noexcept +{ + if (enable == enabled_) + return; + enabled_ = enable; +} + +bool Filter::applyAgain(DebugCategory& cat) const noexcept +{ + if (!enabled_) + return false; + if (!std::regex_search(cat.category(), category_)) + return false; + if (!std::regex_search(cat.category(), plugin_)) + return false; + TRACE(filter) << "apply " << cat.plugin() << ':' << cat.category() << " matches '" + << pluginText() << "' '" << categoryText() << '\'' << std::endl; + cat.allowed(level_); + return true; +} + +void Filter::apply(DebugCategory& cat) noexcept +{ + if (applyAgain(cat)) + ++matches_; +} + +bool Filter::remove(const DebugCategory& cat) noexcept +{ + if (!enabled_) + return false; + if (!std::regex_search(cat.category(), category_)) + return false; + if (!std::regex_search(cat.category(), plugin_)) + return false; + TRACE(filter) << "remove " << cat.plugin() << ':' << cat.category() << " matches '" + << pluginText() << "' '" << categoryText() << std::endl; + --matches_; + return true; +} + +/** + * Storage for enabled and disabled filters. It uses ordered map because common + * case is to iterate filters from oldest to the newest. + * + * Data races: any reader and writer must lock + * DFHack::DebugManager::access_lock_. Sharing DebugManager make sense because + * most of functionality related to filters requires locking DebugManager too. + * Remaining functionality can share same lock because they are rarely called + * functions. + */ +struct FilterManager : public std::map +{ + using parent_t = std::map; + //! Current configuration version implemented by the code + constexpr static Json::UInt configVersion{1}; + //! Path to the configuration file + constexpr static const char* configPath{"dfhack-config/runtime-debug.json"}; + + //! Get reference to the singleton + static FilterManager& getInstance() noexcept + { + static FilterManager instance; + return instance; + } + + //! Add a new filter which gets automatically a new id + template + std::pair emplaceNew( Args&&... args ) { + return emplace(std::piecewise_construct, + std::forward_as_tuple(nextId_++), + std::forward_as_tuple(std::forward(args)...)); + } + + //! Load state from the configuration file + DFHack::command_result loadConfig(DFHack::color_ostream& out) noexcept; + //! Save state to the configuration file + DFHack::command_result saveConfig(DFHack::color_ostream& out) const noexcept; + + //! Connect FilterManager to DFHack::DebugManager::categorySignal + void connectTo(DebugManager::categorySignal_t& signal) noexcept; + //! Temporary block DFHack::DebugManager::categorySignal + DebugManager::categorySignal_t::BlockGuard blockSlot() noexcept; + + ~FilterManager(); + + FilterManager(const FilterManager&) = delete; + FilterManager(FilterManager&&) = delete; + FilterManager& operator=(FilterManager) = delete; + FilterManager& operator=(FilterManager&&) = delete; + + void load(JsonArchive& ar) + { + Json::UInt version = -1; + ar >> serialization::make_nvp("configVersion", version); + if (version > configVersion) { + std::stringstream ss; + ss << "The saved config version (" + << version + << ") is newer than code supported version (" + << configVersion + << ")"; + throw std::runtime_error{ss.str()}; + } + ar >> NVP(nextId_); + JsonArchive& children = ar["filters"]; + for (auto iter = children.begin(); iter != children.end(); ++iter) { + Filter child; + child.load(*iter, version); + std::stringstream ss(iter.name()); + size_t id; + ss >> id; + insert(std::make_pair(id, child)); + } + } + + void save(JsonArchive& ar) const + { + JsonArchive children{Json::objectValue}; + for (const auto& filterPair: *this) { + if (!filterPair.second.persistent()) + continue; + std::stringstream ss; + ss << filterPair.first; + filterPair.second.save(children[ss.str()], configVersion); + } + auto configVersion = FilterManager::configVersion; + ar << NVP(configVersion) + << NVP(nextId_); + ar["filters"] = children; + } + +private: + FilterManager() = default; + //! The next integer to use as id to have each with unique number + Json::UInt64 nextId_; + DebugManager::categorySignal_t::Connection connection_; +}; + +FilterManager::~FilterManager() +{ +} + +command_result FilterManager::loadConfig(DFHack::color_ostream& out) noexcept +{ + nextId_ = 1; + if (!Filesystem::isfile(configPath)) + return CR_OK; + try { + DEBUG(command, out) << "Load config from '" << configPath << "'" << std::endl; + JsonArchive archive; + std::ifstream ifs(configPath); + if (!ifs.good()) + throw std::runtime_error{"Failed to open configuration file for reading"}; + ifs >> archive; + load(archive); + } catch(std::exception& e) { + ERR(command, out) << "Serializing filters from '" << configPath << "' failed: " + << e.what() << std::endl; + return CR_FAILURE; + } + return CR_OK; +} + +command_result FilterManager::saveConfig(DFHack::color_ostream& out) const noexcept +{ + try { + DEBUG(command, out) << "Save config to '" << configPath << "'" << std::endl; + JsonArchive archive; + save(archive); + std::ofstream ofs(configPath); + if (!ofs.good()) + throw std::runtime_error{"Failed to open configuration file for writing"}; + ofs << archive; + } catch(std::exception e) { + ERR(command, out) << "Serializing filters to '" << configPath << "' failed: " + << e.what() << std::endl; + return CR_FAILURE; + } + return CR_OK; +} + +void FilterManager::connectTo(DebugManager::categorySignal_t& signal) noexcept +{ + connection_ = signal.connect( + [this](DebugManager::signalType t, DebugCategory& cat) { + TRACE(filter) << "sig type: " << t << ' ' + << cat.plugin() << ':' + << cat.category() << std::endl; + switch (t) { + case DebugManager::CAT_ADD: + for (auto& filterPair: *this) + filterPair.second.apply(cat); + break; + case DebugManager::CAT_REMOVE: + for (auto& filterPair: *this) + filterPair.second.remove(cat); + break; + case DebugManager::CAT_MODIFIED: + break; + } + }); +} + +DebugManager::categorySignal_t::BlockGuard FilterManager::blockSlot() noexcept +{ + TRACE(filter) << "Temporary disable FilterManager::connection_" << std::endl; + return {connection_}; +} + +//! \brief Helper to parse optional regex string safely +static command_result parseRegexParam(std::regex& target, + color_ostream& out, + std::vector& parameters, + size_t pos) +{ + if (parameters.size() <= pos) + return CR_OK; + try { + std::regex temp{parameters[pos], defaultRegex}; + target = std::move(temp); + } catch(std::regex_error e) { + ERR(command,out) << "Failed to parse regular expression '" + << parameters[pos] << "'\n"; + ERR(command,out) << "Parser message: " << e.what() << std::endl; + return CR_WRONG_USAGE; + } + return CR_OK; +} + +/*! + * "Algorithm" to apply category filters based on optional regex parameters. + * \param out The output stream where errors should be written + * \param paramters The list of parameters for the command + * \param pos The position where first optional regular expression parameter is + * \param header The callback after locking DebugManager and before loop + * \param categoryMatch The callback for each matching category + */ +template +static command_result applyCategoryFilters(color_ostream& out, + std::vector& parameters, + size_t pos, Callable1 header, + Callable2 categoryMatch, + Callable3 listComplete) +{ + std::regex pluginRegex{".", defaultRegex}; + std::regex categoryRegex{".", defaultRegex}; + command_result rv = CR_OK; + + DEBUG(command,out) << "applying category filters '" + << (parameters.size() >= pos + 1 ? parameters[pos] : "") + << "' and plugin filter '" + << (parameters.size() >= pos + 2 ? parameters[pos+1] : "") + << '\'' << std::endl; + // Parse parameters + if ((rv = parseRegexParam(pluginRegex, out, parameters, pos)) != CR_OK) + return rv; + if ((rv = parseRegexParam(categoryRegex, out, parameters, pos+1)) != CR_OK) + return rv; + // Lock the manager to have consistent view of categories + auto& manager = DebugManager::getInstance(); + std::lock_guard lock(manager.access_mutex_); + out << std::left; + auto guard = header(manager, categoryRegex, pluginRegex); + for (auto* category: manager) { + DebugCategory::cstring_ref p = category->plugin(); + DebugCategory::cstring_ref c = category->category(); + // Apply filters to the category and plugin names + if (!std::regex_search(c, categoryRegex)) + continue; + if (!std::regex_search(p, pluginRegex)) + continue; + categoryMatch(*category); + } + out << std::flush << std::right; + out.color(COLOR_RESET); + return listComplete(); +} + +static void printCategoryListHeader(color_ostream& out) +{ + // Output the header. + out.color(COLOR_GREEN); + out << std::setw(12) << "Plugin" + << std::setw(12) << "Category" + << std::setw(18) << "Lowest printed" << '\n'; +} + +static void printCategoryListEntry(color_ostream& out, + unsigned& line, + DebugCategory& cat, + DebugCategory::level old = static_cast(-1)) +{ + if ((line & 31) == 0) + printCategoryListHeader(out); + // Output matching categories. + out.color((line++ & 1) == 0 ? COLOR_CYAN : COLOR_LIGHTCYAN); + const std::string& level = (old != static_cast(-1)) ? + levelNames[static_cast(old)] + "->" + + levelNames[static_cast(cat.allowed())] : + levelNames[static_cast(cat.allowed())].str(); + out << std::setw(12) << cat.plugin() + << std::setw(12) << cat.category() + << std::setw(18) << level << '\n'; +} + +//! Handler for debugfilter category +static command_result listCategories(color_ostream& out, + std::vector& parameters) +{ + unsigned line = 0; + return applyCategoryFilters(out, parameters, 1u, + // After parameter parsing + [](DebugManager&, const std::regex&, const std::regex&) { + return 0; + }, + // Per category + [&out, &line](DebugCategory& cat) { + printCategoryListEntry(out, line, cat); + }, + // After list + []() {return CR_OK;}); +} + +//! Type that prints parameter string in center of output stream field +template> +struct center { + using string = std::basic_string; + center(const string& str) : + str_(str) + {} + + const string& str_; +}; + +/*! + * Helper to construct a center object to print fields centered + * \code{.cpp} + * out << std::setw(20) << centered("centered"_s); + * \endcode + */ +template +center centered(const ST& str) +{ + return {str}; +} + +//! c++14 string conversion literal to std::string +std::string operator "" _s(const char* cstr, size_t len) +{ + return {cstr, len}; +} + +//! Output centered string, the stream must be using std::ios::right +//! \sa DFHack::debugPlugin::centered +template +std::basic_ostream& operator<<(std::basic_ostream& os, const center& toCenter) +{ + std::streamsize w = os.width(); + const auto& str = toCenter.str_; + mbstate_t ps{}; + std::streamsize ccount = 0; + auto iter = str.cbegin(); + // Check multibyte character length. It will break when mbrlen find the '\0' + for (;ccount < w; ++ccount) { + const size_t n = std::distance(iter, str.cend()); + using ss_t = std::make_signed::type; + ss_t bytes = mbrlen(&*iter, n, &ps); + if (bytes <= 0) /* Check for errors and null */ + break; + std::advance(iter, bytes); + } + + if (ccount < w) { + // Center the character when there is less than the width + // fillw = w - count + // cw = w - (fillw)/2 = (2w - w + count)/2 + // Extra one for rounding half results up + std::streamsize cw = (w + ccount + 1)/2; + os << std::setw(cw) << str + << std::setw(w - cw) << ""; + } else { + // Truncate characters to the width of field + os.write(&str[0], std::distance(str.begin(), iter)); + // Reset the field width because we wrote the string with write + os << std::setw(0); + } + return os; +} + +static FilterManager::iterator parseFilterId(color_ostream& out, + const std::string& parameter) +{ + unsigned long id = 0; + try { + id = std::stoul(parameter); + } catch(...) { + } + auto& filMan = FilterManager::getInstance(); + auto iter = filMan.find(id); + if (iter == filMan.end()) { + WARN(command,out) << "The optional parameter (" + << parameter << ") must be an filter id." << std::endl; + } + return iter; +} + +static void printFilterListEntry(color_ostream& out, + unsigned line, + color_value lineColor, + size_t id, + const Filter& filter) +{ + if ((line & 31) == 0) { + out.color(COLOR_GREEN); + out << std::setw(4) << "ID" + << std::setw(8) << "enabled" + << std::setw(8) << "persist" + << std::setw(9) << centered("level"_s) + << ' ' + << std::setw(15) << centered("category"_s) + << ' ' + << std::setw(15) << centered("plugin"_s) + << std::setw(8) << "matches" + << '\n'; + } + out.color(lineColor); + out << std::setw(4) << id + << std::setw(8) << centered(filter.enabled() ? "X"_s:""_s) + << std::setw(8) << centered(filter.persistent() ? "X"_s:""_s) + << std::setw(9) << centered(levelNames[filter.level()].str()) + << ' ' + << std::setw(15) << centered(filter.categoryText()) + << ' ' + << std::setw(15) << centered(filter.pluginText()) + << std::setw(8) << filter.matches() + << '\n'; +} + +//! Handler for debugfilter filter +static command_result listFilters(color_ostream& out, + std::vector& parameters) +{ + if (1u < parameters.size()) { + auto& catMan = DebugManager::getInstance(); + std::lock_guard lock(catMan.access_mutex_); + auto iter = parseFilterId(out, parameters[1]); + if (iter == FilterManager::getInstance().end()) + return CR_WRONG_USAGE; + + auto id = iter->first; + Filter& filter = iter->second; + + out << std::left + << std::setw(10) << "ID:" << id << '\n' + << std::setw(10) << "Enabled:" << (filter.enabled() ? "Yes"_s:"No"_s) << '\n' + << std::setw(10) << "Persist:" << (filter.persistent() ? "Yes"_s:"No"_s) << '\n' + << std::setw(10) << "Level:" << levelNames[filter.level()].str() << '\n' + << std::setw(10) << "category:" << filter.categoryText() << '\n' + << std::setw(10) << "plugin:" << filter.pluginText() << '\n' + << std::setw(10) << "matches:" << filter.matches() << '\n' + << std::right + << std::endl; + return CR_OK; + } + auto& catMan = DebugManager::getInstance(); + { + std::lock_guard lock(catMan.access_mutex_); + auto& filMan = FilterManager::getInstance(); + unsigned line = 0; + for (auto& filterPair: filMan) { + size_t id = filterPair.first; + const Filter& filter = filterPair.second; + + color_value c = (line & 1) == 0 ? COLOR_CYAN : COLOR_LIGHTCYAN; + printFilterListEntry(out,line++,c,id,filter); + } + } + out.color(COLOR_RESET); + out.flush(); + return CR_OK; +} + +static const std::string persistent("persistent"); + +//! Handler for debugfilter set +static command_result setFilter(color_ostream& out, + std::vector& parameters) +{ + bool persist = false; + size_t pos = 1u; + if (pos < parameters.size() && + parameters[pos] == persistent) { + pos++; + persist = true; + } + if (pos >= parameters.size()) { + ERR(command,out).print("set requires at least the level parameter\n"); + return CR_WRONG_USAGE; + } + const std::string& level = parameters[pos]; + auto iter = std::find_if(levelNames.begin(), levelNames.end(), + [&level](const LevelName& v) -> bool { + return v.match(level); + }); + if (iter == levelNames.end()) { + ERR(command,out).print("level ('%s') parameter must be one of " + "trace, debug, info, warning, error.\n", + parameters[pos].c_str()); + return CR_WRONG_USAGE; + } + + DebugCategory::level filterLevel = static_cast( + iter - levelNames.begin()); + + unsigned line = 0; + Filter* newFilter = nullptr; + return applyCategoryFilters(out, parameters, pos + 1, + // After parameters parsing + [¶meters, pos, filterLevel,persist,&newFilter](DebugManager&, const std::regex& catRegex, const std::regex& pluginRegex) + { + auto& filMan = FilterManager::getInstance(); + newFilter = &filMan.emplaceNew(filterLevel, + pos+1 < parameters.size()?parameters[pos+1]:".", + catRegex, + pos+2 < parameters.size()?parameters[pos+2]:".", + pluginRegex, + persist).first->second; + return filMan.blockSlot(); + }, + // Per item + [filterLevel, &line, &out, &newFilter](DebugCategory& cat) { + auto old = cat.allowed(); + cat.allowed(filterLevel); + newFilter->addMatch(); + printCategoryListEntry(out, line, cat, old); + }, + // After list + [&out,&persist]() { + if (persist) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +template +static command_result applyFilterIds(color_ostream& out, + std::vector& parameters, + const char* name, + HighlightRed hlRed, + ListComplete listComplete) +{ + if (1u >= parameters.size()) { + ERR(command,out) << name << " requires at least a filter id" << std::endl; + return CR_WRONG_USAGE; + } + command_result rv = CR_OK; + { + auto& catMan = DebugManager::getInstance(); + std::lock_guard lock(catMan.access_mutex_); + auto& filMan = FilterManager::getInstance(); + unsigned line = 0; + for (size_t pos = 1; pos < parameters.size(); ++pos) { + const std::string& p = parameters[pos]; + auto iter = parseFilterId(out, p); + if (iter == filMan.end()) + continue; + color_value c = (line & 1) == 0 ? COLOR_CYAN : COLOR_LIGHTCYAN; + if (hlRed(iter)) + c = COLOR_RED; + printFilterListEntry(out,line++,c,iter->first,iter->second); + } + rv = listComplete(); + } + out.color(COLOR_RESET); + out.flush(); + return rv; +} + +//! Handler for debugfilter disable +static command_result disableFilter(color_ostream& out, + std::vector& parameters) +{ + std::set modified; + bool mustSave = false; + return applyFilterIds(out,parameters,"disable", + // Per item + [&modified,&mustSave](FilterManager::iterator& iter) -> bool { + Filter& filter = iter->second; + bool enabled = filter.enabled(); + if (enabled == false) + return true; + auto& catMan = DebugManager::getInstance(); + for (DebugCategory* cat: catMan) { + if (filter.remove(*cat)) + modified.emplace(cat); + } + filter.enabled(false); + mustSave = mustSave || filter.persistent(); + return false; + }, + // After list + [&modified,&mustSave,&out]() { + for (DebugCategory* cat: modified) { + // Reset filtering back to default + cat->allowed(DebugCategory::LWARNING); + auto& filMan = FilterManager::getInstance(); + // Reapply all remaining filters + for (auto& filterPair: filMan) + filterPair.second.applyAgain(*cat); + } + if (mustSave) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +//! Handler for debugfilter enable +static command_result enableFilter(color_ostream& out, + std::vector& parameters) +{ + std::set modified; + bool mustSave = false; + return applyFilterIds(out,parameters,"enable", + // Per item + [&modified,&mustSave](FilterManager::iterator& iter) -> bool { + Filter& filter = iter->second; + bool enabled = filter.enabled(); + if (enabled == true) + return true; + filter.enabled(true); + auto& catMan = DebugManager::getInstance(); + for (DebugCategory* cat: catMan) { + if (filter.applyAgain(*cat)) { + modified.emplace(cat); + filter.addMatch(); + } + } + mustSave = mustSave || filter.persistent(); + return false; + }, + // After list + [&modified,&mustSave,&out]() { + for (DebugCategory* cat: modified) { + // Reset filtering back to default + cat->allowed(DebugCategory::LWARNING); + auto& filMan = FilterManager::getInstance(); + // Reapply all remaining filters + for (auto& filterPair: filMan) + filterPair.second.applyAgain(*cat); + } + if (mustSave) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +//! Handler for debugfilter unset +static command_result unsetFilter(color_ostream& out, + std::vector& parameters) +{ + std::set modified; + std::vector toErase; + return applyFilterIds(out,parameters,"unset", + // Per item + [&modified, &toErase](FilterManager::iterator& iter) -> bool { + Filter& filter = iter->second; + if (filter.enabled()) { + auto& catMan = DebugManager::getInstance(); + for (DebugCategory* cat: catMan) { + if (filter.remove(*cat)) + modified.emplace(cat); + } + } + toErase.emplace_back(iter); + return false; + }, + // After list + [&modified,&toErase,&out]() { + auto& filMan = FilterManager::getInstance(); + bool mustSave = false; + for (auto iter: toErase) { + mustSave = mustSave || iter->second.persistent(); + filMan.erase(iter); + } + + for (DebugCategory* cat: modified) { + // Reset filtering back to default + cat->allowed(DebugCategory::LWARNING); + // Reapply all remaining filters + for (auto& filterPair: filMan) + filterPair.second.applyAgain(*cat); + } + if (mustSave) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +using DFHack::debugPlugin::CommandDispatch; + +static command_result printHelp(color_ostream& out, + std::vector& parameters) +{ + const char* help = commandHelp; + auto iter = CommandDispatch::dispatch.end(); + if (1u < parameters.size()) + iter = CommandDispatch::dispatch.find(parameters[1]); + if (iter != CommandDispatch::dispatch.end()) + help = iter->second.help(); + out << help << std::flush; + return CR_OK; +} + +CommandDispatch::dispatch_t CommandDispatch::dispatch { + {"category", {listCategories,commandCategory}}, + {"filter", {listFilters,commandFilters}}, + {"set", {setFilter,commandSet}}, + {"unset", {unsetFilter,commandUnset}}, + {"enable", {enableFilter,commandEnable}}, + {"disable", {disableFilter,commandDisable}}, + {"help", {printHelp,commandHelpDetails}}, +}; + +//! Dispatch command handling to the subcommand or help +static command_result commandDebugFilter(color_ostream& out, + std::vector& parameters) +{ + DEBUG(command,out).print("debugfilter %s, parameter count %zu\n", + parameters.size() > 0 ? parameters[0].c_str() : "", + parameters.size()); + auto handler = printHelp; + auto iter = CommandDispatch::dispatch.end(); + if (0u < parameters.size()) + iter = CommandDispatch::dispatch.find(parameters[0]); + if (iter != CommandDispatch::dispatch.end()) + handler = iter->second.handler(); + return (handler)(out, parameters); +} + +} } /* namespace debug */ + +DFhackCExport DFHack::command_result plugin_init(DFHack::color_ostream& out, + std::vector& commands) +{ + commands.emplace_back( + "debugfilter", + "Manage runtime debug print filters", + DFHack::debugPlugin::commandDebugFilter, + false, + DFHack::debugPlugin::commandHelp); + auto& filMan = DFHack::debugPlugin::FilterManager::getInstance(); + DFHack::command_result rv = DFHack::CR_OK; + if ((rv = filMan.loadConfig(out)) != DFHack::CR_OK) + return rv; + auto& catMan = DFHack::DebugManager::getInstance(); + std::lock_guard lock(catMan.access_mutex_); + for (auto* cat: catMan) { + for (auto& filterPair: filMan) { + DFHack::debugPlugin::Filter& filter = filterPair.second; + filter.apply(*cat); + } + } + INFO(init,out).print("plugin_init with %zu commands, %zu filters and %zu categories\n", + commands.size(), filMan.size(), catMan.size()); + filMan.connectTo(catMan.categorySignal); + return rv; +} + +DFhackCExport DFHack::command_result plugin_shutdown(DFHack::color_ostream& out) +{ + INFO(init,out).print("plugin_shutdown\n"); + return DFHack::CR_OK; +} From 490a8557766bf3459ff85d8e0cb2ea941f311c24 Mon Sep 17 00:00:00 2001 From: Pauli Date: Mon, 2 Jul 2018 18:48:33 +0300 Subject: [PATCH 4/5] Add a test for signal_shared_tag implementation The test cases check that the signal_shared_tag implementation can be used and destructed safely from multiple threads. --- plugins/CMakeLists.txt | 2 + plugins/devel/CMakeLists.txt | 2 +- plugins/devel/kittens.cpp | 178 +++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 1 deletion(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index a05d680ea..07a94112c 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -1,5 +1,7 @@ INCLUDE(Plugins.cmake) +find_package(Threads) + OPTION(BUILD_STONESENSE "Build stonesense (needs a checkout first)." OFF) if(BUILD_STONESENSE) add_subdirectory (stonesense) diff --git a/plugins/devel/CMakeLists.txt b/plugins/devel/CMakeLists.txt index 4fb4b5cf5..879edd5bb 100644 --- a/plugins/devel/CMakeLists.txt +++ b/plugins/devel/CMakeLists.txt @@ -9,7 +9,7 @@ DFHACK_PLUGIN(counters counters.cpp) DFHACK_PLUGIN(dumpmats dumpmats.cpp) DFHACK_PLUGIN(eventExample eventExample.cpp) DFHACK_PLUGIN(frozen frozen.cpp) -DFHACK_PLUGIN(kittens kittens.cpp) +DFHACK_PLUGIN(kittens kittens.cpp LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) DFHACK_PLUGIN(memview memview.cpp memutils.cpp LINK_LIBRARIES lua) DFHACK_PLUGIN(nestboxes nestboxes.cpp) DFHACK_PLUGIN(notes notes.cpp) diff --git a/plugins/devel/kittens.cpp b/plugins/devel/kittens.cpp index 5de94ced3..26a2655e8 100644 --- a/plugins/devel/kittens.cpp +++ b/plugins/devel/kittens.cpp @@ -1,12 +1,16 @@ #include #include +#include #include +#include #include "Console.h" #include "Core.h" +#include "Debug.h" #include "Export.h" #include "MiscUtils.h" #include "PluginManager.h" +#include "Signal.hpp" #include "modules/Gui.h" #include "modules/Items.h" @@ -25,6 +29,10 @@ DFHACK_PLUGIN_IS_ENABLED(is_enabled); REQUIRE_GLOBAL(ui); REQUIRE_GLOBAL(world); +namespace DFHack { +DBG_DECLARE(kittens,command); +} + std::atomic shutdown_flag{false}; std::atomic final_flag{true}; std::atomic timering{false}; @@ -42,6 +50,7 @@ command_result trackmenu (color_ostream &out, vector & parameters); command_result trackpos (color_ostream &out, vector & parameters); command_result trackstate (color_ostream &out, vector & parameters); command_result colormods (color_ostream &out, vector & parameters); +command_result sharedsignal (color_ostream &out, vector & parameters); DFhackCExport command_result plugin_init ( color_ostream &out, std::vector &commands) { @@ -51,6 +60,7 @@ DFhackCExport command_result plugin_init ( color_ostream &out, std::vector & parameters) return CR_OK; } +struct Connected; +using shared = std::shared_ptr; +using weak = std::weak_ptr; + +static constexpr std::chrono::microseconds delay{1}; + +template +struct ClearMem : public ConnectedBase { + ~ClearMem() + { + memset(reinterpret_cast(this), 0xDE, sizeof(Derived)); + } +}; + +struct Connected : public ClearMem { + using Sig = Signal; + std::array con; + Sig signal; + weak other; + Sig::weak_ptr other_sig; + color_ostream *out; + int id; + uint32_t count; + uint32_t caller; + alignas(64) std::atomic callee; + Connected() = default; + Connected(int id) : + Connected{} + { + this->id = id; + } + void connect(color_ostream& o, shared& b, size_t pos, uint32_t c) + { + out = &o; + count = c*2; + other = b; + other_sig = b->signal.weak_from_this(); + // Externally synchronized object destruction is only safe to this + // connect. + con[pos] = b->signal.connect( + [this](int) { + uint32_t old = callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(callee != 0xDEDEDEDE); + }); + // Shared object managed object with possibility of destruction while + // other threads calling emit must pass the shared_ptr to connect. + Connected *bptr = b.get(); + b->con[pos] = signal.connect(b, + [bptr](int) { + uint32_t old = bptr->callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(bptr->callee != 0xDEDEDEDE); + }); + } + void reconnect(size_t pos) { + auto b = other.lock(); + if (!b) + return; + // Not required to use Sig::lock because other holds strong reference to + // Signal. But this just shows how weak_ref could be used. + auto sig = Sig::lock(other_sig); + if (!sig) + return; + con[pos] = sig->connect(b, + [this](int) { + uint32_t old = callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(callee != 0xDEDEDEDE); + }); + } + void connect(color_ostream& o, shared& a, shared& b,size_t pos, uint32_t c) + { + out = &o; + count = c; + con[pos] = b->signal.connect(a, + [this](int) { + uint32_t old = callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(callee != 0xDEDEDEDE); + }); + } + Connected* operator->() noexcept + { + return this; + } + ~Connected() { + INFO(command,*out).print("Connected %d had %d count. " + "It was caller %d times. " + "It was callee %d times.\n", + id, count, caller, callee.load()); + } +}; + +command_result sharedsignal (color_ostream &out, vector & parameters) +{ + using rng_t = std::linear_congruential_engine; + rng_t rng(std::random_device{}()); + size_t count = 10; + if (0 < parameters.size()) { + std::stringstream ss(parameters[0]); + ss >> count; + DEBUG(command, out) << "Parsed " << count + << " from paramters[0] '" << parameters[0] << '\'' << std::endl; + } + + + std::uniform_int_distribution dis(4096,8192); + out << "Running signal_shared_tag destruction test " + << count << " times" << std::endl; + for (size_t nr = 0; nr < count; ++nr) { + std::array t{}; + // Make an object which destruction is protected by std::thread::join() + Connected external{static_cast(t.size())}; + TRACE(command, out) << "begin " << std::endl; + { + int id = 0; + // Make objects that are automatically protected using weak_ptr + // references that are promoted to shared_ptr when Signal is + // accessed. + std::array c = { + std::make_shared(id++), + std::make_shared(id++), + std::make_shared(id++), + std::make_shared(id++), + }; + assert(t.size() == c.size()); + for (unsigned i = 1; i < c.size(); ++i) { + c[0]->connect(out, c[0], c[i], i - 1, dis(rng)); + c[i]->connect(out, c[i], c[0], 0, dis(rng)); + } + external.connect(out, c[1], 1, dis(rng)); + auto thr = [&out](shared c) { + TRACE(command, out) << "Thread " << c->id << " started." << std::endl; + weak ref = c; + for (;c->caller < c->count; ++c->caller) { + c->signal(c->caller); + } + TRACE(command, out) << "Thread " << c->id << " resets shared." << std::endl; + c.reset(); + while((c = ref.lock())) { + ++c->caller; + c->signal(c->caller); + c.reset(); + std::this_thread::sleep_for(delay*25); + } + }; + for (unsigned i = 0; i < c.size(); ++i) { + TRACE(command, out) << "start thread " << i << std::endl; + t[i] = std::thread{thr, c[i]}; + } + } + TRACE(command, out) << "running " << std::endl; + for (;external->caller < external->count; ++external->caller) { + external->signal(external->caller); + external->reconnect(1); + } + TRACE(command, out) << "join " << std::endl; + for (unsigned i = 0; i < t.size(); ++i) + t[i].join(); + } + return CR_OK; +} + command_result kittens (color_ostream &out, vector & parameters) { if (parameters.size() >= 1) From c201cf5b7b3faf44e109484ebdcb84a7a0589346 Mon Sep 17 00:00:00 2001 From: Pauli Date: Wed, 4 Jul 2018 14:54:00 +0300 Subject: [PATCH 5/5] Documentation and Changelog for debug printing and Signal --- docs/Plugins.rst | 96 ++++++++++++++++++++++++++++++++++++++++++++++ docs/changelog.txt | 15 ++++++++ 2 files changed, 111 insertions(+) diff --git a/docs/Plugins.rst b/docs/Plugins.rst index 5f41bafed..bbeeb813a 100644 --- a/docs/Plugins.rst +++ b/docs/Plugins.rst @@ -392,6 +392,102 @@ Otherwise somewhat similar to `gui/quickcmd`. .. image:: images/command-prompt.png +.. _debug: + +debug +===== +Manager DFHack runtime debug prints. Debug prints are grouped by plugin name, +category name and print level. Levels are ``trace``, ``debug``, ``info``, +``warning`` and ``error``. + +The runtime message printing is controlled using filters. Filters set minimum +visible message to all matching categories. Matching uses regular expression +that allows listing multiple alternative matches or partial name matches. +Persistent filters are stored in ``dfhack-config/runtime-debug.json``. + +Oldest filters are applied first. That means a newer filter can override the +older printing level selection. + +Usage: ``debugfilter [subcommand] [parameters...]`` + +Following subcommands are supported. + +Regular expression syntax +------------------------- + +Syntax is C++ version of ECMA-262 grammar (Javascript regular expression). +Deails of differences can be found from +https://en.cppreference.com/w/cpp/regex/ecmascript + +help +---- +Give overall help or a detailed help for a subcommand. + +Usage: ``debugfilter help [subcommand]`` + +category +-------- +List available debug plugin and category names. + +Usage: ``debugfilter category [plugin regex] [category regex]`` + +The list can be filtered using optional regex parameters. If filters aren't +given then the it uses ``"."`` regex which matches any character. The regex +parameters are good way to test regex before passing them to ``set``. + +filter +------ +List active and passive debug print level changes. + +Usage: ``debugfilter filter [id]`` + +Optional ``id`` parameter is the id listed as first column in the filter list. +If id is given then the command shows information for the given filter only in +multi line format that is better format if filter has long regex. + +set +--- +Creates a new debug filter to set category printing levels. + +Usage: ``debugfilter set [level] [plugin regex] [category regex]`` + +Adds a filter that will be deleted when DF process exists or plugin is unloaded. + +Usage: ``debugfilter set persistent [level] [plugin regex] [category regex]`` + +Stores the filter in the configuration file to until ``unset`` is used to remove +it. + +Level is the minimum debug printing level to show in log. + +* ``trace``: Possibly very noisy messages which can be printed many times per second + +* ``debug``: Messages that happen often but they should happen only a couple of times per second + +* ``info``: Important state changes that happen rarely during normal execution + +* ``warining``: Enabled by default. Shows warnings about unexpected events which code managed to handle correctly. + +* ``error``: Enabled by default. Shows errors which code can't handle without user intervention. + +unset +----- +Delete a space separated list of filters + +Usage: ``debugfilter unset [id...]`` + +disable +------- +Disable a space separated list of filters but keep it in the filter list + +Usage: ``debugfilter disable [id...]`` + +enable +------ +Enable a space sperate list of filters + +Usage: ``debugfilter enable [id...]`` + .. _hotkeys: hotkeys diff --git a/docs/changelog.txt b/docs/changelog.txt index 083697b3b..3bf19b338 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -37,6 +37,9 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: ================================================================================ # Future +## New Plugins +- `debug`: manages runtime debug print category filtering + ## Fixes - `fix/dead-units`: fixed script trying to use missing isDiplomat function @@ -56,6 +59,18 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: - New functions: - ``Units::isDiplomat(unit)`` - Exposed ``Screen::zoom()`` to C++ (was Lua-only) +- New classes: + - ``Signal`` to C++ only + - ``DebugCategory`` to C++ only (used through new macros) + - ``DebugManager`` to C++ only +- New macros: + - ``DBG_DECLARE`` + - ``DBG_EXTERN`` + - ``TRACE`` + - ``DEBUG`` + - ``INFO`` + - ``WARN`` + - ``ERR`` ## Lua - ``gui.widgets``: ``List:setChoices`` clones ``choices`` for internal table changes