diff --git a/docs/Plugins.rst b/docs/Plugins.rst index ee2c56e02..180915527 100644 --- a/docs/Plugins.rst +++ b/docs/Plugins.rst @@ -392,6 +392,102 @@ Otherwise somewhat similar to `gui/quickcmd`. .. image:: images/command-prompt.png +.. _debug: + +debug +===== +Manager DFHack runtime debug prints. Debug prints are grouped by plugin name, +category name and print level. Levels are ``trace``, ``debug``, ``info``, +``warning`` and ``error``. + +The runtime message printing is controlled using filters. Filters set minimum +visible message to all matching categories. Matching uses regular expression +that allows listing multiple alternative matches or partial name matches. +Persistent filters are stored in ``dfhack-config/runtime-debug.json``. + +Oldest filters are applied first. That means a newer filter can override the +older printing level selection. + +Usage: ``debugfilter [subcommand] [parameters...]`` + +Following subcommands are supported. + +Regular expression syntax +------------------------- + +Syntax is C++ version of ECMA-262 grammar (Javascript regular expression). +Deails of differences can be found from +https://en.cppreference.com/w/cpp/regex/ecmascript + +help +---- +Give overall help or a detailed help for a subcommand. + +Usage: ``debugfilter help [subcommand]`` + +category +-------- +List available debug plugin and category names. + +Usage: ``debugfilter category [plugin regex] [category regex]`` + +The list can be filtered using optional regex parameters. If filters aren't +given then the it uses ``"."`` regex which matches any character. The regex +parameters are good way to test regex before passing them to ``set``. + +filter +------ +List active and passive debug print level changes. + +Usage: ``debugfilter filter [id]`` + +Optional ``id`` parameter is the id listed as first column in the filter list. +If id is given then the command shows information for the given filter only in +multi line format that is better format if filter has long regex. + +set +--- +Creates a new debug filter to set category printing levels. + +Usage: ``debugfilter set [level] [plugin regex] [category regex]`` + +Adds a filter that will be deleted when DF process exists or plugin is unloaded. + +Usage: ``debugfilter set persistent [level] [plugin regex] [category regex]`` + +Stores the filter in the configuration file to until ``unset`` is used to remove +it. + +Level is the minimum debug printing level to show in log. + +* ``trace``: Possibly very noisy messages which can be printed many times per second + +* ``debug``: Messages that happen often but they should happen only a couple of times per second + +* ``info``: Important state changes that happen rarely during normal execution + +* ``warining``: Enabled by default. Shows warnings about unexpected events which code managed to handle correctly. + +* ``error``: Enabled by default. Shows errors which code can't handle without user intervention. + +unset +----- +Delete a space separated list of filters + +Usage: ``debugfilter unset [id...]`` + +disable +------- +Disable a space separated list of filters but keep it in the filter list + +Usage: ``debugfilter disable [id...]`` + +enable +------ +Enable a space sperate list of filters + +Usage: ``debugfilter enable [id...]`` + .. _hotkeys: hotkeys diff --git a/docs/changelog.txt b/docs/changelog.txt index 586d21311..2635e1497 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -38,6 +38,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: # Future ## New Plugins +- `debug`: manages runtime debug print category filtering - `nestboxes`: automatically scan for and forbid fertile eggs incubating in a nestbox ## Fixes @@ -76,6 +77,10 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: - ``incident_sub6_performance.participants``: named performance_event and role_index - ``incident_sub6_performance``: named poetic_form_id, musical_form_id, and dance_form_id +## API +- New debug features related to `debug` plugin: + - Classes (C++ only): ``Signal``, ``DebugCategory``, ``DebugManager`` + - Macros: ``TRACE``, ``DEBUG``, ``INFO``, ``WARN``, ``ERR``, ``DBG_DECLARE``, ``DBG_EXTERN`` ================================================================================ # 0.44.12-r1 diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 33a430337..a9255ae00 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -28,6 +28,8 @@ include/Core.h include/ColorText.h include/DataDefs.h include/DataIdentity.h +include/Debug.h +include/DebugManager.h include/VTableInterpose.h include/LuaWrapper.h include/LuaTools.h @@ -38,6 +40,7 @@ include/MiscUtils.h include/Module.h include/Pragma.h include/MemAccess.h +include/Signal.hpp include/TileTypes.h include/Types.h include/VersionInfo.h @@ -54,7 +57,9 @@ include/wdirent.h SET(MAIN_SOURCES Core.cpp ColorText.cpp +CompilerWorkAround.cpp DataDefs.cpp +Debug.cpp Error.cpp VTableInterpose.cpp LuaWrapper.cpp diff --git a/library/CompilerWorkAround.cpp b/library/CompilerWorkAround.cpp new file mode 100644 index 000000000..402553bd8 --- /dev/null +++ b/library/CompilerWorkAround.cpp @@ -0,0 +1,32 @@ +#include + +namespace DFHack { +namespace neverCalled { + +/** + * gcc/linstdc++ seems to generate code that links libstdc++ back to first + * shared object using std::regex. To allow plugins unload with std::regex in + * the code we need the std::regex functions inside libdfhack.so. + * + * If your plugin decides to use any overloads that aren't listed here it may + * stay in memory after dlclose. + */ +std::regex stdRegexPluginUnloadWorkaround() +{ + std::regex fake("foo"); + std::string haystack("bar is foo in the world"); + std::regex fake2(std::string("bar")); + if (std::regex_match(haystack, fake)) + std::swap(fake, fake2); + if (std::regex_search(haystack, fake)) + std::swap(fake, fake2); + const char* haystack2 = "foo"; + if (std::regex_match(haystack2, fake)) + std::swap(fake, fake2); + if (std::regex_search(haystack2, fake)) + std::swap(fake, fake2); + return fake; +} + +} +} diff --git a/library/Debug.cpp b/library/Debug.cpp new file mode 100644 index 000000000..aa0cf8cdf --- /dev/null +++ b/library/Debug.cpp @@ -0,0 +1,186 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ +#define _POSIX_C_SOURCE 200809L +#include "Core.h" + +#include "Debug.h" +#include "DebugManager.h" + +#include +#include +#include + +#ifdef _MSC_VER +static tm* localtime_r(const time_t* time, tm* result) +{ + localtime_s(result, time); + return result; +} +#endif + +namespace DFHack { +DBG_DECLARE(core,debug); + +void DebugManager::registerCategory(DebugCategory& cat) +{ + DEBUG(debug) << "register DebugCategory '" << cat.category() + << "' from '" << cat.plugin() + << "' allowed " << cat.allowed() << std::endl; + std::lock_guard guard(access_mutex_); + push_back(&cat); + categorySignal(CAT_ADD, cat); +} + +void DebugManager::unregisterCategory(DebugCategory& cat) +{ + DEBUG(debug) << "unregister DebugCategory '" << cat.category() + << "' from '" << cat.plugin() + << "' allowed " << cat.allowed() << std::endl; + std::lock_guard guard(access_mutex_); + auto iter = std::find(begin(), end(), &cat); + std::swap(*iter, back()); + pop_back(); + categorySignal(CAT_REMOVE, cat); +} + +DebugRegisterBase::DebugRegisterBase(DebugCategory* cat) +{ + // Make sure Core lives at least as long any DebugCategory to + // allow debug prints until all Debugcategories has been destructed + Core::getInstance(); + DebugManager::getInstance().registerCategory(*cat); +} + +void DebugRegisterBase::unregister(DebugCategory* cat) +{ + DebugManager::getInstance().unregisterCategory(*cat); +} + +static color_value selectColor(const DebugCategory::level msgLevel) +{ + switch(msgLevel) { + case DebugCategory::LTRACE: + return COLOR_GREY; + case DebugCategory::LDEBUG: + return COLOR_LIGHTBLUE; + case DebugCategory::LINFO: + return COLOR_CYAN; + case DebugCategory::LWARNING: + return COLOR_YELLOW; + case DebugCategory::LERROR: + return COLOR_LIGHTRED; + } + return COLOR_WHITE; +} + +#if __GNUC__ +// Allow gcc to optimize tls access. It also makes sure initialized is done as +// early as possible. The early initialization helps to give threads same ids as +// gdb shows. +#define EXEC_ATTR __attribute__((tls_model("initial-exec"))) +#else +#define EXEC_ATTR +#endif + +namespace { +static std::atomic nextId{0}; +static EXEC_ATTR thread_local uint32_t thread_id{nextId.fetch_add(1)+1}; +} + +DebugCategory::ostream_proxy_prefix::ostream_proxy_prefix( + const DebugCategory& cat, + color_ostream& target, + const DebugCategory::level msgLevel) : + color_ostream_proxy(target) +{ + color(selectColor(msgLevel)); + auto now = std::chrono::system_clock::now(); + tm local{}; + //! \todo c++ 2020 will have std::chrono::to_stream(fmt, system_clock::now()) + //! but none implements it yet. + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + auto ms = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + // Output time in format %02H:%02M:%02S.%03ms +#if __GNUC__ < 5 + // Fallback for gcc 4 + char buffer[32]; + size_t sz = strftime(buffer, sizeof(buffer)/sizeof(buffer[0]), + "%T.", localtime_r(&now_c, &local)); + *this << (sz > 0 ? buffer : "HH:MM:SS.") +#else + *this << std::put_time(localtime_r(&now_c, &local),"%T.") +#endif + << std::setfill('0') << std::setw(3) << ms.count() + // Thread id is allocated in the thread creation order to a thread_local + // variable + << ":t" << thread_id + // Output plugin and category names to make it easier to locate where + // the message is coming. It would be easy replaces these with __FILE__ + // and __LINE__ passed from the macro if that would be preferred prefix. + << ':' << cat.plugin() << ':' << cat.category() << ": "; +} + + +DebugCategory::level DebugCategory::allowed() const noexcept +{ + return allowed_.load(std::memory_order_relaxed); +} + +void DebugCategory::allowed(DebugCategory::level value) noexcept +{ + level old = allowed_.exchange(value, std::memory_order_relaxed); + if (old == value) + return; + TRACE(debug) << "modify DebugCategory '" << category() + << "' from '" << plugin() + << "' allowed " << value << std::endl; + auto& manager = DebugManager::getInstance(); + manager.categorySignal(DebugManager::CAT_MODIFIED, *this); +} + +DebugCategory::cstring_ref DebugCategory::category() const noexcept +{ + return category_; +} + +DebugCategory::cstring_ref DebugCategory::plugin() const noexcept +{ + return plugin_; +} + +#if __cplusplus < 201703L && __cpp_lib_atomic_is_always_lock_free < 201603 +//! C++17 has std::atomic::is_always_lock_free for static_assert. Older +//! standards only provide runtime checks if an atomic type is lock free +struct failIfEnumAtomicIsNotLockFree { + failIfEnumAtomicIsNotLockFree() { + std::atomic test; + if (test.is_lock_free()) + return; + std::cerr << __FILE__ << ':' << __LINE__ + << ": error: std::atomic should be lock free. Your compiler reports the atomic requires runtime locks. Either you are using a very old CPU or we need to change code to use integer atomic type." << std::endl; + std::abort(); + } +} failIfEnumAtomicIsNotLockFree; +#endif + +} diff --git a/library/include/Debug.h b/library/include/Debug.h new file mode 100644 index 000000000..976172c8d --- /dev/null +++ b/library/include/Debug.h @@ -0,0 +1,369 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ + +#pragma once + +#include "ColorText.h" + +#include +#include "Core.h" + +namespace DFHack { + +/*! \file Debug.h + * Light weight wrappers to runtime debug output filtering. Idea is to add as + * little as possible code compared to debug output without filtering. The + * effect is archived using #TRACE, #DEBUG, #INFO, #WARN and #ERR macros. They + * "return" color_ostream object or reference that can be then used normally for + * either printf or stream style debug output. + * + * Internally macros do inline filtering check which allows compiler to have a + * fast path without debug output only checking unlikely condition. But if + * output is enabled then runtime code will jump to debug printing function + * calls. The macro setup code will also print standardized leading part of + * debug string including time stamp, plugin name and debug category name. + * + * \code{.cpp} + * #include "Debug.h" + * DBG_DECLARE(myplugin,init); + * + * DFhackCExport command_result plugin_init ( color_ostream &out, std::vector &commands) + * { + * command_result rv = CR_OK; + * DEBUG(init, out).print("initializing\n") + * if ((rv = initWork()) != CR_OK) { + * ERR(init, out) << "initWork failed with " + * << rv << " error code" << std::endl; + * return rv; + * } + * return rv + * } + * \endcode + * + * The debug print filtering levels can be changed using debugger. Following + * gdb example would automatically setup core/init and core/render to trace + * level when SDL_init is called. + * + * \code{.unparsed} + * break SDL_init + * commands + * silent + * p DFHack::debug::core::debug_init.allowed_ = 0 + * p DFHack::debug::core::debug_render.allowed_ = 0 + * c + * end + * \endcode + * + */ + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + +/*! + * \defgroup debug_branch_prediction Branch prediction helper macros + * Helper macro tells compiler that debug output branch is unlikely and should + * be optimized to cold section of the function. + * \{ + */ +#if __cplusplus >= 202000L || __has_cpp_attribute(likely) +// c++20 will have standard branch prediction hint attributes +#define likely(x) (x) [[likely]] +#define unlikely(x) (x) [[unlikely]] +#elif defined(__GNUC__) +// gcc has builtin functions that give hints to the branch prediction +#define likely(x) (__builtin_expect(!!(x), 1)) +#define unlikely(x) (__builtin_expect(!!(x), 0)) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif +//! \} + +#ifdef NDEBUG +//! Reduce minimum compiled in debug levels if NDEBUG is defined +#define DBG_FILTER DFHack::DebugCategory::LINFO +#else +//! Set default compiled in debug levels to include all prints +#define DBG_FILTER DFHack::DebugCategory::LTRACE +#endif + +/*! + * DebugCategory is used to enable and disable debug messages in runtime. + * Declaration and definition are handled by #DBG_DECLARE and #DBG_DEFINE + * macros. Runtime filtering support is handled by #TRACE, #DEBUG, #INFO, #WARN + * and #ERR macros. + */ +class DFHACK_EXPORT DebugCategory final { +public: + //! type helper to maybe make it easier to convert to std::string_view when + //! c++17 can be required. + using cstring = const char*; + using cstring_ref = const char*; + /*! + * Debug level enum for message filtering + */ + enum level { + LTRACE = 0, + LDEBUG = 1, + LINFO = 2, + LWARNING = 3, + LERROR = 4, + }; + + /*! + * \param plugin the name of plugin the category belongs to + * \param category the name of category + * \param defaultLevel optional default filtering level for the category + */ + constexpr DebugCategory(cstring_ref plugin, + cstring_ref category, + level defaultLevel = LWARNING) noexcept : + plugin_{plugin}, + category_{category}, + allowed_{defaultLevel} + {} + + + DebugCategory(const DebugCategory&) = delete; + DebugCategory(DebugCategory&&) = delete; + DebugCategory& operator=(DebugCategory) = delete; + DebugCategory& operator=(DebugCategory&&) = delete; + + /*! + * Used by debug macros to check if message should be printed. + * + * It is defined in the header to allow compiler inline it and make disabled + * state a fast path without function calls. + * + * \param msgLevel the debug message level the following print belongs to + * \return boolean with true indicating that message should be printed + */ + bool isEnabled(const level msgLevel) const noexcept { + const uint32_t intLevel = static_cast(msgLevel); + // Compile time filtering to allow compiling out debug checks prints + // from binary. + return static_cast(DBG_FILTER) <= intLevel && + // Runtime filtering for debug messages + static_cast(allowed_.load(std::memory_order_relaxed)) <= intLevel; + } + + struct DFHACK_EXPORT ostream_proxy_prefix : public color_ostream_proxy { + ostream_proxy_prefix(const DebugCategory& cat, + color_ostream& target, + DebugCategory::level level); + ~ostream_proxy_prefix() + { + flush(); + } + }; + + /*! + * Fetch a steam object proxy object for output. It also adds standard + * message components like time and plugin and category names to the line. + * + * User must make sure that the line is terminated with a line end. + * + * \param msgLevel Specifies the level which next debug message belongs + * \return color_ostream_proxy that can be used to print the message + * \sa DFHack::Core::getConsole() + */ + ostream_proxy_prefix getStream(const level msgLevel) const + { + return {*this,Core::getInstance().getConsole(),msgLevel}; + } + /*! + * Add standard message components to existing output stream object to begin + * a new message line to an shared buffered object. + * + * \param msgLevel Specifies the level which next debug message belongs + * \param target An output stream object where a debug output is printed + * \return color_ostream reference that was passed as second parameter + */ + ostream_proxy_prefix getStream(const level msgLevel, color_ostream& target) const + { + return {*this,target,msgLevel}; + } + + /*! + * \brief Allow management code to set a new filtering level + * Caller must have locked DebugManager::access_mutex_. + */ + void allowed(level value) noexcept; + //! Query current filtering level + level allowed() const noexcept; + //! Query plugin name + cstring_ref plugin() const noexcept; + //! Query category name + cstring_ref category() const noexcept; +private: + + cstring plugin_; + cstring category_; + std::atomic allowed_; +#if __cplusplus >= 201703L || __cpp_lib_atomic_is_always_lock_free >= 201603 + static_assert(std::atomic::is_always_lock_free, + "std::atomic should be lock free. You are using a very old CPU or code needs to use std::atomic"); +#endif +}; + +/** + * Handle actual registering wrong template parameter generated pointer + * calculation. + */ +class DFHACK_EXPORT DebugRegisterBase { +protected: + DebugRegisterBase(DebugCategory* category); + void unregister(DebugCategory* category); +}; + +/** + * Register DebugCategory to DebugManager + */ +template +class DebugRegister final : public DebugRegisterBase { +public: + DebugRegister() : + DebugRegisterBase{category} + {} + ~DebugRegister() { + unregister(category); + } +}; + +#define DBG_NAME(category) debug_ ## category + + +/*! + * Declares a debug category. There must be only a declaration per category. + * Declaration should be in same plugin where it is used. If same category name + * is used in core and multiple plugins they all are changed with same command + * unless user specifies explicitly plugin name. + * + * Must be used in one translation unit only. + * + * \param plugin the name of plugin where debug category is used + * \param category the name of category + * \param level the initial DebugCategory::level filtering level. + */ +#define DBG_DECLARE(plugin,category, ...) \ + namespace debug { namespace plugin { \ + DebugCategory DBG_NAME(category){#plugin,#category,__VA_ARGS__}; \ + DebugRegister<&DBG_NAME(category)> register_ ## category; \ + } } \ + using debug::plugin::DBG_NAME(category) + +/*! + * Can be used to access a shared DBG_DECLARE category. But may not be used from + * static initializer because translation unit order is undefined. + * + * Can be used in shared headers to gain access to one definition from + * DBG_DECLARE. + * \param plugin The plugin name that must match DBG_DECLARE + * \param category The category name that must matnch DBG_DECLARE + */ +#define DBG_EXTERN(plugin,category) \ + namespace debug { namespace plugin { \ + extern DFHack::DebugCategory DBG_NAME(category); \ + } } \ + using debug::plugin::DBG_NAME(category) + +#define DBG_PRINT(category,pred,level,...) \ + if pred(!DFHack::DBG_NAME(category).isEnabled(level)) \ + ; /* nop fast path when debug category is disabled */ \ + else /* else to allow macro use in if-else branches */ \ + DFHack::DBG_NAME(category).getStream(level, ## __VA_ARGS__) \ +/* end of DBG_PRINT */ + +/*! + * Open a line for trace level debug output if enabled + * + * Preferred category for inside loop debug messages or callbacks/methods called + * multiple times per second. Good example would be render or onUpdate methods. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define TRACE(category, ...) DBG_PRINT(category, likely, \ + DFHack::DebugCategory::LTRACE, ## __VA_ARGS__) + +/*! + * Open a line for debug level debug output if enabled + * + * Preferred place to use it would be commonly called functions that don't fall + * into trace category. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define DEBUG(category, ...) DBG_PRINT(category, likely, \ + DFHack::DebugCategory::LDEBUG, ## __VA_ARGS__) + +/*! + * Open a line for error level debug output if enabled + * + * Important debug messages when some rarely changed state changes. Example + * would be when a debug category filtering level changes. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define INFO(category, ...) DBG_PRINT(category, likely, \ + DFHack::DebugCategory::LINFO, ## __VA_ARGS__) + +/*! + * Open a line for warning level debug output if enabled + * + * Warning category is for recoverable errors. This generally signals that + * something unusual happened but there is code handling the error which should + * allow df continue running without issues. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define WARN(category, ...) DBG_PRINT(category, unlikely, \ + DFHack::DebugCategory::LWARNING, ## __VA_ARGS__) + +/*! + * Open a line for error level error output if enabled + * + * Errors should be printed only for cases where plugin or dfhack can't recover + * from reported error and it requires manual handling from the user. + * + * \param category the debug category + * \param optional the optional second parameter is an existing + * color_ostream_proxy object + * \return color_ostream object that can be used for stream output + */ +#define ERR(category, ...) DBG_PRINT(category, unlikely, \ + DFHack::DebugCategory::LERROR, ## __VA_ARGS__) + +} diff --git a/library/include/DebugManager.h b/library/include/DebugManager.h new file mode 100644 index 000000000..fb4c635fb --- /dev/null +++ b/library/include/DebugManager.h @@ -0,0 +1,111 @@ +/** + Copyright © 2018 Pauli + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any + damages arising from the use of this software. + + Permission is granted to anyone to use this software for any + purpose, including commercial applications, and to alter it and + redistribute it freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. + */ + +#pragma once + +#include "Export.h" +#include "Signal.hpp" + +#include +#include + +namespace DFHack { + +/*! \file DebugManager.h + * Expose an simple interface to runtime debug output filtering. The management + * interface is separated from output interface because output is required in + * many places while management is expected to be required only in a few places. + */ + +class DebugCategory; + +/*! + * \brief Container holding all registered runtime debug categories + * Singleton DebugManager is a minor extension to std::vector that allows signal + * callbacks to be attached from ui code that manages. + * + * To avoid parallel plugin unload causing issues access to DebugManager must be + * protected by mutex. The access mutex will be taken when + * DFHack::DebugCategory::~DebugCategory performs unregister calls to + * DFHack::DebugManager. The mutex will protect from memory disappearing while + * ui code is accessing or changing the runtime state. + * + * Signal emitting happens from a locked contexts. Taking the + * DFHack::DebugManager::access_mutex_ in a signal callback will results to a + * deadlock. + * + * The interface is extremely simple but enough to implement persistent filter + * states and runtime configuration code in a plugin. + */ +class DFHACK_EXPORT DebugManager : public std::vector { +public: + friend class DebugRegisterBase; + + //! access_mutex_ protects all readers and writers to DFHack::DebugManager + std::mutex access_mutex_; + + //! Different signals that all will be routed to + //! DebugManager::categorySignal + enum signalType { + CAT_ADD, + CAT_REMOVE, + CAT_MODIFIED, + }; + + //! type to help access signal features like Connection and BlockGuard + using categorySignal_t = Signal; + + /*! + * Signal object where callbacks can be connected. Connecting to a class + * method can use a lambda wrapper to the capture object pointer and correctly + * call required method. + * + * Signal is internally serialized allowing multiple threads call it + * freely. + */ + categorySignal_t categorySignal; + + //! Get the singleton object + static DebugManager& getInstance() { + static DebugManager instance; + return instance; + } + + //! Prevent copies + DebugManager(const DebugManager&) = delete; + //! Prevent copies + DebugManager(DebugManager&&) = delete; + //! Prevent copies + DebugManager& operator=(DebugManager) = delete; + //! Prevent copies + DebugManager& operator=(DebugManager&&) = delete; +protected: + DebugManager() = default; + + //! Helper for automatic category registering and signaling + void registerCategory(DebugCategory &); + //! Helper for automatic category unregistering and signaling + void unregisterCategory(DebugCategory &); +private: +}; +} diff --git a/library/include/Signal.hpp b/library/include/Signal.hpp new file mode 100644 index 000000000..acc8f4a0a --- /dev/null +++ b/library/include/Signal.hpp @@ -0,0 +1,782 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#ifdef __SSE__ +#include +#endif + +namespace DFHack { + +/*! + * Select inline implementation for Signal members + * This requires careful destruction order where all connection has been + * disconnected before Signal::~Signal() + */ +class signal_inline_tag; +/*! + * Select share_ptr managed implementation for Signal members. + * + * If Connection holding object may be deleted without full serialization + * between disconnect and signal emit the holding object must be managed by + * shared_ptr and derive from ConnectedBase. It will also have to pass the + * std::shared_ptr to connect. + + * It uses two way std::weak_ptr reference to guarantee destruction of either + * object doesn't happen when call is made to them. + * + * It is still possible to get a callback call after manual disconnect from + * outside destructor. But without destruction risk the disconnect race can be + * handled by slot implementation side. + */ +class signal_shared_tag; + +/** + * Used for signal_shared_tag holders that may race with destructor triggered + * disconnect and emit from Signal. + */ +class ConnectedBase { +}; + +template +class Signal; + +namespace details { + +template +struct SignalImpl; + +template +struct selectImpl; + +//! Manage callback states in thread safe manner +template +class CallbackHolderImpl; + +template +struct CallbackHolderBase { + using Callback = std::function; + + CallbackHolderBase(const Callback& cb) : + cb_{cb}, + state_{} + {} + + //! Block the connection + void block() noexcept + { + state_ += blocked; + } + + //! Unblock the connection + void unblock() noexcept + { + state_ -= blocked; + } + + //! Check if connection is deleted + bool erased() const noexcept + { + return state_ & deleted; + } + + //! Check if connection is still active (not blocked or erased) + operator bool() const noexcept + { + return !(state_ & ~inCall); + } + +protected: + //! Immutable callback object + const Callback cb_; + using state_t = unsigned; + //! Single shared state as a bitfield to simplify synchronization + //! between state changes. + std::atomic state_; + static constexpr state_t deleted = 0x1 << (sizeof(state_t)*CHAR_BIT - 1); + static constexpr state_t inCall = deleted >> (sizeof(state_t)*CHAR_BIT/2); + static constexpr state_t blocked = 0x1; + static constexpr state_t blockedMask = inCall - 1; + static constexpr state_t inCallMask = (deleted - 1) ^ blockedMask; +}; + +template +class CallbackHolderImpl : + public CallbackHolderBase { + using parent_t = CallbackHolderBase; +public: + using Callback = typename parent_t::Callback; +private: + using state_t = typename parent_t::state_t; + //! Make sure callback pointed object doesn't disappear under us + //! while we call it. + struct CallGuard { + + //! Prevent copies but allow copy elision + CallGuard(const CallGuard&); + + //! Allow implicit conversion to callback for simply syntax + const Callback& operator*() const noexcept + { + return holder_->cb_; + } + + operator bool() const noexcept + { + return *holder_; + } + + //! Mark call not to be called any more + ~CallGuard() { + holder_->state_ -= parent_t::inCall; + } + private: + //! Reference to the connection + CallbackHolderImpl* holder_; + + //! Mark call to be in process + CallGuard(CallbackHolderImpl* holder) : + holder_{holder} + { + holder_->state_ += parent_t::inCall; + } + //! Only allow construction from the CallbackHolderImpl::prepareCall + friend class CallbackHolderImpl; + }; +public: + //! Construct the callback state for a callback + CallbackHolderImpl(const Callback& cb) : + parent_t{cb} + {} + + /*! + * Data race free disconnection for the connection. It spins until + * no more callers to wait. Spinning should be problem as callbacks + * are expected to be simple and fast to execute. + * + * Must not be called from withing callback! + * + * \todo Maybe use monitor instruction to avoid busy wait and call + * std::thread::yield() if wait is longer than expected. + */ + void erase() noexcept + { + state_t oldstate; + state_t newstate; + /** Spin until no callers to this callback */ +spin: + while ((oldstate = parent_t::state_) & parent_t::inCallMask) { + // pause would be portable to all old processors but there + // isn't portable way to generate it without SSE header. +#ifdef __SSE__ + _mm_pause(); +#endif + } + do { + if (oldstate & parent_t::inCallMask) + goto spin; + newstate = oldstate | parent_t::deleted; + } while(!parent_t::state_.compare_exchange_weak(oldstate, newstate)); + } + + //! Return RAII CallGuard to protect race between callback and + //! disconnect. + CallGuard prepareCall() + { + return {this}; + } +}; + +template +class CallbackHolderImpl : + public CallbackHolderBase { + using parent_t = CallbackHolderBase; +public: + using Callback = typename parent_t::Callback; +private: + using state_t = typename parent_t::state_t; + //! Make sure callback pointed object doesn't disappear under us + //! while we call it. + struct CallGuard { + + //! Prevent copies but allow copy elision + CallGuard(const CallGuard&); + + //! Allow implicit conversion to callback for simply syntax + const Callback& operator*() const noexcept + { + return holder_->cb_; + } + + operator bool() const noexcept + { + // If this is not marked erased then weak_ref->lock succeeded or + // the slot isn't managed by shared_ptr + return *holder_; + } + + private: + //! Reference to the connection + CallbackHolderImpl* holder_; + std::shared_ptr strong_ref_; + + //! Mark call to be in process + CallGuard(CallbackHolderImpl* holder) : + holder_{holder}, + strong_ref_{holder->weak_ref_.lock()} + { + } + //! Only allow construction from the CallbackHolderImpl::prepareCall + friend class CallbackHolderImpl; + }; + + std::weak_ptr weak_ref_; + friend CallGuard; +public: + //! Construct the callback state for an automatically synchronized object + CallbackHolderImpl(const Callback& cb, + std::shared_ptr& ref) : + parent_t{cb}, + weak_ref_{ref} + {} + + //! Construct the callback state for an externally synchronized object + CallbackHolderImpl(const Callback& cb) : + parent_t{cb}, + weak_ref_{} + {} + + /*! + * erase from destructor can't happen while we are in call because + */ + void erase() noexcept + { + parent_t::state_ |= parent_t::deleted; + } + + //! Return RAII CallGuard to protect race between callback and + //! disconnect. + CallGuard prepareCall() + { + return {this}; + } +}; + +template +struct SignalImpl : public selectImpl::parent_t { +protected: + using select_t = selectImpl; + using parent_t = typename select_t::parent_t; +public: + using CallbackHolder = CallbackHolderImpl; + using Callback = typename CallbackHolder::Callback; + + //! The container type used to store callbacks + using CallbackContainer = std::list; + struct BlockGuard; + + //! Simple connection class that is required to disconnect from the + //! signal. + struct Connection { + //! Construct a default Connection object but using it will result + //! to undefined behavior unless proper connection is assigned to it + Connection() = default; + + Connection(Connection&& o) : + iter_{o.iter_}, + signal_{} + { + std::swap(signal_, o.signal_); + } + + Connection& operator=(Connection&& o) + { + disconnect(); + iter_ = o.iter_; + std::swap(signal_, o.signal_); + return *this; + } + + Connection(const Connection&) = delete; + Connection& operator=(const Connection&) = delete; + + //! Disconnect from signal + void disconnect() + { + auto s = select_t::lock(signal_); + if (!s) + return; + + s->disconnect(*this); + } + + ~Connection() + { + disconnect(); + } + private: + //! Block the connection temporary + void block() + { + auto s = select_t::lock(signal_); + if (!s) + return; + iter_->block(); + } + + //! Restore blocked connection + void unblock() + { + auto s = select_t::lock(signal_); + if (!s) + return; + iter_->unblock(); + } + + //! Construct connection object + Connection(const typename CallbackContainer::iterator &iter, + typename select_t::weak_ptr ptr) : + iter_{iter}, + signal_{ptr} + {} + + //! std::list iterator that is used to access the callback and allow + //! removal from the list. + typename CallbackContainer::iterator iter_; + //! Reference to signal object + typename select_t::weak_ptr signal_; + friend SignalImpl; + friend BlockGuard; + }; + + /*! + * BlockGuard allows temporary RAII guard managed blocking of a + * connection object. + */ + struct BlockGuard { + /*! + * Block a connection that belongs to signal + * \param connection The connection that will be temporary blocked + */ + BlockGuard(Connection& connection) : + blocked_{&connection} + { + connection.block(); + } + + /*! + * Unblock the temporary blocked connection + */ + ~BlockGuard() + { + blocked_->unblock(); + } + + //! Prevent copies but allow copy elision + BlockGuard(const BlockGuard&); + private: + Connection* blocked_; + }; + + Connection connect(const Callback& f) + { + std::lock_guard lock(access_); + auto iter = callbacks_.emplace(callbacks_.begin(), f); + return {iter, parent_t::shared_from_this()}; + } + + Connection connect(std::shared_ptr c, const Callback& f) + { + std::lock_guard lock(access_); + auto iter = callbacks_.emplace(callbacks_.begin(), f, c); + return {iter, parent_t::shared_from_this()}; + } + + void disconnect(Connection& connection) { + std::lock_guard lock(access_); + if (recursion_) { + deleted_ = true; + connection.iter_->erase(); + } else { + callbacks_.erase(connection.iter_); + } + select_t::reset(connection.signal_); + } + + template + void operator()(Combiner &combiner, Args&&... arg) + { + std::unique_lock lock(access_); + struct RecursionGuard { + SignalImpl* signal_; + std::unique_lock* lock_; + //! Increment access count to make sure disconnect doesn't erase + RecursionGuard(SignalImpl *signal, std::unique_lock* lock) : + signal_{signal}, + lock_{lock} + { + ++signal_->recursion_; + } + + /*! + * Clean up deleted functions in data race free and exception + * safe manner. + */ + ~RecursionGuard() + { + lock_->lock(); + if (--signal_->recursion_ == 0 && signal_->deleted_) { + for (auto iter = signal_->callbacks_.begin(); iter != signal_->callbacks_.end();) { + if (iter->erased()) + iter = signal_->callbacks_.erase(iter); + else + ++iter; + } + signal_->deleted_ = false; + } + } + + } guard{this, &lock}; + // Call begin in locked context to allow data race free iteration + // even if there is parallel inserts to the begin after unlocking. + auto iter = callbacks_.begin(); + lock.unlock(); + for (; iter != callbacks_.end(); ++iter) { + // Quickly skip blocked calls without memory writes + if (!*iter) + continue; + // Protect connection from deletion while we are about to call + // it. + auto cb = iter->prepareCall(); + if (cb) + combiner(*cb, std::forward(arg)...); + } + } + + void operator()(Args&&... arg) + { + auto combiner = [](const Callback& cb, Args&&... arg2) + { + cb(std::forward(arg2)...); + }; + (*this)(combiner,std::forward(arg)...); + } + + ~SignalImpl() { + // Check that callbacks are empty. If this triggers then signal may + // have to be extended to allow automatic disconnection of active + // connections in the destructor. + if (std::is_same::value) + assert(callbacks_.empty() && "It is very likely that this signal should use signal_shared_tag"); + } + + //! Simplify access to pimpl when it is inline + SignalImpl* operator->() { + return this; + } + SignalImpl& operator*() { + return *this; + } + + SignalImpl() = default; +private: + SignalImpl(const SignalImpl&) : + SignalImpl{} + {} + std::mutex access_; + CallbackContainer callbacks_; + int recursion_; + bool deleted_; + friend Signal; +}; + +template +struct selectImpl { + using impl_t = SignalImpl; + using interface_t = Signal; + using type = impl_t; + using weak_ptr = impl_t*; + struct ptr_from_this { + weak_ptr shared_from_this() + { + return static_cast(this); + } + }; + using parent_t = ptr_from_this; + + selectImpl() = default; + + // Disallow copies for inline version. + selectImpl(const selectImpl&) = delete; + selectImpl(selectImpl&&) = delete; + selectImpl& operator=(const selectImpl&) = delete; + selectImpl& operator=(selectImpl&&) = delete; + + + static type make() { + return {}; + } + + static void reset(weak_ptr& ptr) { + ptr = nullptr; + } + + static weak_ptr lock(weak_ptr& ptr) { + return ptr; + } + + static weak_ptr get(interface_t& signal) { + return &signal.pimpl; + } +}; + +template +struct selectImpl { + using impl_t = SignalImpl; + using interface_t = Signal; + using type = std::shared_ptr; + using weak_ptr = std::weak_ptr; + using parent_t = std::enable_shared_from_this; + + // Allow copies for shared version + + static type make() { + return std::make_shared>(); + } + + static void reset(weak_ptr& ptr) { + ptr.reset(); + } + + static type lock(weak_ptr& ptr) { + return ptr.lock(); + } + + static weak_ptr get(interface_t& signal) { + return signal.pimpl; + } +}; +} + +/*! + * As I couldn't figure out which signal library would be a good. Too bad all + * signal libraries seem to be either heavy with unnecessary features or written + * before C++11/14 have become useable targets. That seems to indicate everyone + * is now building signal system with standard components. + * + * Implementation and interface is build around std::function holding delegates + * to a function pointer or a functor. One can put there example lambda function + * that captures this pointer from connect side. The lambda function then calls + * the slot method of object correctly. + * + * It is fairly simple to change the signal signature to directly call methods + * but internally that std::function becomes more complex. The pointer to + * member function is problematic because multiple inheritance requires + * adjustments to this. The lambda capture approach should be easy to use while + * letting compiler optimize method call in the callee side. + * + * DFHack::Signal::Connection is an connection handle. The handle can be used to + * disconnect and block a callback. Connection destructor will automatically + * disconnect from the signal. + * + * DFHack::Signal::BlockGuard is an automatic blocked callback guard object. It + * prevents any signals from calling the slot as long the BlockGuard object is + * alive. Internally it replaces the callback with an empty callback and stores + * the real callback in a member variable. Destructor then puts back the real + * callback. This allows easily recursive BlockGuard work correctly because only + * the first BlockGuard has the real callback. + * + * signal_inline_tag requires careful destruction order where all connection are + * disconnected before signal destruction. The implementation is specifically + * targeting places like static and singleton variables and widget hierarchies. + * It provides data race free connect, disconnect and emit operations. + * + * signal_shared_tag allows a bit more freedom when destroying the Signal. It + * adds data race safety between Connection, BlockGuard and destructor. If + * multiple callers need access to Signal with potential of destruction of + * original owner then callers can use Signal copy constructor to take a strong + * reference managed by shared_ptr or weak_ptr with Signal::weak_from_this(). + * weak_from_this returns an object that forwards call directly to + * implementation when the shared_ptr is created using Signal::lock + * + * \param RT return type is derived from a single signature template argument + * \param Args Variable argument type list that is derived from a signature + * template argument. + * \param tag The tag type which selects between shared_ptr managed pimpl and + * inline member variables. + */ +template +class Signal : protected details::selectImpl { +public: + //! Type of callable that can be connected to the signal. + using Callback = std::function; + +protected: + using select_t = details::selectImpl; + using CallbackContainer = typename select_t::impl_t::CallbackContainer; +public: + using weak_ptr = typename select_t::weak_ptr; + + /*! + * Simple connection class that is required to disconnect from the + * signal. + * \sa SignalImpl::Connection + */ + using Connection = typename select_t::impl_t::Connection; + /*! + * BlockGuard allows temporary RAII guard managed blocking of a + * connection object. + * \sa SignalImpl::BlockGuard + */ + using BlockGuard = typename select_t::impl_t::BlockGuard; + + /*! + * Connect a callback function to the signal + * + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + * + * \param f callable that will connected to the signal + * \return connection handle that can be used to disconnect it + */ + Connection connect(const Callback& f) + { + return pimpl->connect(f); + } + + /*! + * Thread safe connect variant connection and Connected object destruction + * can't race with emit from different threads. + * + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + */ + Connection connect(std::shared_ptr c, const Callback& f) + { + static_assert(std::is_same::value, + "Race free destruction is only possible with signal_shared_tag"); + return pimpl->connect(c, f); + } + + /*! + * Disconnection a callback from slots + * + * signal_inline_tag: + * This may not be called if the callback has been called in same + * thread. If callback should trigger destruction an object then + * deletion must use deferred. This rule prevents issues if other thread + * are trying to call the callback when disconnecting. + * + * signal_shared_tag: + * disconnect can be freely called from anywhere as long as caller holds a + * strong reference to the Signal. Strong reference can be obtained by using + * Connection::disconnect, Signal copy constructor to have a copy of signal + * or weak_ptr from weak_from_this() passed to Signal::lock(). + * + * \param connection the object returned from DFHack::Signal::connect + */ + void disconnect(Connection& connection) + { + pimpl->disconnect(connection); + } + + /*! + * Call all connected callbacks using passed arguments. + * + * signal_inline_tag: + * Must not call operator() from callbacks. + * Must not disconnect called callback from inside callback. Solution often + * is to set just atomic state variables in callback and do actual + * processing including deletion in update handler or logic vmethod. + * + * signal_shared_tag: + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + * Safe to disconnect any connection from callbacks. + * + * \param combiner that calls callbacks and processes return values + * \param arg arguments list defined by template parameter signature. + */ + template + void operator()(Combiner &combiner, Args&&... arg) + { + (*pimpl)(combiner, std::forward(arg)...); + } + + /*! + * Call all connected callbacks using passed arguments. + * + * signal_inline_tag: + * Must not call operator() from callbacks. + * Must not disconnect called callback from inside callback. Solution often + * is to set just atomic state variables in callback and do actual + * processing including deletion in update handler or logic vmethod. + * + * signal_shared_tag: + * Safe to call from any context as long as SignalImpl destructor can't be + * called simultaneously from other thread. + * Safe to disconnect any connection from callbacks. + * + * \param arg arguments list defined by template parameter signature. + */ + void operator()(Args&&... arg) + { + (*pimpl)(std::forward(arg)...); + } + + /*! + * Helper to lock the weak_ptr + */ + static typename select_t::type lock(weak_ptr& ptr) + { + return select_t::lock(ptr); + } + + /*! + * Helper to create a weak reference to pimpl which can be used to access + * pimpl directly. If the tag is signal_shared_tag then it provides race + * free access to Signal when using Signal::lock and checking returned + * shared_ptr. + */ + weak_ptr weak_from_this() noexcept + { + return select_t::get(*this); + } + + Signal() : + pimpl{select_t::make()} + {} +private: + typename select_t::type pimpl; + friend select_t; +}; +} diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index d0e7df34b..6eb4ae6ff 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -1,5 +1,7 @@ INCLUDE(Plugins.cmake) +find_package(Threads) + OPTION(BUILD_STONESENSE "Build stonesense (needs a checkout first)." OFF) if(BUILD_STONESENSE) add_subdirectory (stonesense) @@ -107,6 +109,7 @@ if (BUILD_SUPPORTED) DFHACK_PLUGIN(cursecheck cursecheck.cpp) DFHACK_PLUGIN(cxxrandom cxxrandom.cpp LINK_LIBRARIES lua) DFHACK_PLUGIN(deramp deramp.cpp) + DFHACK_PLUGIN(debug debug.cpp LINK_LIBRARIES jsoncpp_lib_static) DFHACK_PLUGIN(dig dig.cpp) DFHACK_PLUGIN(digFlood digFlood.cpp) add_subdirectory(diggingInvaders) diff --git a/plugins/debug.cpp b/plugins/debug.cpp new file mode 100644 index 000000000..9464305dd --- /dev/null +++ b/plugins/debug.cpp @@ -0,0 +1,1139 @@ +/** +Copyright © 2018 Pauli + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. + */ + +#include "Core.h" +#include "PluginManager.h" +#include "DebugManager.h" +#include "Debug.h" +#include "modules/Filesystem.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +DFHACK_PLUGIN("debug"); + +namespace DFHack { +DBG_DECLARE(debug,filter); +DBG_DECLARE(debug,init); +DBG_DECLARE(debug,command); +DBG_DECLARE(debug,ui); +} + +namespace serialization { + +template +struct nvp : public std::pair { + using parent_t = std::pair; + nvp(const char* name, T& value) : + parent_t{name, &value} + {} +}; + +template +nvp make_nvp(const char* name, T& value) { + return {name, value}; +} + +} + +#define NVP(variable) serialization::make_nvp(#variable, variable) + +namespace Json { +template +typename std::enable_if::value, ET>::type +get(Json::Value& ar, const std::string &key, const ET& default_) +{ + return static_cast(as(ar.get(key, static_cast(default_)))); +} +} + +namespace DFHack { namespace debugPlugin { + +using JsonArchive = Json::Value; + +//! Write a named and type value to Json::Value. enable_if makes sure this is +//! only available for types that Json::Value supports directly. +template::value, int>::type = 0> +JsonArchive& operator<<(JsonArchive& ar, const serialization::nvp& target) +{ + ar[target.first] = *target.second; + return ar; +} + +//! Read a named and typed value from Json::Value +template +JsonArchive& operator>>(JsonArchive& ar, const serialization::nvp& target) +{ + *target.second = Json::get(ar, target.first, T{}); + return ar; +} + +/*! + * Default regex flags optimized for matching speed because objects are stored + * long time in memory. + */ +static constexpr auto defaultRegex = + std::regex::optimize | std::regex::nosubs | std::regex::collate; + +static const char* const commandHelp = + " Manage runtime debug print filters.\n" + "\n" + " debugfilter category [ []]\n" + " List categories matching regular expressions.\n" + " debugfilter filter []\n" + " List active filters or show detailed information for a filter.\n" + " debugfilter set [persistent] [ []]\n" + " Set a filter level to categories matching regular expressions.\n" + " debugfilter unset [ ...]\n" + " Unset filters matching space separated list of ids from 'filter'.\n" + " debugfilter disable [ ...]\n" + " Disable filters matching space separated list of ids from 'filter'.\n" + " debugfilter enable [ ...]\n" + " Enable filters matching space separated list of ids from 'filter'.\n" + " debugfilter help []\n" + " Show detailed help for a command or this help.\n"; +static const char* const commandCategory = + " category [ []]\n" + " List categories with optional filters. Parameters are passed to\n" + " std::regex to limit which once are shown. The first regular\n" + " expression is used to match category and the second is used match\n" + " plugin name.\n"; +static const char* const commandSet = + " set [persistent] [ []]\n" + " Set filtering level for matching categories. 'level' must be one of\n" + " trace, debug, info, warning and error. The 'level' parameter sets\n" + " the lowest message level that will be shown. The command doesn't\n" + " allow filters to disable any error messages.\n" + " Default filter life time is until Dwarf Fortress process exists or\n" + " plugin is unloaded. Passing 'persistent' as second parameter tells\n" + " the plugin to store the filter to dfhack-config. Stored filters\n" + " will be active until always when the plugin is loaded. 'unset'\n" + " command can be used to remove persistent filters.\n" + " Filters are applied FIFO order. The latest filter will override any\n" + " older filter that also matches.\n"; +static const char* const commandFilters = + " filter []\n" + " Show the list of active filters. The first column is 'id' which can\n" + " be used to deactivate filters using 'unset' command.\n" + " Filters are printed in same order as applied - the oldest first.\n"; +static const char* const commandUnset = + " unset [ ...]\n" + " 'unset' takes space separated list of filter ids from 'filter'.\n" + " It will reset any matching category back to the default 'warning'\n" + " level or any other still active matching filter level.\n"; +static const char* const commandDisable = + " disable [ ...]\n" + " 'disable' takes space separated list of filter ids from 'filter'.\n" + " It will reset any matching category back to the default 'warning'\n" + " level or any other still active matching filter level.\n" + " 'disable' will print red filters that were already disabled.\n"; +static const char* const commandEnable = + " enable [ ...]\n" + " 'enable' takes space separated list of filter ids from 'filter'.\n" + " It will reset any matching category back to the default 'warning'\n" + " level or any other still active matching filter level.\n" + " 'enable' will print red filters that were already enabled.\n"; +static const char* const commandHelpDetails = + " help []\n" + " Show help for any of subcommands. Without any parameters it shows\n" + " short help for all subcommands.\n"; + +//! Helper type to hold static dispatch table for subcommands +struct CommandDispatch { + //! Store handler function pointer and help message for commands + struct Command { + using handler_t = command_result(*)(color_ostream&,std::vector&); + Command(handler_t handler, const char* help) : + handler_(handler), + help_(help) + {} + + command_result operator()(color_ostream& out, + std::vector& parameters) const noexcept + { + return handler_(out, parameters); + } + + handler_t handler() const noexcept + { return handler_; } + + const char* help() const noexcept + { return help_; } + private: + handler_t handler_; + const char* help_; + }; + using dispatch_t = const std::map; + //! Name to handler function and help message mapping + static dispatch_t dispatch; +}; + +struct LevelName { + static constexpr auto regex_opt = std::regex::icase | std::regex::optimize | std::regex::nosubs; + LevelName(const std::string& name) : + name_{name}, + match_{name, regex_opt} + {} + + bool match(const std::string& value) const + { + return std::regex_match(value, match_); + } + + operator const std::string&() const noexcept + { + return name_; + } + + const std::string& str() const noexcept + { + return name_; + } + + template + std::string operator+(const T& v) const + { + return name_ + v; + } +private: + std::string name_; + std::regex match_; +}; + +std::string operator+(const std::string& a, const LevelName& b) +{ + return a + static_cast(b); +} + +//! List of DebugCategory::level's in human readable form +static const std::array levelNames{ + LevelName{"Trace"}, + LevelName{"Debug"}, + LevelName{"Info"}, + LevelName{"Warning"}, + LevelName{"Error"}, +}; + +/*! + * Filter applies a runtime filter to matching DFHack::DebugCategory 's. + * Filters are stored in DFHack::debugPlugin::FilterManager which applies + * filters to dynamically added categories. The manager also stores and loads + * persistent Filters from the configuration file. + */ +struct Filter { + explicit Filter(DebugCategory::level level, + const std::string& categoryText, + const std::regex& category, + const std::string& pluginText, + const std::regex& plugin, + bool persistent = true, + bool enabled = true) noexcept; + + explicit Filter(DebugCategory::level level, + const std::string& categoryText, + const std::string& pluginText, + bool persistent = true, + bool enabled = true); + + explicit Filter() = default; + + //! Apply the filter to a category if regex's match + void apply(DFHack::DebugCategory& cat) noexcept; + //! Apply the filter to a category that has been already matched + bool applyAgain(DFHack::DebugCategory& cat) const noexcept; + //! Remove the category from matching count if regex's match + //! \return true if regex's matched + bool remove(const DFHack::DebugCategory& cat) noexcept; + + //! Query if filter is enabled + bool enabled() const noexcept {return enabled_;} + //! Set the enable status of filter + void enabled(bool enable) noexcept; + //! Query if filter is persistent + bool persistent() const noexcept {return persistent_;} + //! Query if the filter level + bool level() const noexcept {return level_;} + //! Query number of matching categories + size_t matches() const noexcept {return matches_;} + //! Add matches count for the initial category filter matching + void addMatch() noexcept {++matches_;} + //! Return the category filter text + const std::string& categoryText() const noexcept {return categoryText_;} + //! Return the plugin filter text + const std::string& pluginText() const noexcept {return pluginText_;} + + //! Load Filter from configuration file. Second parameter would be version + //! number if format changes in future to include more fields. Then new + //! fields would have to be loaded conditionally + void load(JsonArchive& ar, const unsigned int) + { + ar >> NVP(categoryText_) + >> NVP(pluginText_) + >> NVP(enabled_) + >> NVP(level_); + TRACE(filter) << "Loading filter cat: " << categoryText_ + << " plug: " << pluginText_ + << " ena " << enabled_ + << " level: " << level_ + << std::endl; + persistent_ = true; + matches_ = 0; + category_ = std::regex{categoryText_}; + plugin_ = std::regex{pluginText_}; + } + + //! Save the Filter to json configuration file + void save(JsonArchive& ar, const unsigned int) const + { + ar << NVP(categoryText_) + << NVP(pluginText_) + << NVP(enabled_) + << NVP(level_); + } + +private: + std::regex category_; + std::regex plugin_; + DebugCategory::level level_; + size_t matches_; + bool persistent_; + bool enabled_; + std::string categoryText_; + std::string pluginText_; +}; + +Filter::Filter(DebugCategory::level level, + const std::string& categoryText, + const std::regex& category, + const std::string& pluginText, + const std::regex& plugin, + bool persistent, + bool enabled) noexcept : + category_{category}, + plugin_{plugin}, + level_{level}, + matches_{0}, + persistent_{persistent}, + enabled_{enabled}, + categoryText_{categoryText}, + pluginText_{pluginText} +{} + +Filter::Filter(DebugCategory::level level, + const std::string& categoryText, + const std::string& pluginText, + bool persistent, + bool enabled) : + Filter{ + level, + categoryText, + std::regex{categoryText, defaultRegex}, + pluginText, + std::regex{pluginText, defaultRegex}, + persistent, + enabled, + } +{} + +void Filter::enabled(bool enable) noexcept +{ + if (enable == enabled_) + return; + enabled_ = enable; +} + +bool Filter::applyAgain(DebugCategory& cat) const noexcept +{ + if (!enabled_) + return false; + if (!std::regex_search(cat.category(), category_)) + return false; + if (!std::regex_search(cat.category(), plugin_)) + return false; + TRACE(filter) << "apply " << cat.plugin() << ':' << cat.category() << " matches '" + << pluginText() << "' '" << categoryText() << '\'' << std::endl; + cat.allowed(level_); + return true; +} + +void Filter::apply(DebugCategory& cat) noexcept +{ + if (applyAgain(cat)) + ++matches_; +} + +bool Filter::remove(const DebugCategory& cat) noexcept +{ + if (!enabled_) + return false; + if (!std::regex_search(cat.category(), category_)) + return false; + if (!std::regex_search(cat.category(), plugin_)) + return false; + TRACE(filter) << "remove " << cat.plugin() << ':' << cat.category() << " matches '" + << pluginText() << "' '" << categoryText() << std::endl; + --matches_; + return true; +} + +/** + * Storage for enabled and disabled filters. It uses ordered map because common + * case is to iterate filters from oldest to the newest. + * + * Data races: any reader and writer must lock + * DFHack::DebugManager::access_lock_. Sharing DebugManager make sense because + * most of functionality related to filters requires locking DebugManager too. + * Remaining functionality can share same lock because they are rarely called + * functions. + */ +struct FilterManager : public std::map +{ + using parent_t = std::map; + //! Current configuration version implemented by the code + constexpr static Json::UInt configVersion{1}; + //! Path to the configuration file + constexpr static const char* configPath{"dfhack-config/runtime-debug.json"}; + + //! Get reference to the singleton + static FilterManager& getInstance() noexcept + { + static FilterManager instance; + return instance; + } + + //! Add a new filter which gets automatically a new id + template + std::pair emplaceNew( Args&&... args ) { + return emplace(std::piecewise_construct, + std::forward_as_tuple(nextId_++), + std::forward_as_tuple(std::forward(args)...)); + } + + //! Load state from the configuration file + DFHack::command_result loadConfig(DFHack::color_ostream& out) noexcept; + //! Save state to the configuration file + DFHack::command_result saveConfig(DFHack::color_ostream& out) const noexcept; + + //! Connect FilterManager to DFHack::DebugManager::categorySignal + void connectTo(DebugManager::categorySignal_t& signal) noexcept; + //! Temporary block DFHack::DebugManager::categorySignal + DebugManager::categorySignal_t::BlockGuard blockSlot() noexcept; + + ~FilterManager(); + + FilterManager(const FilterManager&) = delete; + FilterManager(FilterManager&&) = delete; + FilterManager& operator=(FilterManager) = delete; + FilterManager& operator=(FilterManager&&) = delete; + + void load(JsonArchive& ar) + { + Json::UInt version = -1; + ar >> serialization::make_nvp("configVersion", version); + if (version > configVersion) { + std::stringstream ss; + ss << "The saved config version (" + << version + << ") is newer than code supported version (" + << configVersion + << ")"; + throw std::runtime_error{ss.str()}; + } + ar >> NVP(nextId_); + JsonArchive& children = ar["filters"]; + for (auto iter = children.begin(); iter != children.end(); ++iter) { + Filter child; + child.load(*iter, version); + std::stringstream ss(iter.name()); + size_t id; + ss >> id; + insert(std::make_pair(id, child)); + } + } + + void save(JsonArchive& ar) const + { + JsonArchive children{Json::objectValue}; + for (const auto& filterPair: *this) { + if (!filterPair.second.persistent()) + continue; + std::stringstream ss; + ss << filterPair.first; + filterPair.second.save(children[ss.str()], configVersion); + } + auto configVersion = FilterManager::configVersion; + ar << NVP(configVersion) + << NVP(nextId_); + ar["filters"] = children; + } + +private: + FilterManager() = default; + //! The next integer to use as id to have each with unique number + Json::UInt64 nextId_; + DebugManager::categorySignal_t::Connection connection_; +}; + +FilterManager::~FilterManager() +{ +} + +command_result FilterManager::loadConfig(DFHack::color_ostream& out) noexcept +{ + nextId_ = 1; + if (!Filesystem::isfile(configPath)) + return CR_OK; + try { + DEBUG(command, out) << "Load config from '" << configPath << "'" << std::endl; + JsonArchive archive; + std::ifstream ifs(configPath); + if (!ifs.good()) + throw std::runtime_error{"Failed to open configuration file for reading"}; + ifs >> archive; + load(archive); + } catch(std::exception& e) { + ERR(command, out) << "Serializing filters from '" << configPath << "' failed: " + << e.what() << std::endl; + return CR_FAILURE; + } + return CR_OK; +} + +command_result FilterManager::saveConfig(DFHack::color_ostream& out) const noexcept +{ + try { + DEBUG(command, out) << "Save config to '" << configPath << "'" << std::endl; + JsonArchive archive; + save(archive); + std::ofstream ofs(configPath); + if (!ofs.good()) + throw std::runtime_error{"Failed to open configuration file for writing"}; + ofs << archive; + } catch(std::exception e) { + ERR(command, out) << "Serializing filters to '" << configPath << "' failed: " + << e.what() << std::endl; + return CR_FAILURE; + } + return CR_OK; +} + +void FilterManager::connectTo(DebugManager::categorySignal_t& signal) noexcept +{ + connection_ = signal.connect( + [this](DebugManager::signalType t, DebugCategory& cat) { + TRACE(filter) << "sig type: " << t << ' ' + << cat.plugin() << ':' + << cat.category() << std::endl; + switch (t) { + case DebugManager::CAT_ADD: + for (auto& filterPair: *this) + filterPair.second.apply(cat); + break; + case DebugManager::CAT_REMOVE: + for (auto& filterPair: *this) + filterPair.second.remove(cat); + break; + case DebugManager::CAT_MODIFIED: + break; + } + }); +} + +DebugManager::categorySignal_t::BlockGuard FilterManager::blockSlot() noexcept +{ + TRACE(filter) << "Temporary disable FilterManager::connection_" << std::endl; + return {connection_}; +} + +//! \brief Helper to parse optional regex string safely +static command_result parseRegexParam(std::regex& target, + color_ostream& out, + std::vector& parameters, + size_t pos) +{ + if (parameters.size() <= pos) + return CR_OK; + try { + std::regex temp{parameters[pos], defaultRegex}; + target = std::move(temp); + } catch(std::regex_error e) { + ERR(command,out) << "Failed to parse regular expression '" + << parameters[pos] << "'\n"; + ERR(command,out) << "Parser message: " << e.what() << std::endl; + return CR_WRONG_USAGE; + } + return CR_OK; +} + +/*! + * "Algorithm" to apply category filters based on optional regex parameters. + * \param out The output stream where errors should be written + * \param paramters The list of parameters for the command + * \param pos The position where first optional regular expression parameter is + * \param header The callback after locking DebugManager and before loop + * \param categoryMatch The callback for each matching category + */ +template +static command_result applyCategoryFilters(color_ostream& out, + std::vector& parameters, + size_t pos, Callable1 header, + Callable2 categoryMatch, + Callable3 listComplete) +{ + std::regex pluginRegex{".", defaultRegex}; + std::regex categoryRegex{".", defaultRegex}; + command_result rv = CR_OK; + + DEBUG(command,out) << "applying category filters '" + << (parameters.size() >= pos + 1 ? parameters[pos] : "") + << "' and plugin filter '" + << (parameters.size() >= pos + 2 ? parameters[pos+1] : "") + << '\'' << std::endl; + // Parse parameters + if ((rv = parseRegexParam(pluginRegex, out, parameters, pos)) != CR_OK) + return rv; + if ((rv = parseRegexParam(categoryRegex, out, parameters, pos+1)) != CR_OK) + return rv; + // Lock the manager to have consistent view of categories + auto& manager = DebugManager::getInstance(); + std::lock_guard lock(manager.access_mutex_); + out << std::left; + auto guard = header(manager, categoryRegex, pluginRegex); + for (auto* category: manager) { + DebugCategory::cstring_ref p = category->plugin(); + DebugCategory::cstring_ref c = category->category(); + // Apply filters to the category and plugin names + if (!std::regex_search(c, categoryRegex)) + continue; + if (!std::regex_search(p, pluginRegex)) + continue; + categoryMatch(*category); + } + out << std::flush << std::right; + out.color(COLOR_RESET); + return listComplete(); +} + +static void printCategoryListHeader(color_ostream& out) +{ + // Output the header. + out.color(COLOR_GREEN); + out << std::setw(12) << "Plugin" + << std::setw(12) << "Category" + << std::setw(18) << "Lowest printed" << '\n'; +} + +static void printCategoryListEntry(color_ostream& out, + unsigned& line, + DebugCategory& cat, + DebugCategory::level old = static_cast(-1)) +{ + if ((line & 31) == 0) + printCategoryListHeader(out); + // Output matching categories. + out.color((line++ & 1) == 0 ? COLOR_CYAN : COLOR_LIGHTCYAN); + const std::string& level = (old != static_cast(-1)) ? + levelNames[static_cast(old)] + "->" + + levelNames[static_cast(cat.allowed())] : + levelNames[static_cast(cat.allowed())].str(); + out << std::setw(12) << cat.plugin() + << std::setw(12) << cat.category() + << std::setw(18) << level << '\n'; +} + +//! Handler for debugfilter category +static command_result listCategories(color_ostream& out, + std::vector& parameters) +{ + unsigned line = 0; + return applyCategoryFilters(out, parameters, 1u, + // After parameter parsing + [](DebugManager&, const std::regex&, const std::regex&) { + return 0; + }, + // Per category + [&out, &line](DebugCategory& cat) { + printCategoryListEntry(out, line, cat); + }, + // After list + []() {return CR_OK;}); +} + +//! Type that prints parameter string in center of output stream field +template> +struct center { + using string = std::basic_string; + center(const string& str) : + str_(str) + {} + + const string& str_; +}; + +/*! + * Helper to construct a center object to print fields centered + * \code{.cpp} + * out << std::setw(20) << centered("centered"_s); + * \endcode + */ +template +center centered(const ST& str) +{ + return {str}; +} + +//! c++14 string conversion literal to std::string +std::string operator "" _s(const char* cstr, size_t len) +{ + return {cstr, len}; +} + +//! Output centered string, the stream must be using std::ios::right +//! \sa DFHack::debugPlugin::centered +template +std::basic_ostream& operator<<(std::basic_ostream& os, const center& toCenter) +{ + std::streamsize w = os.width(); + const auto& str = toCenter.str_; + mbstate_t ps{}; + std::streamsize ccount = 0; + auto iter = str.cbegin(); + // Check multibyte character length. It will break when mbrlen find the '\0' + for (;ccount < w; ++ccount) { + const size_t n = std::distance(iter, str.cend()); + using ss_t = std::make_signed::type; + ss_t bytes = mbrlen(&*iter, n, &ps); + if (bytes <= 0) /* Check for errors and null */ + break; + std::advance(iter, bytes); + } + + if (ccount < w) { + // Center the character when there is less than the width + // fillw = w - count + // cw = w - (fillw)/2 = (2w - w + count)/2 + // Extra one for rounding half results up + std::streamsize cw = (w + ccount + 1)/2; + os << std::setw(cw) << str + << std::setw(w - cw) << ""; + } else { + // Truncate characters to the width of field + os.write(&str[0], std::distance(str.begin(), iter)); + // Reset the field width because we wrote the string with write + os << std::setw(0); + } + return os; +} + +static FilterManager::iterator parseFilterId(color_ostream& out, + const std::string& parameter) +{ + unsigned long id = 0; + try { + id = std::stoul(parameter); + } catch(...) { + } + auto& filMan = FilterManager::getInstance(); + auto iter = filMan.find(id); + if (iter == filMan.end()) { + WARN(command,out) << "The optional parameter (" + << parameter << ") must be an filter id." << std::endl; + } + return iter; +} + +static void printFilterListEntry(color_ostream& out, + unsigned line, + color_value lineColor, + size_t id, + const Filter& filter) +{ + if ((line & 31) == 0) { + out.color(COLOR_GREEN); + out << std::setw(4) << "ID" + << std::setw(8) << "enabled" + << std::setw(8) << "persist" + << std::setw(9) << centered("level"_s) + << ' ' + << std::setw(15) << centered("category"_s) + << ' ' + << std::setw(15) << centered("plugin"_s) + << std::setw(8) << "matches" + << '\n'; + } + out.color(lineColor); + out << std::setw(4) << id + << std::setw(8) << centered(filter.enabled() ? "X"_s:""_s) + << std::setw(8) << centered(filter.persistent() ? "X"_s:""_s) + << std::setw(9) << centered(levelNames[filter.level()].str()) + << ' ' + << std::setw(15) << centered(filter.categoryText()) + << ' ' + << std::setw(15) << centered(filter.pluginText()) + << std::setw(8) << filter.matches() + << '\n'; +} + +//! Handler for debugfilter filter +static command_result listFilters(color_ostream& out, + std::vector& parameters) +{ + if (1u < parameters.size()) { + auto& catMan = DebugManager::getInstance(); + std::lock_guard lock(catMan.access_mutex_); + auto iter = parseFilterId(out, parameters[1]); + if (iter == FilterManager::getInstance().end()) + return CR_WRONG_USAGE; + + auto id = iter->first; + Filter& filter = iter->second; + + out << std::left + << std::setw(10) << "ID:" << id << '\n' + << std::setw(10) << "Enabled:" << (filter.enabled() ? "Yes"_s:"No"_s) << '\n' + << std::setw(10) << "Persist:" << (filter.persistent() ? "Yes"_s:"No"_s) << '\n' + << std::setw(10) << "Level:" << levelNames[filter.level()].str() << '\n' + << std::setw(10) << "category:" << filter.categoryText() << '\n' + << std::setw(10) << "plugin:" << filter.pluginText() << '\n' + << std::setw(10) << "matches:" << filter.matches() << '\n' + << std::right + << std::endl; + return CR_OK; + } + auto& catMan = DebugManager::getInstance(); + { + std::lock_guard lock(catMan.access_mutex_); + auto& filMan = FilterManager::getInstance(); + unsigned line = 0; + for (auto& filterPair: filMan) { + size_t id = filterPair.first; + const Filter& filter = filterPair.second; + + color_value c = (line & 1) == 0 ? COLOR_CYAN : COLOR_LIGHTCYAN; + printFilterListEntry(out,line++,c,id,filter); + } + } + out.color(COLOR_RESET); + out.flush(); + return CR_OK; +} + +static const std::string persistent("persistent"); + +//! Handler for debugfilter set +static command_result setFilter(color_ostream& out, + std::vector& parameters) +{ + bool persist = false; + size_t pos = 1u; + if (pos < parameters.size() && + parameters[pos] == persistent) { + pos++; + persist = true; + } + if (pos >= parameters.size()) { + ERR(command,out).print("set requires at least the level parameter\n"); + return CR_WRONG_USAGE; + } + const std::string& level = parameters[pos]; + auto iter = std::find_if(levelNames.begin(), levelNames.end(), + [&level](const LevelName& v) -> bool { + return v.match(level); + }); + if (iter == levelNames.end()) { + ERR(command,out).print("level ('%s') parameter must be one of " + "trace, debug, info, warning, error.\n", + parameters[pos].c_str()); + return CR_WRONG_USAGE; + } + + DebugCategory::level filterLevel = static_cast( + iter - levelNames.begin()); + + unsigned line = 0; + Filter* newFilter = nullptr; + return applyCategoryFilters(out, parameters, pos + 1, + // After parameters parsing + [¶meters, pos, filterLevel,persist,&newFilter](DebugManager&, const std::regex& catRegex, const std::regex& pluginRegex) + { + auto& filMan = FilterManager::getInstance(); + newFilter = &filMan.emplaceNew(filterLevel, + pos+1 < parameters.size()?parameters[pos+1]:".", + catRegex, + pos+2 < parameters.size()?parameters[pos+2]:".", + pluginRegex, + persist).first->second; + return filMan.blockSlot(); + }, + // Per item + [filterLevel, &line, &out, &newFilter](DebugCategory& cat) { + auto old = cat.allowed(); + cat.allowed(filterLevel); + newFilter->addMatch(); + printCategoryListEntry(out, line, cat, old); + }, + // After list + [&out,&persist]() { + if (persist) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +template +static command_result applyFilterIds(color_ostream& out, + std::vector& parameters, + const char* name, + HighlightRed hlRed, + ListComplete listComplete) +{ + if (1u >= parameters.size()) { + ERR(command,out) << name << " requires at least a filter id" << std::endl; + return CR_WRONG_USAGE; + } + command_result rv = CR_OK; + { + auto& catMan = DebugManager::getInstance(); + std::lock_guard lock(catMan.access_mutex_); + auto& filMan = FilterManager::getInstance(); + unsigned line = 0; + for (size_t pos = 1; pos < parameters.size(); ++pos) { + const std::string& p = parameters[pos]; + auto iter = parseFilterId(out, p); + if (iter == filMan.end()) + continue; + color_value c = (line & 1) == 0 ? COLOR_CYAN : COLOR_LIGHTCYAN; + if (hlRed(iter)) + c = COLOR_RED; + printFilterListEntry(out,line++,c,iter->first,iter->second); + } + rv = listComplete(); + } + out.color(COLOR_RESET); + out.flush(); + return rv; +} + +//! Handler for debugfilter disable +static command_result disableFilter(color_ostream& out, + std::vector& parameters) +{ + std::set modified; + bool mustSave = false; + return applyFilterIds(out,parameters,"disable", + // Per item + [&modified,&mustSave](FilterManager::iterator& iter) -> bool { + Filter& filter = iter->second; + bool enabled = filter.enabled(); + if (enabled == false) + return true; + auto& catMan = DebugManager::getInstance(); + for (DebugCategory* cat: catMan) { + if (filter.remove(*cat)) + modified.emplace(cat); + } + filter.enabled(false); + mustSave = mustSave || filter.persistent(); + return false; + }, + // After list + [&modified,&mustSave,&out]() { + for (DebugCategory* cat: modified) { + // Reset filtering back to default + cat->allowed(DebugCategory::LWARNING); + auto& filMan = FilterManager::getInstance(); + // Reapply all remaining filters + for (auto& filterPair: filMan) + filterPair.second.applyAgain(*cat); + } + if (mustSave) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +//! Handler for debugfilter enable +static command_result enableFilter(color_ostream& out, + std::vector& parameters) +{ + std::set modified; + bool mustSave = false; + return applyFilterIds(out,parameters,"enable", + // Per item + [&modified,&mustSave](FilterManager::iterator& iter) -> bool { + Filter& filter = iter->second; + bool enabled = filter.enabled(); + if (enabled == true) + return true; + filter.enabled(true); + auto& catMan = DebugManager::getInstance(); + for (DebugCategory* cat: catMan) { + if (filter.applyAgain(*cat)) { + modified.emplace(cat); + filter.addMatch(); + } + } + mustSave = mustSave || filter.persistent(); + return false; + }, + // After list + [&modified,&mustSave,&out]() { + for (DebugCategory* cat: modified) { + // Reset filtering back to default + cat->allowed(DebugCategory::LWARNING); + auto& filMan = FilterManager::getInstance(); + // Reapply all remaining filters + for (auto& filterPair: filMan) + filterPair.second.applyAgain(*cat); + } + if (mustSave) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +//! Handler for debugfilter unset +static command_result unsetFilter(color_ostream& out, + std::vector& parameters) +{ + std::set modified; + std::vector toErase; + return applyFilterIds(out,parameters,"unset", + // Per item + [&modified, &toErase](FilterManager::iterator& iter) -> bool { + Filter& filter = iter->second; + if (filter.enabled()) { + auto& catMan = DebugManager::getInstance(); + for (DebugCategory* cat: catMan) { + if (filter.remove(*cat)) + modified.emplace(cat); + } + } + toErase.emplace_back(iter); + return false; + }, + // After list + [&modified,&toErase,&out]() { + auto& filMan = FilterManager::getInstance(); + bool mustSave = false; + for (auto iter: toErase) { + mustSave = mustSave || iter->second.persistent(); + filMan.erase(iter); + } + + for (DebugCategory* cat: modified) { + // Reset filtering back to default + cat->allowed(DebugCategory::LWARNING); + // Reapply all remaining filters + for (auto& filterPair: filMan) + filterPair.second.applyAgain(*cat); + } + if (mustSave) + return FilterManager::getInstance().saveConfig(out); + return CR_OK; + }); +} + +using DFHack::debugPlugin::CommandDispatch; + +static command_result printHelp(color_ostream& out, + std::vector& parameters) +{ + const char* help = commandHelp; + auto iter = CommandDispatch::dispatch.end(); + if (1u < parameters.size()) + iter = CommandDispatch::dispatch.find(parameters[1]); + if (iter != CommandDispatch::dispatch.end()) + help = iter->second.help(); + out << help << std::flush; + return CR_OK; +} + +CommandDispatch::dispatch_t CommandDispatch::dispatch { + {"category", {listCategories,commandCategory}}, + {"filter", {listFilters,commandFilters}}, + {"set", {setFilter,commandSet}}, + {"unset", {unsetFilter,commandUnset}}, + {"enable", {enableFilter,commandEnable}}, + {"disable", {disableFilter,commandDisable}}, + {"help", {printHelp,commandHelpDetails}}, +}; + +//! Dispatch command handling to the subcommand or help +static command_result commandDebugFilter(color_ostream& out, + std::vector& parameters) +{ + DEBUG(command,out).print("debugfilter %s, parameter count %zu\n", + parameters.size() > 0 ? parameters[0].c_str() : "", + parameters.size()); + auto handler = printHelp; + auto iter = CommandDispatch::dispatch.end(); + if (0u < parameters.size()) + iter = CommandDispatch::dispatch.find(parameters[0]); + if (iter != CommandDispatch::dispatch.end()) + handler = iter->second.handler(); + return (handler)(out, parameters); +} + +} } /* namespace debug */ + +DFhackCExport DFHack::command_result plugin_init(DFHack::color_ostream& out, + std::vector& commands) +{ + commands.emplace_back( + "debugfilter", + "Manage runtime debug print filters", + DFHack::debugPlugin::commandDebugFilter, + false, + DFHack::debugPlugin::commandHelp); + auto& filMan = DFHack::debugPlugin::FilterManager::getInstance(); + DFHack::command_result rv = DFHack::CR_OK; + if ((rv = filMan.loadConfig(out)) != DFHack::CR_OK) + return rv; + auto& catMan = DFHack::DebugManager::getInstance(); + std::lock_guard lock(catMan.access_mutex_); + for (auto* cat: catMan) { + for (auto& filterPair: filMan) { + DFHack::debugPlugin::Filter& filter = filterPair.second; + filter.apply(*cat); + } + } + INFO(init,out).print("plugin_init with %zu commands, %zu filters and %zu categories\n", + commands.size(), filMan.size(), catMan.size()); + filMan.connectTo(catMan.categorySignal); + return rv; +} + +DFhackCExport DFHack::command_result plugin_shutdown(DFHack::color_ostream& out) +{ + INFO(init,out).print("plugin_shutdown\n"); + return DFHack::CR_OK; +} diff --git a/plugins/devel/CMakeLists.txt b/plugins/devel/CMakeLists.txt index 9478ba805..245bc518c 100644 --- a/plugins/devel/CMakeLists.txt +++ b/plugins/devel/CMakeLists.txt @@ -9,7 +9,7 @@ DFHACK_PLUGIN(counters counters.cpp) DFHACK_PLUGIN(dumpmats dumpmats.cpp) DFHACK_PLUGIN(eventExample eventExample.cpp) DFHACK_PLUGIN(frozen frozen.cpp) -DFHACK_PLUGIN(kittens kittens.cpp) +DFHACK_PLUGIN(kittens kittens.cpp LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) DFHACK_PLUGIN(memview memview.cpp memutils.cpp LINK_LIBRARIES lua) DFHACK_PLUGIN(notes notes.cpp) DFHACK_PLUGIN(onceExample onceExample.cpp) diff --git a/plugins/devel/kittens.cpp b/plugins/devel/kittens.cpp index 5de94ced3..26a2655e8 100644 --- a/plugins/devel/kittens.cpp +++ b/plugins/devel/kittens.cpp @@ -1,12 +1,16 @@ #include #include +#include #include +#include #include "Console.h" #include "Core.h" +#include "Debug.h" #include "Export.h" #include "MiscUtils.h" #include "PluginManager.h" +#include "Signal.hpp" #include "modules/Gui.h" #include "modules/Items.h" @@ -25,6 +29,10 @@ DFHACK_PLUGIN_IS_ENABLED(is_enabled); REQUIRE_GLOBAL(ui); REQUIRE_GLOBAL(world); +namespace DFHack { +DBG_DECLARE(kittens,command); +} + std::atomic shutdown_flag{false}; std::atomic final_flag{true}; std::atomic timering{false}; @@ -42,6 +50,7 @@ command_result trackmenu (color_ostream &out, vector & parameters); command_result trackpos (color_ostream &out, vector & parameters); command_result trackstate (color_ostream &out, vector & parameters); command_result colormods (color_ostream &out, vector & parameters); +command_result sharedsignal (color_ostream &out, vector & parameters); DFhackCExport command_result plugin_init ( color_ostream &out, std::vector &commands) { @@ -51,6 +60,7 @@ DFhackCExport command_result plugin_init ( color_ostream &out, std::vector & parameters) return CR_OK; } +struct Connected; +using shared = std::shared_ptr; +using weak = std::weak_ptr; + +static constexpr std::chrono::microseconds delay{1}; + +template +struct ClearMem : public ConnectedBase { + ~ClearMem() + { + memset(reinterpret_cast(this), 0xDE, sizeof(Derived)); + } +}; + +struct Connected : public ClearMem { + using Sig = Signal; + std::array con; + Sig signal; + weak other; + Sig::weak_ptr other_sig; + color_ostream *out; + int id; + uint32_t count; + uint32_t caller; + alignas(64) std::atomic callee; + Connected() = default; + Connected(int id) : + Connected{} + { + this->id = id; + } + void connect(color_ostream& o, shared& b, size_t pos, uint32_t c) + { + out = &o; + count = c*2; + other = b; + other_sig = b->signal.weak_from_this(); + // Externally synchronized object destruction is only safe to this + // connect. + con[pos] = b->signal.connect( + [this](int) { + uint32_t old = callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(callee != 0xDEDEDEDE); + }); + // Shared object managed object with possibility of destruction while + // other threads calling emit must pass the shared_ptr to connect. + Connected *bptr = b.get(); + b->con[pos] = signal.connect(b, + [bptr](int) { + uint32_t old = bptr->callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(bptr->callee != 0xDEDEDEDE); + }); + } + void reconnect(size_t pos) { + auto b = other.lock(); + if (!b) + return; + // Not required to use Sig::lock because other holds strong reference to + // Signal. But this just shows how weak_ref could be used. + auto sig = Sig::lock(other_sig); + if (!sig) + return; + con[pos] = sig->connect(b, + [this](int) { + uint32_t old = callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(callee != 0xDEDEDEDE); + }); + } + void connect(color_ostream& o, shared& a, shared& b,size_t pos, uint32_t c) + { + out = &o; + count = c; + con[pos] = b->signal.connect(a, + [this](int) { + uint32_t old = callee.fetch_add(1); + assert(old != 0xDEDEDEDE); + std::this_thread::sleep_for(delay); + assert(callee != 0xDEDEDEDE); + }); + } + Connected* operator->() noexcept + { + return this; + } + ~Connected() { + INFO(command,*out).print("Connected %d had %d count. " + "It was caller %d times. " + "It was callee %d times.\n", + id, count, caller, callee.load()); + } +}; + +command_result sharedsignal (color_ostream &out, vector & parameters) +{ + using rng_t = std::linear_congruential_engine; + rng_t rng(std::random_device{}()); + size_t count = 10; + if (0 < parameters.size()) { + std::stringstream ss(parameters[0]); + ss >> count; + DEBUG(command, out) << "Parsed " << count + << " from paramters[0] '" << parameters[0] << '\'' << std::endl; + } + + + std::uniform_int_distribution dis(4096,8192); + out << "Running signal_shared_tag destruction test " + << count << " times" << std::endl; + for (size_t nr = 0; nr < count; ++nr) { + std::array t{}; + // Make an object which destruction is protected by std::thread::join() + Connected external{static_cast(t.size())}; + TRACE(command, out) << "begin " << std::endl; + { + int id = 0; + // Make objects that are automatically protected using weak_ptr + // references that are promoted to shared_ptr when Signal is + // accessed. + std::array c = { + std::make_shared(id++), + std::make_shared(id++), + std::make_shared(id++), + std::make_shared(id++), + }; + assert(t.size() == c.size()); + for (unsigned i = 1; i < c.size(); ++i) { + c[0]->connect(out, c[0], c[i], i - 1, dis(rng)); + c[i]->connect(out, c[i], c[0], 0, dis(rng)); + } + external.connect(out, c[1], 1, dis(rng)); + auto thr = [&out](shared c) { + TRACE(command, out) << "Thread " << c->id << " started." << std::endl; + weak ref = c; + for (;c->caller < c->count; ++c->caller) { + c->signal(c->caller); + } + TRACE(command, out) << "Thread " << c->id << " resets shared." << std::endl; + c.reset(); + while((c = ref.lock())) { + ++c->caller; + c->signal(c->caller); + c.reset(); + std::this_thread::sleep_for(delay*25); + } + }; + for (unsigned i = 0; i < c.size(); ++i) { + TRACE(command, out) << "start thread " << i << std::endl; + t[i] = std::thread{thr, c[i]}; + } + } + TRACE(command, out) << "running " << std::endl; + for (;external->caller < external->count; ++external->caller) { + external->signal(external->caller); + external->reconnect(1); + } + TRACE(command, out) << "join " << std::endl; + for (unsigned i = 0; i < t.size(); ++i) + t[i].join(); + } + return CR_OK; +} + command_result kittens (color_ostream &out, vector & parameters) { if (parameters.size() >= 1)