From fcc49620520bcc48662ed363130c17d662d324e6 Mon Sep 17 00:00:00 2001 From: JapaMala Date: Sat, 2 May 2020 10:58:07 -0700 Subject: [PATCH 01/21] Update stonesense to not crash. --- plugins/stonesense | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/stonesense b/plugins/stonesense index b9fc28836..5b7e7743a 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit b9fc28836f34f7ce1be64de94afd90184a341c7d +Subproject commit 5b7e7743a1372b929acb2ccb75080e139ac11691 From 866660169e89599141264ec3f03691bd4597dc79 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 3 May 2020 17:53:32 -0500 Subject: [PATCH 02/21] Update joblabormapper.cpp Add no-labor mappings for new jobs 241 through 243. --- plugins/labormanager/joblabormapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/labormanager/joblabormapper.cpp b/plugins/labormanager/joblabormapper.cpp index 7c4f80fb7..7b4e77553 100644 --- a/plugins/labormanager/joblabormapper.cpp +++ b/plugins/labormanager/joblabormapper.cpp @@ -896,6 +896,10 @@ JobLaborMapper::JobLaborMapper() job_to_labor_table[df::job_type::PutItemOnDisplay] = jlf_const(df::unit_labor::HAUL_ITEM); job_to_labor_table[df::job_type::StoreItemInLocation] = jlf_no_labor; // StoreItemInLocation + + job_to_labor_table[df::job_type::unk_fake_no_job] = jlf_no_labor; // added for 47.04 - see #1561 + job_to_labor_table[df::job_type::InterrogateSubject] = jlf_no_labor; // added for 47.04 - see #1561 + job_to_labor_table[df::job_type::unk_fake_no_activity] = jlf_no_labor; // added for 47.04 - see #1561 }; df::unit_labor JobLaborMapper::find_job_labor(df::job* j) From 319354f6cd34ad7a68bbbcc90b3c28948322e063 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 3 May 2020 18:16:31 -0500 Subject: [PATCH 03/21] Update labormanager.cpp Add entries for jobs 241-243 to the `dwarf_states` vector as well, derp. --- plugins/labormanager/labormanager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/labormanager/labormanager.cpp b/plugins/labormanager/labormanager.cpp index e416dcf63..accb74b38 100644 --- a/plugins/labormanager/labormanager.cpp +++ b/plugins/labormanager/labormanager.cpp @@ -385,6 +385,9 @@ static const dwarf_state dwarf_states[] = { BUSY /* MakeBracelet */, BUSY /* MakeGem */, BUSY /* PutItemOnDisplay */, + BUSY /* unk_fake_no_job */, + BUSY /* InterrogateSubject */, + BUSY /* unk_fake_no_activity */, }; struct labor_info From 2dfd7ea0d0399b603b5461b4fee34c21079bd651 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 3 May 2020 18:19:49 -0500 Subject: [PATCH 04/21] Update labormanager.cpp again These should probably be OTHER rather than BUSY. --- plugins/labormanager/labormanager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/labormanager/labormanager.cpp b/plugins/labormanager/labormanager.cpp index accb74b38..fad421104 100644 --- a/plugins/labormanager/labormanager.cpp +++ b/plugins/labormanager/labormanager.cpp @@ -385,9 +385,9 @@ static const dwarf_state dwarf_states[] = { BUSY /* MakeBracelet */, BUSY /* MakeGem */, BUSY /* PutItemOnDisplay */, - BUSY /* unk_fake_no_job */, - BUSY /* InterrogateSubject */, - BUSY /* unk_fake_no_activity */, + OTHER /* unk_fake_no_job */, + OTHER /* InterrogateSubject */, + OTHER /* unk_fake_no_activity */, }; struct labor_info From af33f71aa220a5b42dd640a3233f4ae57d75789b Mon Sep 17 00:00:00 2001 From: lethosor Date: Sun, 3 May 2020 22:54:43 -0400 Subject: [PATCH 05/21] Update changelog Includes #1561/#1566 --- docs/changelog.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index d3ed3e51c..1aa7f916e 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -39,6 +39,13 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: # Future +## Fixes +- Fixed a segfault when attempting to start a headless session with a graphical PRINT_MODE setting +- `labormanager`: fixed handling of new jobs in 0.47 + +## Ruby +- Updated ``item_find`` and ``building_find`` to use centralized logic that works on more screens + # 0.47.04-r1 ## Fixes From be517370a69ab6d8b35ffff58c6c5fc5068257ec Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 18:38:54 -0400 Subject: [PATCH 06/21] rendermax: move to std::mutex fast_mutex can deadlock on Linux with optimization enabled --- plugins/rendermax/renderer_light.cpp | 4 ++-- plugins/rendermax/renderer_light.hpp | 24 +++++++++++++----------- plugins/rendermax/renderer_opengl.hpp | 22 ++++++++++------------ plugins/rendermax/rendermax.cpp | 24 ++++++++++-------------- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/plugins/rendermax/renderer_light.cpp b/plugins/rendermax/renderer_light.cpp index dda928afa..d76889220 100644 --- a/plugins/rendermax/renderer_light.cpp +++ b/plugins/rendermax/renderer_light.cpp @@ -270,7 +270,7 @@ rgbf blend(const rgbf& a,const rgbf& b) void lightingEngineViewscreen::clear() { lightMap.assign(lightMap.size(),rgbf(1,1,1)); - tthread::lock_guard guard(myRenderer->dataMutex); + std::lock_guard guard{myRenderer->dataMutex}; if(lightMap.size()==myRenderer->lightGrid.size()) { std::swap(myRenderer->lightGrid,lightMap); @@ -299,7 +299,7 @@ void lightingEngineViewscreen::calculate() } void lightingEngineViewscreen::updateWindow() { - tthread::lock_guard guard(myRenderer->dataMutex); + std::lock_guard guard{myRenderer->dataMutex}; if(lightMap.size()!=myRenderer->lightGrid.size()) { reinit(); diff --git a/plugins/rendermax/renderer_light.hpp b/plugins/rendermax/renderer_light.hpp index b0fb8c5d5..4e779f3fa 100644 --- a/plugins/rendermax/renderer_light.hpp +++ b/plugins/rendermax/renderer_light.hpp @@ -1,11 +1,14 @@ -#ifndef RENDERER_LIGHT_INCLUDED -#define RENDERER_LIGHT_INCLUDED -#include "renderer_opengl.hpp" -#include "Types.h" -#include -#include +#pragma once + #include +#include +#include +#include #include + +#include "renderer_opengl.hpp" +#include "Types.h" + // we are not using boost so let's cheat: template inline void hash_combine(std::size_t & seed, const T & v) @@ -91,7 +94,7 @@ private: } void reinitLightGrid(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; lightGrid.resize(w*h,rgbf(1,1,1)); } void reinitLightGrid() @@ -100,7 +103,7 @@ private: } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector lightGrid; renderer_light(renderer* parent):renderer_wrap(parent),light_adaptation(1) { @@ -108,12 +111,12 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; colorizeTile(x,y); }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) colorizeTile(x,y); @@ -374,4 +377,3 @@ private: }; rgbf blend(const rgbf& a,const rgbf& b); rgbf blendMax(const rgbf& a,const rgbf& b); -#endif diff --git a/plugins/rendermax/renderer_opengl.hpp b/plugins/rendermax/renderer_opengl.hpp index 76a45918c..a0a920e69 100644 --- a/plugins/rendermax/renderer_opengl.hpp +++ b/plugins/rendermax/renderer_opengl.hpp @@ -1,9 +1,7 @@ //original file from https://github.com/Baughn/Dwarf-Fortress--libgraphics- -#ifndef RENDERER_OPENGL_INCLUDED -#define RENDERER_OPENGL_INCLUDED +#pragma once #include "tinythread.h" -#include "fast_mutex.h" #include "Core.h" #include @@ -15,6 +13,7 @@ #include "df/graphic.h" #include #include +#include using df::renderer; using df::init; @@ -281,7 +280,7 @@ private: } void reinitLightGrid(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; lightGrid.resize(w*h); } void reinitLightGrid() @@ -289,7 +288,7 @@ private: reinitLightGrid(df::global::gps->dimy,df::global::gps->dimx); } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector lightGrid; renderer_test(renderer* parent):renderer_wrap(parent) { @@ -297,14 +296,14 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; colorizeTile(x,y); //some sort of mutex or sth? //and then map read }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) colorizeTile(x,y); @@ -366,7 +365,7 @@ private: } void reinitGrids(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; foreOffset.resize(w*h); foreMult.resize(w*h); backOffset.resize(w*h); @@ -377,7 +376,7 @@ private: reinitGrids(df::global::gps->dimy,df::global::gps->dimx); } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector foreOffset,foreMult; std::vector backOffset,backMult; inline int xyToTile(int x, int y) @@ -390,14 +389,14 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; overwriteTile(x,y); //some sort of mutex or sth? //and then map read }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) overwriteTile(x,y); @@ -414,4 +413,3 @@ public: reinitGrids(w,h); } }; -#endif \ No newline at end of file diff --git a/plugins/rendermax/rendermax.cpp b/plugins/rendermax/rendermax.cpp index be8be07ba..17fd6fa95 100644 --- a/plugins/rendermax/rendermax.cpp +++ b/plugins/rendermax/rendermax.cpp @@ -1,27 +1,23 @@ -#include +#include +#include #include +#include -#include - -#include - -#include "Core.h" #include "Console.h" +#include "Core.h" #include "Export.h" +#include "LuaTools.h" #include "PluginManager.h" +#include "VTableInterpose.h" -#include -#include "df/renderer.h" #include "df/enabler.h" +#include "df/renderer.h" +#include "df/viewscreen_dungeonmodest.h" +#include "df/viewscreen_dwarfmodest.h" #include "renderer_opengl.hpp" #include "renderer_light.hpp" -#include "df/viewscreen_dwarfmodest.h" -#include "df/viewscreen_dungeonmodest.h" - -#include - using df::viewscreen_dungeonmodest; using df::viewscreen_dwarfmodest; @@ -367,7 +363,7 @@ static command_result rendermax(color_ostream &out, vector & parameters cur=blue; renderer_test* r=reinterpret_cast(enabler->renderer); - tthread::lock_guard guard(r->dataMutex); + std::lock_guard guard{r->dataMutex}; int h=gps->dimy; int w=gps->dimx; int cx=w/2; From 1dc2d4b6aafd948518c37709a4979a6458674be3 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 18:41:59 -0400 Subject: [PATCH 07/21] Remove tthread::fast_mutex entirely This can deadlock on Linux, per be517370a69ab6d8b35ffff58c6c5fc5068257ec, and rendermax was the only tool that used it --- depends/tthread/fast_mutex.h | 249 ----------------------------------- depends/tthread/tinythread.h | 1 - 2 files changed, 250 deletions(-) delete mode 100644 depends/tthread/fast_mutex.h diff --git a/depends/tthread/fast_mutex.h b/depends/tthread/fast_mutex.h deleted file mode 100644 index a381b2c75..000000000 --- a/depends/tthread/fast_mutex.h +++ /dev/null @@ -1,249 +0,0 @@ -/* -*- mode: c++; tab-width: 2; indent-tabs-mode: nil; -*- -Copyright (c) 2010-2012 Marcus Geelnard - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*/ - -#ifndef _FAST_MUTEX_H_ -#define _FAST_MUTEX_H_ - -/// @file - -// Which platform are we on? -#if !defined(_TTHREAD_PLATFORM_DEFINED_) - #if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__) - #define _TTHREAD_WIN32_ - #else - #define _TTHREAD_POSIX_ - #endif - #define _TTHREAD_PLATFORM_DEFINED_ -#endif - -// Check if we can support the assembly language level implementation (otherwise -// revert to the system API) -#if (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) || \ - (defined(_MSC_VER) && (defined(_M_IX86) /*|| defined(_M_X64)*/)) || \ - (defined(__GNUC__) && (defined(__ppc__))) - #define _FAST_MUTEX_ASM_ -#else - #define _FAST_MUTEX_SYS_ -#endif - -#if defined(_TTHREAD_WIN32_) - #define NOMINMAX - #ifndef WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN - #define __UNDEF_LEAN_AND_MEAN - #endif - #include - #ifdef __UNDEF_LEAN_AND_MEAN - #undef WIN32_LEAN_AND_MEAN - #undef __UNDEF_LEAN_AND_MEAN - #endif -#else - #ifdef _FAST_MUTEX_ASM_ - #include - #else - #include - #endif -#endif - -namespace tthread { - -/// Fast mutex class. -/// This is a mutual exclusion object for synchronizing access to shared -/// memory areas for several threads. It is similar to the tthread::mutex class, -/// but instead of using system level functions, it is implemented as an atomic -/// spin lock with very low CPU overhead. -/// -/// The \c fast_mutex class is NOT compatible with the \c condition_variable -/// class (however, it IS compatible with the \c lock_guard class). It should -/// also be noted that the \c fast_mutex class typically does not provide -/// as accurate thread scheduling as a the standard \c mutex class does. -/// -/// Because of the limitations of the class, it should only be used in -/// situations where the mutex needs to be locked/unlocked very frequently. -/// -/// @note The "fast" version of this class relies on inline assembler language, -/// which is currently only supported for 32/64-bit Intel x86/AMD64 and -/// PowerPC architectures on a limited number of compilers (GNU g++ and MS -/// Visual C++). -/// For other architectures/compilers, system functions are used instead. -class fast_mutex { - public: - /// Constructor. -#if defined(_FAST_MUTEX_ASM_) - fast_mutex() : mLock(0) {} -#else - fast_mutex() - { - #if defined(_TTHREAD_WIN32_) - InitializeCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_init(&mHandle, NULL); - #endif - } -#endif - -#if !defined(_FAST_MUTEX_ASM_) - /// Destructor. - ~fast_mutex() - { - #if defined(_TTHREAD_WIN32_) - DeleteCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_destroy(&mHandle); - #endif - } -#endif - - /// Lock the mutex. - /// The method will block the calling thread until a lock on the mutex can - /// be obtained. The mutex remains locked until \c unlock() is called. - /// @see lock_guard - inline void lock() - { -#if defined(_FAST_MUTEX_ASM_) - bool gotLock; - do { - gotLock = try_lock(); - if(!gotLock) - { - #if defined(_TTHREAD_WIN32_) - Sleep(0); - #elif defined(_TTHREAD_POSIX_) - sched_yield(); - #endif - } - } while(!gotLock); -#else - #if defined(_TTHREAD_WIN32_) - EnterCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_lock(&mHandle); - #endif -#endif - } - - /// Try to lock the mutex. - /// The method will try to lock the mutex. If it fails, the function will - /// return immediately (non-blocking). - /// @return \c true if the lock was acquired, or \c false if the lock could - /// not be acquired. - inline bool try_lock() - { -#if defined(_FAST_MUTEX_ASM_) - int oldLock; - #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - asm volatile ( - "movl $1,%%eax\n\t" - "xchg %%eax,%0\n\t" - "movl %%eax,%1\n\t" - : "=m" (mLock), "=m" (oldLock) - : - : "%eax", "memory" - ); - #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int *ptrLock = &mLock; - __asm { - mov eax,1 - mov ecx,ptrLock - xchg eax,[ecx] - mov oldLock,eax - } - #elif defined(__GNUC__) && (defined(__ppc__)) - int newLock = 1; - asm volatile ( - "\n1:\n\t" - "lwarx %0,0,%1\n\t" - "cmpwi 0,%0,0\n\t" - "bne- 2f\n\t" - "stwcx. %2,0,%1\n\t" - "bne- 1b\n\t" - "isync\n" - "2:\n\t" - : "=&r" (oldLock) - : "r" (&mLock), "r" (newLock) - : "cr0", "memory" - ); - #endif - return (oldLock == 0); -#else - #if defined(_TTHREAD_WIN32_) - return TryEnterCriticalSection(&mHandle) ? true : false; - #elif defined(_TTHREAD_POSIX_) - return (pthread_mutex_trylock(&mHandle) == 0) ? true : false; - #endif -#endif - } - - /// Unlock the mutex. - /// If any threads are waiting for the lock on this mutex, one of them will - /// be unblocked. - inline void unlock() - { -#if defined(_FAST_MUTEX_ASM_) - #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - asm volatile ( - "movl $0,%%eax\n\t" - "xchg %%eax,%0\n\t" - : "=m" (mLock) - : - : "%eax", "memory" - ); - #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int *ptrLock = &mLock; - __asm { - mov eax,0 - mov ecx,ptrLock - xchg eax,[ecx] - } - #elif defined(__GNUC__) && (defined(__ppc__)) - asm volatile ( - "sync\n\t" // Replace with lwsync where possible? - : : : "memory" - ); - mLock = 0; - #endif -#else - #if defined(_TTHREAD_WIN32_) - LeaveCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_unlock(&mHandle); - #endif -#endif - } - - private: -#if defined(_FAST_MUTEX_ASM_) - int mLock; -#else - #if defined(_TTHREAD_WIN32_) - CRITICAL_SECTION mHandle; - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_t mHandle; - #endif -#endif -}; - -} - -#endif // _FAST_MUTEX_H_ - diff --git a/depends/tthread/tinythread.h b/depends/tthread/tinythread.h index d575bae33..b5ff2f077 100644 --- a/depends/tthread/tinythread.h +++ b/depends/tthread/tinythread.h @@ -46,7 +46,6 @@ freely, subject to the following restrictions: /// @li tthread::recursive_mutex /// @li tthread::condition_variable /// @li tthread::lock_guard -/// @li tthread::fast_mutex /// /// @section misc_sec Miscellaneous /// The following special keywords are available: #thread_local. From f3b3cb1b3316dee9f960c3670f94494ead27fb29 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 19:26:49 -0400 Subject: [PATCH 08/21] Remove fast_mutex from CMakeLists --- depends/tthread/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/tthread/CMakeLists.txt b/depends/tthread/CMakeLists.txt index 9e0e85dc5..dfb1d9901 100644 --- a/depends/tthread/CMakeLists.txt +++ b/depends/tthread/CMakeLists.txt @@ -1,5 +1,5 @@ project(dfhack-tinythread) -add_library(dfhack-tinythread STATIC EXCLUDE_FROM_ALL tinythread.cpp tinythread.h fast_mutex.h) +add_library(dfhack-tinythread STATIC EXCLUDE_FROM_ALL tinythread.cpp tinythread.h) if(UNIX) target_link_libraries(dfhack-tinythread pthread) endif() From d0c030c3da62b88d97a05c417fce3d86e80fa7de Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 19:27:17 -0400 Subject: [PATCH 09/21] Update xml (changelog only) --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index b4dfab6d0..0686cfdfc 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b4dfab6d08635da8b01504a289e8cf4e62c859c5 +Subproject commit 0686cfdfc18e9f606f2c296b617d5b3ab6b83889 From 1058e40cb0f42820ba1364477537bc9c668fb85f Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 28 Apr 2020 18:18:28 -0500 Subject: [PATCH 10/21] Enable /bigobj globally on MSVC. This changes the intermediate format used for compiled files from one that can hold up to 65279 (slightly less than 2^16) addressable sections to one that can hold 4294967296 (2^32). Code that uses templates, such as... C++ code... can easily run up against the smaller limit. This does not affect the finished executable, only the intermediate object files. --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a5e8312b..de0c858af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,10 @@ if(MSVC) # a smaller type, and most of the time this is just conversion from 64 to 32 bits # for things like vector sizes, which are never that big anyway. add_definitions("/wd4267") + + # MSVC panics if an object file contains more than 65,279 sections. this + # happens quite frequently with code that uses templates, such as vectors. + add_definitions("/bigobj") endif() # Automatically detect architecture based on Visual Studio generator From 0ffafbde298cc84003ebec0de75013d6baa547e9 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 9 May 2020 18:28:19 -0500 Subject: [PATCH 11/21] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 0686cfdfc..efc5c2610 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 0686cfdfc18e9f606f2c296b617d5b3ab6b83889 +Subproject commit efc5c2610bd0345e9102279be0a9f154e5862050 From fc10ed3df63f610fa4c3756ae121201f3c7a79f4 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 27 Apr 2020 18:26:31 -0500 Subject: [PATCH 12/21] Implementation for df-other-vectors-type in C++. Still needs Lua and Ruby support. --- library/include/BitArray.h | 10 ++++++++++ library/include/DataDefs.h | 1 + 2 files changed, 11 insertions(+) diff --git a/library/include/BitArray.h b/library/include/BitArray.h index cfb859bd0..dfd187363 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -568,4 +568,14 @@ namespace DFHack root->next = link; } }; + + template + struct DfOtherVectors + { + std::vector & operator[](O other_id) + { + auto vectors = reinterpret_cast *>(this); + return vectors[other_id]; + } + }; } diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index fd70b7ba8..3b961df20 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -474,6 +474,7 @@ namespace df using DFHack::BitArray; using DFHack::DfArray; using DFHack::DfLinkedList; + using DFHack::DfOtherVectors; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" From 7cf9624de9d25383ba548d49fa180f28409c8202 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 27 Apr 2020 18:56:48 -0500 Subject: [PATCH 13/21] add assert to make sure other_id array index is in bounds --- library/include/BitArray.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/include/BitArray.h b/library/include/BitArray.h index dfd187363..5858897a3 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -574,6 +574,8 @@ namespace DFHack { std::vector & operator[](O other_id) { + CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(*this) / sizeof(std::vector)); + auto vectors = reinterpret_cast *>(this); return vectors[other_id]; } From 6c20fe4c804fa97018bb1a4374c0d86470d5e26f Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 28 Apr 2020 13:55:34 -0500 Subject: [PATCH 14/21] implement df-other-vectors-type for lua --- library/LuaTypes.cpp | 22 ++++++++++++++++++++++ library/include/DataDefs.h | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index 306538f51..c46d7ce05 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -1422,6 +1422,28 @@ void struct_identity::build_metatable(lua_State *state) SetPtrMethods(state, base+1, base+2); } +void other_vectors_identity::build_metatable(lua_State *state) +{ + int base = lua_gettop(state); + MakeFieldMetatable(state, this, meta_struct_index, meta_struct_newindex); + + EnableMetaField(state, base+2, "_enum"); + + LookupInTable(state, index_enum, &DFHACK_TYPEID_TABLE_TOKEN); + lua_setfield(state, base+1, "_enum"); + + auto keys = &index_enum->getKeys()[-index_enum->getFirstItem()]; + + for (int64_t i = 0; i < index_enum->getLastItem(); i++) + { + lua_getfield(state, base+2, keys[i]); + lua_rawseti(state, base+2, int(i)); + } + + SetStructMethod(state, base+1, base+2, meta_struct_field_reference, "_field"); + SetPtrMethods(state, base+1, base+2); +} + void global_identity::build_metatable(lua_State *state) { int base = lua_gettop(state); diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index 3b961df20..f4eff8510 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -334,6 +334,23 @@ namespace DFHack virtual identity_type type() { return IDTYPE_UNION; } }; + class DFHACK_EXPORT other_vectors_identity : public struct_identity { + enum_identity *index_enum; + + public: + other_vectors_identity(size_t size, TAllocateFn alloc, + compound_identity *scope_parent, const char *dfhack_name, + struct_identity *parent, const struct_field_info *fields, + enum_identity *index_enum) : + struct_identity(size, alloc, scope_parent, dfhack_name, parent, fields), + index_enum(index_enum) + {} + + enum_identity *getIndexEnum() { return index_enum; } + + virtual void build_metatable(lua_State *state); + }; + #ifdef _MSC_VER typedef void *virtual_ptr; #else @@ -465,6 +482,7 @@ namespace df using DFHack::global_identity; using DFHack::struct_identity; using DFHack::union_identity; + using DFHack::other_vectors_identity; using DFHack::struct_field_info; using DFHack::struct_field_info_extra; using DFHack::bitfield_item_info; From 28c34217d86a77b6eafe26aca1c3ff371cb65ecb Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 28 Apr 2020 14:26:33 -0500 Subject: [PATCH 15/21] Ruby implementation of df-other-vectors-type --- plugins/ruby/codegen.pl | 8 ++++++++ plugins/ruby/ruby-autogen-defs.rb | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/plugins/ruby/codegen.pl b/plugins/ruby/codegen.pl index 2a12a1bea..535a49694 100755 --- a/plugins/ruby/codegen.pl +++ b/plugins/ruby/codegen.pl @@ -240,6 +240,12 @@ sub render_global_class { } my $rbparent = ($parent ? rb_ucase($parent) : 'MemHack::Compound'); + my $ienum; + if (($type->getAttribute('ld:subtype') or '') eq 'df-other-vectors-type') + { + $rbparent = 'MemHack::OtherVectors'; + $ienum = rb_ucase($type->getAttribute('index-enum')); + } push @lines_rb, "class $rbname < $rbparent"; indent_rb { my $sz = sizeof($type); @@ -249,6 +255,8 @@ sub render_global_class { push @lines_rb, "rtti_classname :$rtti_name\n" if $rtti_name; + push @lines_rb, "ienum $ienum\n" if $ienum; + render_struct_fields($type); my $vms = $type->findnodes('child::virtual-methods')->[0]; diff --git a/plugins/ruby/ruby-autogen-defs.rb b/plugins/ruby/ruby-autogen-defs.rb index 10727894d..c72ffeb2b 100644 --- a/plugins/ruby/ruby-autogen-defs.rb +++ b/plugins/ruby/ruby-autogen-defs.rb @@ -205,6 +205,22 @@ module DFHack end end + class OtherVectors < Compound + class << self + attr_accessor :_enum + def ienum(enum) + @_enum = enum + end + end + + def [](i) + self.send(self.class._enum.sym(i)) + end + def []=(i, v) + self.send((self.class._enum.sym(i).to_s + "=").to_sym, v) + end + end + class Enum # number -> symbol def self.enum From 94d7f3b3475a6b1ff1c773e81395acb88f660ff7 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:25:42 -0400 Subject: [PATCH 16/21] Update docs on cloning and git stuff --- docs/Compile.rst | 106 ++++++++++++++++++++++++++++-------------- docs/Introduction.rst | 1 + 2 files changed, 71 insertions(+), 36 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index f30775a03..3e7f9701e 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -2,10 +2,16 @@ Compiling DFHack ################ -You don't need to compile DFHack unless you're developing plugins or working on the core. +DFHack builds are available for all supported platforms; see `installing` for +installation instructions. If you are a DFHack end-user, modder, or plan on +writing scripts (not plugins), it is generally recommended (and easier) to use +these builds instead of compiling DFHack from source. -For users, modders, and authors of scripts it's better to download -and `install the latest release instead `. +However, if you are looking to develop plugins, work on the DFHack core, make +complex changes to DF-structures, or anything else that requires compiling +DFHack from source, this document will walk you through the build process. Note +that some steps may be unconventional compared to other projects, so be sure to +pay close attention if this is your first time compiling DFHack. .. contents:: :depth: 2 @@ -14,35 +20,60 @@ and `install the latest release instead `. How to get the code =================== -DFHack doesn't have any kind of system of code snapshots in place, so you will have to -get code from the GitHub repository using Git. How to get Git is described under -the instructions for each platform. - -To get the latest release code (master branch):: +DFHack uses Git for source control; instructions for installing Git can be found +in the platform-specific sections below. The code is hosted on +`GitHub `_, and can be downloaded with:: git clone --recursive https://github.com/DFHack/dfhack cd dfhack -If your version of Git does not support the ``--recursive`` flag, you will need to omit it and run -``git submodule update --init`` after entering the dfhack directory. +If your version of Git does not support the ``--recursive`` flag, you will need +to omit it and run ``git submodule update --init`` after entering the dfhack +directory. -To get the latest development code (develop branch), clone as above and then:: +This will check out the code on the default branch of the GitHub repo, currently +``develop``, which may be unstable. If you want code for the latest stable +release, you can check out the ``master`` branch instead:: - git checkout develop + git checkout master git submodule update -Generally, you should only need to clone DFHack once. - -**Important note regarding submodule update after pulling or changing branches**: - -You must run ``git submodule update`` every time you change branches, such as -when switching between the master and develop branches or vice versa. You also -must run it after pulling any changes to submodules from the DFHack repo. If a -submodule only exists on the newer branch, or if a commit you just pulled -contains a new submodule, you need to run ``git submodule update --init``. -Failure to do this may result in a variety of errors, including ``fatal: -does not exist`` when using Git, errors when building DFHack, and ``not a known -DF version`` when starting DF. +In general, a single DFHack clone is suitable for development - most Git +operations such as switching branches can be done on an existing clone. If you +find yourself cloning DFHack frequently as part of your development process, or +getting stuck on anything else Git-related, feel free to reach out to us for +assistance. + +.. admonition:: A note on submodules + + DFHack uses submodules extensively to manage its subprojects (including the + ``scripts`` folder and DF-structures in ``library/xml``). Failing to keep + submodules in sync when switching between branches can result in build errors + or scripts that don't work. In general, you should always update submodules + whenever you switch between branches in the main DFHack repo with + ``git submodule update``. (If you are working on bleeding-edge DFHack and + have checked out the master branch of some submodules, running ``git pull`` + in those submodules is also an option.) + + Rarely, we add or remove submodules. If there are any changes to the existence + of submodules when you switch between branches, you should run + ``git submodule update --init`` instead (adding ``--init`` to the above + command). + + Some common errors that can arise when failing to update submodules include: + + * ``fatal: does not exist`` when performing Git operations + * Build errors, particularly referring to structures in the ``df::`` namespace + or the ``library/include/df`` folder + * ``Not a known DF version`` when starting DF + * ``Run 'git submodule update --init'`` when running CMake + + Submodules are a particularly confusing feature of Git. The + `Git Book `_ has a + thorough explanation of them (as well as of many other aspects of Git) and + is a recommended resource if you run into any issues. Other DFHack developers + are also able to help with any submodule-related (or Git-related) issues + you may encounter. **More notes**: @@ -51,18 +82,21 @@ DF version`` when starting DF. Contributing to DFHack ====================== -If you want to get involved with the development, create an account on -GitHub, make a clone there and then use that as your remote repository instead. - -We'd love that; join us on IRC_ (#dfhack channel on freenode) for discussion, -and whenever you need help. - -.. _IRC: https://webchat.freenode.net/?channels=dfhack - -(Note: for submodule issues, please see the above instructions first!) - -For lots more details on contributing to DFHack, including pull requests, code format, -and more, please see `contributing-code`. +To contribute to DFHack on GitHub, you will need a GitHub account. Only some +DFHack developers can push directly to the DFHack repositories; we recommend +making a fork of whatever repos you are interested in contributing to, making +changes there, and submitting pull requests. `GitHub's pull request tutorial +`_ +is a good resource for getting started with pull requests (some things to note: +our work mainly happens on the ``develop`` branch, and you will need to use +your own fork, assuming that you don't have write access to the DFHack repos). + +Most development-related discussion happens on IRC or in individual GitHub +issues and pull requests, but there are also other ways to reach out - see +`support` for details. + +For more details on contributing to DFHack, including pull requests, code +format, and more, please see `contributing-code`. Build settings diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 3a36aa538..09e38e52f 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -87,6 +87,7 @@ the console. .. _troubleshooting: +.. _support: Troubleshooting =============== From df06f31137cfc931dcd37e81263e6a55f8586acf Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:25:56 -0400 Subject: [PATCH 17/21] Update forum thread link --- docs/Introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 09e38e52f..22033d528 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -102,7 +102,7 @@ If the search function in this documentation isn't enough and :wiki:`the DF Wiki <>` hasn't helped, try asking in: - the `#dfhack IRC channel on freenode `_ -- the :forums:`Bay12 DFHack thread <139553>` +- the `Bay12 DFHack thread `_ - the `/r/dwarffortress `_ questions thread - the thread for the mod or Starter Pack you're using (if any) From 7a9be264c6464a482418b92a8578d158c1789a0d Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:46:09 -0400 Subject: [PATCH 18/21] Update Linux GCC docs and some ninja notes --- docs/Compile.rst | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index 3e7f9701e..f61d89e2e 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -112,7 +112,8 @@ Generator The ``Ninja`` CMake build generator is the prefered build method on Linux and macOS, instead of ``Unix Makefiles``, which is the default. You can select Ninja by passing ``-G Ninja`` to CMake. Incremental builds using Unix Makefiles can be -much slower than Ninja builds. +much slower than Ninja builds. Note that you will probably need to install +Ninja; see the platform-specific sections for details. :: @@ -178,13 +179,18 @@ DFHack is meant to be installed into an existing DF folder, so get one ready. We assume that any Linux platform will have ``git`` available (though it may need to be installed with your package manager.) -To build DFHack you need GCC version 4.8 or later. GCC 4.8 is easiest to work -with due to avoiding libstdc++ issues (see below), but any version from 4.8 -onwards (including 5.x) will work. +To build DFHack, you need GCC 4.8 or newer. GCC 4.8 has the benefit of avoiding +`libstdc++ compatibility issues `, but can be hard +to obtain on modern distributions, and working around these issues is done +automatically by the ``dfhack`` launcher script. As long as your system-provided +GCC is new enough, it should work. Note that extremely new GCC versions may not +have been used to build DFHack yet, so if you run into issues with these, please +let us know (e.g. by opening a GitHub issue). Before you can build anything, you'll also need ``cmake``. It is advisable to also get ``ccmake`` on distributions that split the cmake package into multiple -parts. +parts. As mentioned above, ``ninja`` is recommended (many distributions call +this package ``ninja-build``). You will need pthread; most systems should have this already. Note that older CMake versions may have trouble detecting pthread, so if you run into @@ -260,29 +266,32 @@ This will show a curses-based interface that lets you set all of the extra options. You can also use a cmake-friendly IDE like KDevelop 4 or the cmake-gui program. +.. _linux-incompatible-libstdcxx: + Incompatible libstdc++ ~~~~~~~~~~~~~~~~~~~~~~ -When compiling dfhack yourself, it builds against your system libstdc++. When -Dwarf Fortress runs, it uses a libstdc++ shipped with the binary, which comes -from GCC 4.8 and is incompatible with code compiled with newer GCC versions. If -you compile DFHack with a GCC version newer than 4.8, you will see an error -message such as:: +When compiling DFHack yourself, it builds against your system libstdc++. When +Dwarf Fortress runs, it uses a libstdc++ shipped in the ``libs`` folder, which +comes from GCC 4.8 and is incompatible with code compiled with newer GCC +versions. As of DFHack 0.42.05-alpha1, the ``dfhack`` launcher script attempts +to fix this by automatically removing the DF-provided libstdc++ on startup. +In rare cases, this may fail and cause errors such as:: ./libs/Dwarf_Fortress: /pathToDF/libs/libstdc++.so.6: version `GLIBCXX_3.4.18' not found (required by ./hack/libdfhack.so) -To fix this you can compile with GCC 4.8 or remove the libstdc++ shipped with +The easiest way to fix this is generally removing the libstdc++ shipped with DF, which causes DF to use your system libstdc++ instead:: cd /path/to/DF/ rm libs/libstdc++.so.6 -Note that distributing binaries compiled with newer GCC versions requires end- -users to delete libstdc++ themselves and have a libstdc++ on their system from -the same GCC version or newer. For this reason, distributing anything compiled -with GCC versions newer than 4.8 is discouraged. In the future we may start -bundling a later libstdc++ as part of the DFHack package, so as to enable -compilation-for-distribution with a GCC newer than 4.8. +Note that distributing binaries compiled with newer GCC versions may result in +the opposite compatibily issue: users with *older* GCC versions may encounter +similar errors. This is why DFHack distributes both GCC 4.8 and GCC 7 builds. If +you are planning on distributing binaries to other users, we recommend using an +older GCC (but still at least 4.8) version if possible. + Mac OS X ======== From d189b91f4997b35fa691dcf2e809ee0860fcba9e Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:58:32 -0400 Subject: [PATCH 19/21] Update macOS -> OS X, tweak a bit and add more anchors --- docs/Compile.rst | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index f61d89e2e..6ebaa8c1f 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -168,6 +168,9 @@ your build folder or by running ``ccmake`` (or another CMake GUI). Most DFHack-specific settings begin with ``BUILD_`` and control which parts of DFHack are built. + +.. _compile-linux: + Linux ===== On Linux, DFHack acts as a library that shadows parts of the SDL API using LD_PRELOAD. @@ -293,13 +296,15 @@ you are planning on distributing binaries to other users, we recommend using an older GCC (but still at least 4.8) version if possible. -Mac OS X -======== -DFHack functions similarly on OS X and Linux, and the majority of the -information above regarding the build process (cmake and ninja) applies here +.. _compile-macos: + +macOS +===== +DFHack functions similarly on macOS and Linux, and the majority of the +information above regarding the build process (CMake and Ninja) applies here as well. -DFHack can officially be built on OS X with GCC 4.8 or 7. Anything newer than 7 +DFHack can officially be built on macOS only with GCC 4.8 or 7. Anything newer than 7 will require you to perform extra steps to get DFHack to run (see `osx-new-gcc-notes`), and your build will likely not be redistributable. @@ -352,7 +357,7 @@ Dependencies and system set-up cleaner, quicker, and smarter. For example, installing MacPort's GCC will install more than twice as many dependencies as Homebrew's will, and all in both 32-bit and 64-bit variants. Homebrew also doesn't require constant use - of sudo. + of ``sudo``. Using `Homebrew `_ (recommended):: @@ -421,9 +426,9 @@ Building export CC=gcc-7 export CXX=g++-7 - etc. + (adjust as needed for different GCC installations) -* Build dfhack:: +* Build DFHack:: mkdir build-osx cd build-osx @@ -433,6 +438,7 @@ Building should be a path to a copy of Dwarf Fortress, of the appropriate version for the DFHack you are building. + .. _compile-windows: Windows @@ -725,9 +731,9 @@ It may be installed in a directory such as ``~/.local/bin/``, so after pip install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``. -Mac OS X --------- -OS X has Python 2.7 installed by default, but it does not have the pip package manager. +macOS +----- +macOS has Python 2.7 installed by default, but it does not have the pip package manager. You can install Homebrew's Python 3, which includes pip, and then install the latest Sphinx using pip:: @@ -739,11 +745,11 @@ Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew:: brew install sphinx-doc -This will install Sphinx for OS X's system Python 2.7, without needing pip. +This will install Sphinx for macOS's system Python 2.7, without needing pip. Either method works; if you plan to use Python for other purposes, it might best to install Homebrew's Python 3 so that you have the latest Python as well as pip. -If not, just installing sphinx-doc for OS X's system Python 2.7 is fine. +If not, just installing sphinx-doc for macOS's system Python 2.7 is fine. Windows @@ -810,7 +816,7 @@ files alphabetically, so all the files you need should be next to each other. It is recommended that you create a build folder and run CMake to verify that you have downloaded everything at this point, assuming your download machine has CMake installed. This involves running a "generate" batch script on Windows, or -a command starting with ``cmake .. -G Ninja`` on Linux and OS X, following the +a command starting with ``cmake .. -G Ninja`` on Linux and macOS, following the instructions in the sections above. CMake should automatically locate files that you placed in ``CMake/downloads``, and use them instead of attempting to download them. From 7dd4b6d9685cbbbb9dda44129967e34b9b5970e8 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 14 May 2020 13:07:11 -0500 Subject: [PATCH 20/21] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index efc5c2610..cbeed8b52 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit efc5c2610bd0345e9102279be0a9f154e5862050 +Subproject commit cbeed8b52482ce749fc2ff13a9db9733931e7da0 From f20446534bb7f39425e102bd70daec46e328004f Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 14 May 2020 14:01:05 -0500 Subject: [PATCH 21/21] fix DfOtherVectors computed size --- library/include/BitArray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/include/BitArray.h b/library/include/BitArray.h index 5858897a3..1f558221a 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -574,7 +574,7 @@ namespace DFHack { std::vector & operator[](O other_id) { - CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(*this) / sizeof(std::vector)); + CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(T) / sizeof(std::vector)); auto vectors = reinterpret_cast *>(this); return vectors[other_id];