From 24cebbda5813450b0196253cc7c26aa773f23622 Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 28 Apr 2020 21:38:47 -0400 Subject: [PATCH 01/97] Fix segfault when setting DFHACK_HEADLESS but not PRINT_MODE:TEXT endwin() is unhappy if initscr() wasn't called --- library/Core.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/library/Core.cpp b/library/Core.cpp index c0a1155c7..a9fe9a69a 100644 --- a/library/Core.cpp +++ b/library/Core.cpp @@ -1692,14 +1692,21 @@ bool Core::Init() if (is_headless) { #ifdef LINUX_BUILD - auto endwin = (int(*)(void))dlsym(RTLD_DEFAULT, "endwin"); - if (endwin) + if (is_text_mode) { - endwin(); + auto endwin = (int(*)(void))dlsym(RTLD_DEFAULT, "endwin"); + if (endwin) + { + endwin(); + } + else + { + cerr << "endwin(): bind failed" << endl; + } } else { - cerr << "endwin(): bind failed" << endl; + cerr << "Headless mode requires PRINT_MODE:TEXT" << endl; } #else cerr << "Headless mode not supported on Windows" << endl; From 1dc59d377275f893d568bc5782107ef3ff609806 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 1 May 2020 22:50:02 -0400 Subject: [PATCH 02/97] Ruby: use core methods for finding selected item and building Similar to c3d566332, the core methods have become more complicated (and are also checked at compile-time). This will give a more consistent experience across scripts. Fixes #1563 --- plugins/ruby/building.rb | 11 +---------- plugins/ruby/item.rb | 36 +----------------------------------- plugins/ruby/ruby.cpp | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 45 deletions(-) diff --git a/plugins/ruby/building.rb b/plugins/ruby/building.rb index 7a85e9314..dae1d455b 100644 --- a/plugins/ruby/building.rb +++ b/plugins/ruby/building.rb @@ -2,16 +2,7 @@ module DFHack class << self def building_find(what=:selected, y=nil, z=nil) if what == :selected - case ui.main.mode - when :LookAround - k = ui_look_list.items[ui_look_cursor] - k.building if k.type == :Building - when :BuildingItems, :QueryBuilding - world.selected_building - when :Zones, :ZonesPenInfo, :ZonesPitInfo, :ZonesHospitalInfo - ui_sidebar_menus.zone.selected - end - + return world.buildings.all.binsearch(df.get_selected_building_id) elsif what.kind_of?(Integer) # search by building.id return world.buildings.all.binsearch(what) if not z diff --git a/plugins/ruby/item.rb b/plugins/ruby/item.rb index 394e545c6..59477c382 100644 --- a/plugins/ruby/item.rb +++ b/plugins/ruby/item.rb @@ -4,41 +4,7 @@ module DFHack # arg similar to unit.rb/unit_find; no arg = 'k' menu def item_find(what=:selected, y=nil, z=nil) if what == :selected - case curview._rtti_classname - when :viewscreen_itemst - if ref = curview.entry_ref[curview.cursor_pos] - ref.item_tg if ref.kind_of?(GeneralRefItem) - else - # not a container - curview.item - end - when :viewscreen_storesst # z/stocks - if curview.in_group_mode == 0 and curview.in_right_list == 1 - curview.items[curview.item_cursor] - end - else - case ui.main.mode - when :LookAround - k = ui_look_list.items[ui_look_cursor] - case k.type - when :Item - k.item - when :Building - # hilight a constructed bed/coffer - mats = k.building.contained_items.find_all { |i| i.use_mode == 2 } - mats[0].item if mats.length == 1 - end - when :BuildingItems - bld = world.selected_building - bld.contained_items[ui_building_item_cursor].item if bld - when :ViewUnits - u = world.units.active[ui_selected_unit] - u.inventory[ui_look_cursor].item if u and u.pos.z == cursor.z and - ui_unit_view_mode.value == :Inventory and u.inventory[ui_look_cursor] - else - ui.follow_item_tg if ui.follow_item != -1 - end - end + return world.items.all.binsearch(df.get_selected_item_id) elsif what.kind_of?(Integer) # search by id return world.items.all.binsearch(what) if not z diff --git a/plugins/ruby/ruby.cpp b/plugins/ruby/ruby.cpp index 240a67cb2..0209806d1 100644 --- a/plugins/ruby/ruby.cpp +++ b/plugins/ruby/ruby.cpp @@ -9,6 +9,8 @@ #include "modules/Gui.h" #include "df/global_objects.h" +#include "df/building.h" +#include "df/item.h" #include "df/unit.h" #include "tinythread.h" @@ -630,6 +632,18 @@ static VALUE rb_dfget_vtable_ptr(VALUE self, VALUE objptr) return rb_uint2inum(*(uintptr_t*)rb_num2ulong(objptr)); } +static VALUE rb_dfget_selected_building_id(VALUE self) +{ + df::building *b = Gui::getAnyBuilding(Core::getTopViewscreen()); + return rb_int2inum(b ? b->id : -1); +} + +static VALUE rb_dfget_selected_item_id(VALUE self) +{ + df::item *i = Gui::getAnyItem(Core::getTopViewscreen()); + return rb_int2inum(i ? i->id : -1); +} + static VALUE rb_dfget_selected_unit_id(VALUE self) { df::unit *u = Gui::getAnyUnit(Core::getTopViewscreen()); @@ -1147,6 +1161,8 @@ static void ruby_bind_dfhack(void) { rb_define_singleton_method(rb_cDFHack, "get_vtable", RUBY_METHOD_FUNC(rb_dfget_vtable), 1); rb_define_singleton_method(rb_cDFHack, "get_rtti_classname", RUBY_METHOD_FUNC(rb_dfget_rtti_classname), 1); rb_define_singleton_method(rb_cDFHack, "get_vtable_ptr", RUBY_METHOD_FUNC(rb_dfget_vtable_ptr), 1); + rb_define_singleton_method(rb_cDFHack, "get_selected_building_id", RUBY_METHOD_FUNC(rb_dfget_selected_building_id), 0); + rb_define_singleton_method(rb_cDFHack, "get_selected_item_id", RUBY_METHOD_FUNC(rb_dfget_selected_item_id), 0); rb_define_singleton_method(rb_cDFHack, "get_selected_unit_id", RUBY_METHOD_FUNC(rb_dfget_selected_unit_id), 0); rb_define_singleton_method(rb_cDFHack, "dfhack_run", RUBY_METHOD_FUNC(rb_dfhack_run), 1); rb_define_singleton_method(rb_cDFHack, "print_str", RUBY_METHOD_FUNC(rb_dfprint_str), 1); From 217535decbc38709cb0870530cf796a9e438b52f Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 1 May 2020 22:52:35 -0400 Subject: [PATCH 03/97] Update xml --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 0792fc020..7941e0ffa 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 0792fc0202fb6a04bfdaa262bc36a3b14c8581e5 +Subproject commit 7941e0ffa7e355cd2363367cdfbb3e085ac4a941 From 947d0db182832c931e26b30eb13daba94219a1cc Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 2 May 2020 01:54:38 -0500 Subject: [PATCH 04/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 7941e0ffa..0231cce29 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 7941e0ffa7e355cd2363367cdfbb3e085ac4a941 +Subproject commit 0231cce29e6014367c0af122aed4f7ba21292b2b From 1c57aa7bb4f103b482c3dabba9bcfcb0bdb47e90 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 2 May 2020 02:01:32 -0500 Subject: [PATCH 05/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 0231cce29..b4dfab6d0 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 0231cce29e6014367c0af122aed4f7ba21292b2b +Subproject commit b4dfab6d08635da8b01504a289e8cf4e62c859c5 From fcc49620520bcc48662ed363130c17d662d324e6 Mon Sep 17 00:00:00 2001 From: JapaMala Date: Sat, 2 May 2020 10:58:07 -0700 Subject: [PATCH 06/97] Update stonesense to not crash. --- plugins/stonesense | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/stonesense b/plugins/stonesense index b9fc28836..5b7e7743a 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit b9fc28836f34f7ce1be64de94afd90184a341c7d +Subproject commit 5b7e7743a1372b929acb2ccb75080e139ac11691 From 60853a543df9e5a90f71f9b315e7169211a126b2 Mon Sep 17 00:00:00 2001 From: JapaMala Date: Sat, 2 May 2020 13:28:28 -0700 Subject: [PATCH 07/97] Remove the site realization building type from the remote reader spec. It has no business being there. --- plugins/proto/RemoteFortressReader.proto | 29 +------------------ .../remotefortressreader.cpp | 2 +- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/plugins/proto/RemoteFortressReader.proto b/plugins/proto/RemoteFortressReader.proto index a54a0397f..864cdba1a 100644 --- a/plugins/proto/RemoteFortressReader.proto +++ b/plugins/proto/RemoteFortressReader.proto @@ -642,33 +642,6 @@ message WorldMap repeated RegionTile region_tiles = 25; } -enum SiteRealizationBuildingType -{ - cottage_plot = 0; - castle_wall = 1; - castle_tower = 2; - castle_courtyard = 3; - house = 4; - temple = 5; - tomb = 6; - shop_house = 7; - warehouse = 8; - market_square = 9; - pasture = 10; - waste = 11; - courtyard = 12; - well = 13; - vault = 14; - great_tower = 15; - trenches = 16; - tree_house = 17; - hillock_house = 18; - mead_hall = 19; - fortress_entrance = 20; - library = 21; - tavern = 22; -} - message SiteRealizationBuildingWall { optional int32 start_x = 1; @@ -702,7 +675,6 @@ message SiteRealizationBuildingTrenches message SiteRealizationBuilding { optional int32 id = 1; - optional SiteRealizationBuildingType type = 2; optional int32 min_x = 3; optional int32 min_y = 4; optional int32 max_x = 5; @@ -711,6 +683,7 @@ message SiteRealizationBuilding optional SiteRealizationBuildingWall wall_info = 8; optional SiteRealizationBuildingTower tower_info = 9; optional SiteRealizationBuildingTrenches trench_info = 10; + optional int32 type = 11; } message RegionTile diff --git a/plugins/remotefortressreader/remotefortressreader.cpp b/plugins/remotefortressreader/remotefortressreader.cpp index e94c3f832..7e654c727 100644 --- a/plugins/remotefortressreader/remotefortressreader.cpp +++ b/plugins/remotefortressreader/remotefortressreader.cpp @@ -2541,7 +2541,7 @@ static void CopyLocalMap(df::world_data * worldData, df::world_region_details* w out_building->set_id(in_building->id); #if DF_VERSION_INT > 34011 - out_building->set_type((SiteRealizationBuildingType)in_building->type); + out_building->set_type(in_building->type); #endif out_building->set_min_x(in_building->min_x - (site_x * 48)); out_building->set_min_y(in_building->min_y - (site_y * 48)); From 63c7b69b5b5a9503b378185bf56b768e1c740d51 Mon Sep 17 00:00:00 2001 From: JapaMala Date: Sat, 2 May 2020 14:38:55 -0700 Subject: [PATCH 08/97] Change item improvement type to bare ints. --- plugins/proto/RemoteFortressReader.proto | 19 +------------------ plugins/remotefortressreader/item_reader.cpp | 2 +- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/plugins/proto/RemoteFortressReader.proto b/plugins/proto/RemoteFortressReader.proto index 864cdba1a..8255f91cd 100644 --- a/plugins/proto/RemoteFortressReader.proto +++ b/plugins/proto/RemoteFortressReader.proto @@ -963,30 +963,13 @@ message Language repeated ShapeDescriptior shapes = 1; } -enum ImprovementType -{ - ART_IMAGE = 0; - COVERED = 1; - RINGS_HANGING = 2; - BANDS = 3; - SPIKES = 4; - ITEMSPECIFIC = 5; - THREAD = 6; - CLOTH = 7; - SEWN_IMAGE = 8; - PAGES = 9; - ILLUSTRATION = 10; - INSTRUMENT_PIECE = 11; - WRITING = 12; -} - message ItemImprovement { optional MatPair material = 1; - optional ImprovementType type = 2; optional int32 shape = 3; optional int32 specific_type= 4; optional ArtImage image = 5; + optional int32 type = 6; } enum ArtImageElementType diff --git a/plugins/remotefortressreader/item_reader.cpp b/plugins/remotefortressreader/item_reader.cpp index 620e06310..56ec9019b 100644 --- a/plugins/remotefortressreader/item_reader.cpp +++ b/plugins/remotefortressreader/item_reader.cpp @@ -450,7 +450,7 @@ void CopyItem(RemoteFortressReader::Item * NetItem, df::item * DfItem) auto netImp = NetItem->add_improvements(); - netImp->set_type((ImprovementType)impType); + netImp->set_type(impType); auto mat = netImp->mutable_material(); mat->set_mat_type(improvement->mat_type); From 866660169e89599141264ec3f03691bd4597dc79 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 3 May 2020 17:53:32 -0500 Subject: [PATCH 09/97] Update joblabormapper.cpp Add no-labor mappings for new jobs 241 through 243. --- plugins/labormanager/joblabormapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/labormanager/joblabormapper.cpp b/plugins/labormanager/joblabormapper.cpp index 7c4f80fb7..7b4e77553 100644 --- a/plugins/labormanager/joblabormapper.cpp +++ b/plugins/labormanager/joblabormapper.cpp @@ -896,6 +896,10 @@ JobLaborMapper::JobLaborMapper() job_to_labor_table[df::job_type::PutItemOnDisplay] = jlf_const(df::unit_labor::HAUL_ITEM); job_to_labor_table[df::job_type::StoreItemInLocation] = jlf_no_labor; // StoreItemInLocation + + job_to_labor_table[df::job_type::unk_fake_no_job] = jlf_no_labor; // added for 47.04 - see #1561 + job_to_labor_table[df::job_type::InterrogateSubject] = jlf_no_labor; // added for 47.04 - see #1561 + job_to_labor_table[df::job_type::unk_fake_no_activity] = jlf_no_labor; // added for 47.04 - see #1561 }; df::unit_labor JobLaborMapper::find_job_labor(df::job* j) From 319354f6cd34ad7a68bbbcc90b3c28948322e063 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 3 May 2020 18:16:31 -0500 Subject: [PATCH 10/97] Update labormanager.cpp Add entries for jobs 241-243 to the `dwarf_states` vector as well, derp. --- plugins/labormanager/labormanager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/labormanager/labormanager.cpp b/plugins/labormanager/labormanager.cpp index e416dcf63..accb74b38 100644 --- a/plugins/labormanager/labormanager.cpp +++ b/plugins/labormanager/labormanager.cpp @@ -385,6 +385,9 @@ static const dwarf_state dwarf_states[] = { BUSY /* MakeBracelet */, BUSY /* MakeGem */, BUSY /* PutItemOnDisplay */, + BUSY /* unk_fake_no_job */, + BUSY /* InterrogateSubject */, + BUSY /* unk_fake_no_activity */, }; struct labor_info From 2dfd7ea0d0399b603b5461b4fee34c21079bd651 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 3 May 2020 18:19:49 -0500 Subject: [PATCH 11/97] Update labormanager.cpp again These should probably be OTHER rather than BUSY. --- plugins/labormanager/labormanager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/labormanager/labormanager.cpp b/plugins/labormanager/labormanager.cpp index accb74b38..fad421104 100644 --- a/plugins/labormanager/labormanager.cpp +++ b/plugins/labormanager/labormanager.cpp @@ -385,9 +385,9 @@ static const dwarf_state dwarf_states[] = { BUSY /* MakeBracelet */, BUSY /* MakeGem */, BUSY /* PutItemOnDisplay */, - BUSY /* unk_fake_no_job */, - BUSY /* InterrogateSubject */, - BUSY /* unk_fake_no_activity */, + OTHER /* unk_fake_no_job */, + OTHER /* InterrogateSubject */, + OTHER /* unk_fake_no_activity */, }; struct labor_info From af33f71aa220a5b42dd640a3233f4ae57d75789b Mon Sep 17 00:00:00 2001 From: lethosor Date: Sun, 3 May 2020 22:54:43 -0400 Subject: [PATCH 12/97] Update changelog Includes #1561/#1566 --- docs/changelog.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index d3ed3e51c..1aa7f916e 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -39,6 +39,13 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: # Future +## Fixes +- Fixed a segfault when attempting to start a headless session with a graphical PRINT_MODE setting +- `labormanager`: fixed handling of new jobs in 0.47 + +## Ruby +- Updated ``item_find`` and ``building_find`` to use centralized logic that works on more screens + # 0.47.04-r1 ## Fixes From be517370a69ab6d8b35ffff58c6c5fc5068257ec Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 18:38:54 -0400 Subject: [PATCH 13/97] rendermax: move to std::mutex fast_mutex can deadlock on Linux with optimization enabled --- plugins/rendermax/renderer_light.cpp | 4 ++-- plugins/rendermax/renderer_light.hpp | 24 +++++++++++++----------- plugins/rendermax/renderer_opengl.hpp | 22 ++++++++++------------ plugins/rendermax/rendermax.cpp | 24 ++++++++++-------------- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/plugins/rendermax/renderer_light.cpp b/plugins/rendermax/renderer_light.cpp index dda928afa..d76889220 100644 --- a/plugins/rendermax/renderer_light.cpp +++ b/plugins/rendermax/renderer_light.cpp @@ -270,7 +270,7 @@ rgbf blend(const rgbf& a,const rgbf& b) void lightingEngineViewscreen::clear() { lightMap.assign(lightMap.size(),rgbf(1,1,1)); - tthread::lock_guard guard(myRenderer->dataMutex); + std::lock_guard guard{myRenderer->dataMutex}; if(lightMap.size()==myRenderer->lightGrid.size()) { std::swap(myRenderer->lightGrid,lightMap); @@ -299,7 +299,7 @@ void lightingEngineViewscreen::calculate() } void lightingEngineViewscreen::updateWindow() { - tthread::lock_guard guard(myRenderer->dataMutex); + std::lock_guard guard{myRenderer->dataMutex}; if(lightMap.size()!=myRenderer->lightGrid.size()) { reinit(); diff --git a/plugins/rendermax/renderer_light.hpp b/plugins/rendermax/renderer_light.hpp index b0fb8c5d5..4e779f3fa 100644 --- a/plugins/rendermax/renderer_light.hpp +++ b/plugins/rendermax/renderer_light.hpp @@ -1,11 +1,14 @@ -#ifndef RENDERER_LIGHT_INCLUDED -#define RENDERER_LIGHT_INCLUDED -#include "renderer_opengl.hpp" -#include "Types.h" -#include -#include +#pragma once + #include +#include +#include +#include #include + +#include "renderer_opengl.hpp" +#include "Types.h" + // we are not using boost so let's cheat: template inline void hash_combine(std::size_t & seed, const T & v) @@ -91,7 +94,7 @@ private: } void reinitLightGrid(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; lightGrid.resize(w*h,rgbf(1,1,1)); } void reinitLightGrid() @@ -100,7 +103,7 @@ private: } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector lightGrid; renderer_light(renderer* parent):renderer_wrap(parent),light_adaptation(1) { @@ -108,12 +111,12 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; colorizeTile(x,y); }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) colorizeTile(x,y); @@ -374,4 +377,3 @@ private: }; rgbf blend(const rgbf& a,const rgbf& b); rgbf blendMax(const rgbf& a,const rgbf& b); -#endif diff --git a/plugins/rendermax/renderer_opengl.hpp b/plugins/rendermax/renderer_opengl.hpp index 76a45918c..a0a920e69 100644 --- a/plugins/rendermax/renderer_opengl.hpp +++ b/plugins/rendermax/renderer_opengl.hpp @@ -1,9 +1,7 @@ //original file from https://github.com/Baughn/Dwarf-Fortress--libgraphics- -#ifndef RENDERER_OPENGL_INCLUDED -#define RENDERER_OPENGL_INCLUDED +#pragma once #include "tinythread.h" -#include "fast_mutex.h" #include "Core.h" #include @@ -15,6 +13,7 @@ #include "df/graphic.h" #include #include +#include using df::renderer; using df::init; @@ -281,7 +280,7 @@ private: } void reinitLightGrid(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; lightGrid.resize(w*h); } void reinitLightGrid() @@ -289,7 +288,7 @@ private: reinitLightGrid(df::global::gps->dimy,df::global::gps->dimx); } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector lightGrid; renderer_test(renderer* parent):renderer_wrap(parent) { @@ -297,14 +296,14 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; colorizeTile(x,y); //some sort of mutex or sth? //and then map read }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) colorizeTile(x,y); @@ -366,7 +365,7 @@ private: } void reinitGrids(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; foreOffset.resize(w*h); foreMult.resize(w*h); backOffset.resize(w*h); @@ -377,7 +376,7 @@ private: reinitGrids(df::global::gps->dimy,df::global::gps->dimx); } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector foreOffset,foreMult; std::vector backOffset,backMult; inline int xyToTile(int x, int y) @@ -390,14 +389,14 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; overwriteTile(x,y); //some sort of mutex or sth? //and then map read }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) overwriteTile(x,y); @@ -414,4 +413,3 @@ public: reinitGrids(w,h); } }; -#endif \ No newline at end of file diff --git a/plugins/rendermax/rendermax.cpp b/plugins/rendermax/rendermax.cpp index be8be07ba..17fd6fa95 100644 --- a/plugins/rendermax/rendermax.cpp +++ b/plugins/rendermax/rendermax.cpp @@ -1,27 +1,23 @@ -#include +#include +#include #include +#include -#include - -#include - -#include "Core.h" #include "Console.h" +#include "Core.h" #include "Export.h" +#include "LuaTools.h" #include "PluginManager.h" +#include "VTableInterpose.h" -#include -#include "df/renderer.h" #include "df/enabler.h" +#include "df/renderer.h" +#include "df/viewscreen_dungeonmodest.h" +#include "df/viewscreen_dwarfmodest.h" #include "renderer_opengl.hpp" #include "renderer_light.hpp" -#include "df/viewscreen_dwarfmodest.h" -#include "df/viewscreen_dungeonmodest.h" - -#include - using df::viewscreen_dungeonmodest; using df::viewscreen_dwarfmodest; @@ -367,7 +363,7 @@ static command_result rendermax(color_ostream &out, vector & parameters cur=blue; renderer_test* r=reinterpret_cast(enabler->renderer); - tthread::lock_guard guard(r->dataMutex); + std::lock_guard guard{r->dataMutex}; int h=gps->dimy; int w=gps->dimx; int cx=w/2; From 1dc2d4b6aafd948518c37709a4979a6458674be3 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 18:41:59 -0400 Subject: [PATCH 14/97] Remove tthread::fast_mutex entirely This can deadlock on Linux, per be517370a69ab6d8b35ffff58c6c5fc5068257ec, and rendermax was the only tool that used it --- depends/tthread/fast_mutex.h | 249 ----------------------------------- depends/tthread/tinythread.h | 1 - 2 files changed, 250 deletions(-) delete mode 100644 depends/tthread/fast_mutex.h diff --git a/depends/tthread/fast_mutex.h b/depends/tthread/fast_mutex.h deleted file mode 100644 index a381b2c75..000000000 --- a/depends/tthread/fast_mutex.h +++ /dev/null @@ -1,249 +0,0 @@ -/* -*- mode: c++; tab-width: 2; indent-tabs-mode: nil; -*- -Copyright (c) 2010-2012 Marcus Geelnard - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*/ - -#ifndef _FAST_MUTEX_H_ -#define _FAST_MUTEX_H_ - -/// @file - -// Which platform are we on? -#if !defined(_TTHREAD_PLATFORM_DEFINED_) - #if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__) - #define _TTHREAD_WIN32_ - #else - #define _TTHREAD_POSIX_ - #endif - #define _TTHREAD_PLATFORM_DEFINED_ -#endif - -// Check if we can support the assembly language level implementation (otherwise -// revert to the system API) -#if (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) || \ - (defined(_MSC_VER) && (defined(_M_IX86) /*|| defined(_M_X64)*/)) || \ - (defined(__GNUC__) && (defined(__ppc__))) - #define _FAST_MUTEX_ASM_ -#else - #define _FAST_MUTEX_SYS_ -#endif - -#if defined(_TTHREAD_WIN32_) - #define NOMINMAX - #ifndef WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN - #define __UNDEF_LEAN_AND_MEAN - #endif - #include - #ifdef __UNDEF_LEAN_AND_MEAN - #undef WIN32_LEAN_AND_MEAN - #undef __UNDEF_LEAN_AND_MEAN - #endif -#else - #ifdef _FAST_MUTEX_ASM_ - #include - #else - #include - #endif -#endif - -namespace tthread { - -/// Fast mutex class. -/// This is a mutual exclusion object for synchronizing access to shared -/// memory areas for several threads. It is similar to the tthread::mutex class, -/// but instead of using system level functions, it is implemented as an atomic -/// spin lock with very low CPU overhead. -/// -/// The \c fast_mutex class is NOT compatible with the \c condition_variable -/// class (however, it IS compatible with the \c lock_guard class). It should -/// also be noted that the \c fast_mutex class typically does not provide -/// as accurate thread scheduling as a the standard \c mutex class does. -/// -/// Because of the limitations of the class, it should only be used in -/// situations where the mutex needs to be locked/unlocked very frequently. -/// -/// @note The "fast" version of this class relies on inline assembler language, -/// which is currently only supported for 32/64-bit Intel x86/AMD64 and -/// PowerPC architectures on a limited number of compilers (GNU g++ and MS -/// Visual C++). -/// For other architectures/compilers, system functions are used instead. -class fast_mutex { - public: - /// Constructor. -#if defined(_FAST_MUTEX_ASM_) - fast_mutex() : mLock(0) {} -#else - fast_mutex() - { - #if defined(_TTHREAD_WIN32_) - InitializeCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_init(&mHandle, NULL); - #endif - } -#endif - -#if !defined(_FAST_MUTEX_ASM_) - /// Destructor. - ~fast_mutex() - { - #if defined(_TTHREAD_WIN32_) - DeleteCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_destroy(&mHandle); - #endif - } -#endif - - /// Lock the mutex. - /// The method will block the calling thread until a lock on the mutex can - /// be obtained. The mutex remains locked until \c unlock() is called. - /// @see lock_guard - inline void lock() - { -#if defined(_FAST_MUTEX_ASM_) - bool gotLock; - do { - gotLock = try_lock(); - if(!gotLock) - { - #if defined(_TTHREAD_WIN32_) - Sleep(0); - #elif defined(_TTHREAD_POSIX_) - sched_yield(); - #endif - } - } while(!gotLock); -#else - #if defined(_TTHREAD_WIN32_) - EnterCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_lock(&mHandle); - #endif -#endif - } - - /// Try to lock the mutex. - /// The method will try to lock the mutex. If it fails, the function will - /// return immediately (non-blocking). - /// @return \c true if the lock was acquired, or \c false if the lock could - /// not be acquired. - inline bool try_lock() - { -#if defined(_FAST_MUTEX_ASM_) - int oldLock; - #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - asm volatile ( - "movl $1,%%eax\n\t" - "xchg %%eax,%0\n\t" - "movl %%eax,%1\n\t" - : "=m" (mLock), "=m" (oldLock) - : - : "%eax", "memory" - ); - #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int *ptrLock = &mLock; - __asm { - mov eax,1 - mov ecx,ptrLock - xchg eax,[ecx] - mov oldLock,eax - } - #elif defined(__GNUC__) && (defined(__ppc__)) - int newLock = 1; - asm volatile ( - "\n1:\n\t" - "lwarx %0,0,%1\n\t" - "cmpwi 0,%0,0\n\t" - "bne- 2f\n\t" - "stwcx. %2,0,%1\n\t" - "bne- 1b\n\t" - "isync\n" - "2:\n\t" - : "=&r" (oldLock) - : "r" (&mLock), "r" (newLock) - : "cr0", "memory" - ); - #endif - return (oldLock == 0); -#else - #if defined(_TTHREAD_WIN32_) - return TryEnterCriticalSection(&mHandle) ? true : false; - #elif defined(_TTHREAD_POSIX_) - return (pthread_mutex_trylock(&mHandle) == 0) ? true : false; - #endif -#endif - } - - /// Unlock the mutex. - /// If any threads are waiting for the lock on this mutex, one of them will - /// be unblocked. - inline void unlock() - { -#if defined(_FAST_MUTEX_ASM_) - #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - asm volatile ( - "movl $0,%%eax\n\t" - "xchg %%eax,%0\n\t" - : "=m" (mLock) - : - : "%eax", "memory" - ); - #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int *ptrLock = &mLock; - __asm { - mov eax,0 - mov ecx,ptrLock - xchg eax,[ecx] - } - #elif defined(__GNUC__) && (defined(__ppc__)) - asm volatile ( - "sync\n\t" // Replace with lwsync where possible? - : : : "memory" - ); - mLock = 0; - #endif -#else - #if defined(_TTHREAD_WIN32_) - LeaveCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_unlock(&mHandle); - #endif -#endif - } - - private: -#if defined(_FAST_MUTEX_ASM_) - int mLock; -#else - #if defined(_TTHREAD_WIN32_) - CRITICAL_SECTION mHandle; - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_t mHandle; - #endif -#endif -}; - -} - -#endif // _FAST_MUTEX_H_ - diff --git a/depends/tthread/tinythread.h b/depends/tthread/tinythread.h index d575bae33..b5ff2f077 100644 --- a/depends/tthread/tinythread.h +++ b/depends/tthread/tinythread.h @@ -46,7 +46,6 @@ freely, subject to the following restrictions: /// @li tthread::recursive_mutex /// @li tthread::condition_variable /// @li tthread::lock_guard -/// @li tthread::fast_mutex /// /// @section misc_sec Miscellaneous /// The following special keywords are available: #thread_local. From f3b3cb1b3316dee9f960c3670f94494ead27fb29 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 19:26:49 -0400 Subject: [PATCH 15/97] Remove fast_mutex from CMakeLists --- depends/tthread/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/tthread/CMakeLists.txt b/depends/tthread/CMakeLists.txt index 9e0e85dc5..dfb1d9901 100644 --- a/depends/tthread/CMakeLists.txt +++ b/depends/tthread/CMakeLists.txt @@ -1,5 +1,5 @@ project(dfhack-tinythread) -add_library(dfhack-tinythread STATIC EXCLUDE_FROM_ALL tinythread.cpp tinythread.h fast_mutex.h) +add_library(dfhack-tinythread STATIC EXCLUDE_FROM_ALL tinythread.cpp tinythread.h) if(UNIX) target_link_libraries(dfhack-tinythread pthread) endif() From d0c030c3da62b88d97a05c417fce3d86e80fa7de Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 4 May 2020 19:27:17 -0400 Subject: [PATCH 16/97] Update xml (changelog only) --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index b4dfab6d0..0686cfdfc 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b4dfab6d08635da8b01504a289e8cf4e62c859c5 +Subproject commit 0686cfdfc18e9f606f2c296b617d5b3ab6b83889 From 1058e40cb0f42820ba1364477537bc9c668fb85f Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 28 Apr 2020 18:18:28 -0500 Subject: [PATCH 17/97] Enable /bigobj globally on MSVC. This changes the intermediate format used for compiled files from one that can hold up to 65279 (slightly less than 2^16) addressable sections to one that can hold 4294967296 (2^32). Code that uses templates, such as... C++ code... can easily run up against the smaller limit. This does not affect the finished executable, only the intermediate object files. --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a5e8312b..de0c858af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,10 @@ if(MSVC) # a smaller type, and most of the time this is just conversion from 64 to 32 bits # for things like vector sizes, which are never that big anyway. add_definitions("/wd4267") + + # MSVC panics if an object file contains more than 65,279 sections. this + # happens quite frequently with code that uses templates, such as vectors. + add_definitions("/bigobj") endif() # Automatically detect architecture based on Visual Studio generator From 0ffafbde298cc84003ebec0de75013d6baa547e9 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 9 May 2020 18:28:19 -0500 Subject: [PATCH 18/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 0686cfdfc..efc5c2610 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 0686cfdfc18e9f606f2c296b617d5b3ab6b83889 +Subproject commit efc5c2610bd0345e9102279be0a9f154e5862050 From fc10ed3df63f610fa4c3756ae121201f3c7a79f4 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 27 Apr 2020 18:26:31 -0500 Subject: [PATCH 19/97] Implementation for df-other-vectors-type in C++. Still needs Lua and Ruby support. --- library/include/BitArray.h | 10 ++++++++++ library/include/DataDefs.h | 1 + 2 files changed, 11 insertions(+) diff --git a/library/include/BitArray.h b/library/include/BitArray.h index cfb859bd0..dfd187363 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -568,4 +568,14 @@ namespace DFHack root->next = link; } }; + + template + struct DfOtherVectors + { + std::vector & operator[](O other_id) + { + auto vectors = reinterpret_cast *>(this); + return vectors[other_id]; + } + }; } diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index fd70b7ba8..3b961df20 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -474,6 +474,7 @@ namespace df using DFHack::BitArray; using DFHack::DfArray; using DFHack::DfLinkedList; + using DFHack::DfOtherVectors; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" From 7cf9624de9d25383ba548d49fa180f28409c8202 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 27 Apr 2020 18:56:48 -0500 Subject: [PATCH 20/97] add assert to make sure other_id array index is in bounds --- library/include/BitArray.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/include/BitArray.h b/library/include/BitArray.h index dfd187363..5858897a3 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -574,6 +574,8 @@ namespace DFHack { std::vector & operator[](O other_id) { + CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(*this) / sizeof(std::vector)); + auto vectors = reinterpret_cast *>(this); return vectors[other_id]; } From 6c20fe4c804fa97018bb1a4374c0d86470d5e26f Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 28 Apr 2020 13:55:34 -0500 Subject: [PATCH 21/97] implement df-other-vectors-type for lua --- library/LuaTypes.cpp | 22 ++++++++++++++++++++++ library/include/DataDefs.h | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index 306538f51..c46d7ce05 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -1422,6 +1422,28 @@ void struct_identity::build_metatable(lua_State *state) SetPtrMethods(state, base+1, base+2); } +void other_vectors_identity::build_metatable(lua_State *state) +{ + int base = lua_gettop(state); + MakeFieldMetatable(state, this, meta_struct_index, meta_struct_newindex); + + EnableMetaField(state, base+2, "_enum"); + + LookupInTable(state, index_enum, &DFHACK_TYPEID_TABLE_TOKEN); + lua_setfield(state, base+1, "_enum"); + + auto keys = &index_enum->getKeys()[-index_enum->getFirstItem()]; + + for (int64_t i = 0; i < index_enum->getLastItem(); i++) + { + lua_getfield(state, base+2, keys[i]); + lua_rawseti(state, base+2, int(i)); + } + + SetStructMethod(state, base+1, base+2, meta_struct_field_reference, "_field"); + SetPtrMethods(state, base+1, base+2); +} + void global_identity::build_metatable(lua_State *state) { int base = lua_gettop(state); diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index 3b961df20..f4eff8510 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -334,6 +334,23 @@ namespace DFHack virtual identity_type type() { return IDTYPE_UNION; } }; + class DFHACK_EXPORT other_vectors_identity : public struct_identity { + enum_identity *index_enum; + + public: + other_vectors_identity(size_t size, TAllocateFn alloc, + compound_identity *scope_parent, const char *dfhack_name, + struct_identity *parent, const struct_field_info *fields, + enum_identity *index_enum) : + struct_identity(size, alloc, scope_parent, dfhack_name, parent, fields), + index_enum(index_enum) + {} + + enum_identity *getIndexEnum() { return index_enum; } + + virtual void build_metatable(lua_State *state); + }; + #ifdef _MSC_VER typedef void *virtual_ptr; #else @@ -465,6 +482,7 @@ namespace df using DFHack::global_identity; using DFHack::struct_identity; using DFHack::union_identity; + using DFHack::other_vectors_identity; using DFHack::struct_field_info; using DFHack::struct_field_info_extra; using DFHack::bitfield_item_info; From 28c34217d86a77b6eafe26aca1c3ff371cb65ecb Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 28 Apr 2020 14:26:33 -0500 Subject: [PATCH 22/97] Ruby implementation of df-other-vectors-type --- plugins/ruby/codegen.pl | 8 ++++++++ plugins/ruby/ruby-autogen-defs.rb | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/plugins/ruby/codegen.pl b/plugins/ruby/codegen.pl index 2a12a1bea..535a49694 100755 --- a/plugins/ruby/codegen.pl +++ b/plugins/ruby/codegen.pl @@ -240,6 +240,12 @@ sub render_global_class { } my $rbparent = ($parent ? rb_ucase($parent) : 'MemHack::Compound'); + my $ienum; + if (($type->getAttribute('ld:subtype') or '') eq 'df-other-vectors-type') + { + $rbparent = 'MemHack::OtherVectors'; + $ienum = rb_ucase($type->getAttribute('index-enum')); + } push @lines_rb, "class $rbname < $rbparent"; indent_rb { my $sz = sizeof($type); @@ -249,6 +255,8 @@ sub render_global_class { push @lines_rb, "rtti_classname :$rtti_name\n" if $rtti_name; + push @lines_rb, "ienum $ienum\n" if $ienum; + render_struct_fields($type); my $vms = $type->findnodes('child::virtual-methods')->[0]; diff --git a/plugins/ruby/ruby-autogen-defs.rb b/plugins/ruby/ruby-autogen-defs.rb index 10727894d..c72ffeb2b 100644 --- a/plugins/ruby/ruby-autogen-defs.rb +++ b/plugins/ruby/ruby-autogen-defs.rb @@ -205,6 +205,22 @@ module DFHack end end + class OtherVectors < Compound + class << self + attr_accessor :_enum + def ienum(enum) + @_enum = enum + end + end + + def [](i) + self.send(self.class._enum.sym(i)) + end + def []=(i, v) + self.send((self.class._enum.sym(i).to_s + "=").to_sym, v) + end + end + class Enum # number -> symbol def self.enum From 94d7f3b3475a6b1ff1c773e81395acb88f660ff7 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:25:42 -0400 Subject: [PATCH 23/97] Update docs on cloning and git stuff --- docs/Compile.rst | 106 ++++++++++++++++++++++++++++-------------- docs/Introduction.rst | 1 + 2 files changed, 71 insertions(+), 36 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index f30775a03..3e7f9701e 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -2,10 +2,16 @@ Compiling DFHack ################ -You don't need to compile DFHack unless you're developing plugins or working on the core. +DFHack builds are available for all supported platforms; see `installing` for +installation instructions. If you are a DFHack end-user, modder, or plan on +writing scripts (not plugins), it is generally recommended (and easier) to use +these builds instead of compiling DFHack from source. -For users, modders, and authors of scripts it's better to download -and `install the latest release instead `. +However, if you are looking to develop plugins, work on the DFHack core, make +complex changes to DF-structures, or anything else that requires compiling +DFHack from source, this document will walk you through the build process. Note +that some steps may be unconventional compared to other projects, so be sure to +pay close attention if this is your first time compiling DFHack. .. contents:: :depth: 2 @@ -14,35 +20,60 @@ and `install the latest release instead `. How to get the code =================== -DFHack doesn't have any kind of system of code snapshots in place, so you will have to -get code from the GitHub repository using Git. How to get Git is described under -the instructions for each platform. - -To get the latest release code (master branch):: +DFHack uses Git for source control; instructions for installing Git can be found +in the platform-specific sections below. The code is hosted on +`GitHub `_, and can be downloaded with:: git clone --recursive https://github.com/DFHack/dfhack cd dfhack -If your version of Git does not support the ``--recursive`` flag, you will need to omit it and run -``git submodule update --init`` after entering the dfhack directory. +If your version of Git does not support the ``--recursive`` flag, you will need +to omit it and run ``git submodule update --init`` after entering the dfhack +directory. -To get the latest development code (develop branch), clone as above and then:: +This will check out the code on the default branch of the GitHub repo, currently +``develop``, which may be unstable. If you want code for the latest stable +release, you can check out the ``master`` branch instead:: - git checkout develop + git checkout master git submodule update -Generally, you should only need to clone DFHack once. - -**Important note regarding submodule update after pulling or changing branches**: - -You must run ``git submodule update`` every time you change branches, such as -when switching between the master and develop branches or vice versa. You also -must run it after pulling any changes to submodules from the DFHack repo. If a -submodule only exists on the newer branch, or if a commit you just pulled -contains a new submodule, you need to run ``git submodule update --init``. -Failure to do this may result in a variety of errors, including ``fatal: -does not exist`` when using Git, errors when building DFHack, and ``not a known -DF version`` when starting DF. +In general, a single DFHack clone is suitable for development - most Git +operations such as switching branches can be done on an existing clone. If you +find yourself cloning DFHack frequently as part of your development process, or +getting stuck on anything else Git-related, feel free to reach out to us for +assistance. + +.. admonition:: A note on submodules + + DFHack uses submodules extensively to manage its subprojects (including the + ``scripts`` folder and DF-structures in ``library/xml``). Failing to keep + submodules in sync when switching between branches can result in build errors + or scripts that don't work. In general, you should always update submodules + whenever you switch between branches in the main DFHack repo with + ``git submodule update``. (If you are working on bleeding-edge DFHack and + have checked out the master branch of some submodules, running ``git pull`` + in those submodules is also an option.) + + Rarely, we add or remove submodules. If there are any changes to the existence + of submodules when you switch between branches, you should run + ``git submodule update --init`` instead (adding ``--init`` to the above + command). + + Some common errors that can arise when failing to update submodules include: + + * ``fatal: does not exist`` when performing Git operations + * Build errors, particularly referring to structures in the ``df::`` namespace + or the ``library/include/df`` folder + * ``Not a known DF version`` when starting DF + * ``Run 'git submodule update --init'`` when running CMake + + Submodules are a particularly confusing feature of Git. The + `Git Book `_ has a + thorough explanation of them (as well as of many other aspects of Git) and + is a recommended resource if you run into any issues. Other DFHack developers + are also able to help with any submodule-related (or Git-related) issues + you may encounter. **More notes**: @@ -51,18 +82,21 @@ DF version`` when starting DF. Contributing to DFHack ====================== -If you want to get involved with the development, create an account on -GitHub, make a clone there and then use that as your remote repository instead. - -We'd love that; join us on IRC_ (#dfhack channel on freenode) for discussion, -and whenever you need help. - -.. _IRC: https://webchat.freenode.net/?channels=dfhack - -(Note: for submodule issues, please see the above instructions first!) - -For lots more details on contributing to DFHack, including pull requests, code format, -and more, please see `contributing-code`. +To contribute to DFHack on GitHub, you will need a GitHub account. Only some +DFHack developers can push directly to the DFHack repositories; we recommend +making a fork of whatever repos you are interested in contributing to, making +changes there, and submitting pull requests. `GitHub's pull request tutorial +`_ +is a good resource for getting started with pull requests (some things to note: +our work mainly happens on the ``develop`` branch, and you will need to use +your own fork, assuming that you don't have write access to the DFHack repos). + +Most development-related discussion happens on IRC or in individual GitHub +issues and pull requests, but there are also other ways to reach out - see +`support` for details. + +For more details on contributing to DFHack, including pull requests, code +format, and more, please see `contributing-code`. Build settings diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 3a36aa538..09e38e52f 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -87,6 +87,7 @@ the console. .. _troubleshooting: +.. _support: Troubleshooting =============== From df06f31137cfc931dcd37e81263e6a55f8586acf Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:25:56 -0400 Subject: [PATCH 24/97] Update forum thread link --- docs/Introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 09e38e52f..22033d528 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -102,7 +102,7 @@ If the search function in this documentation isn't enough and :wiki:`the DF Wiki <>` hasn't helped, try asking in: - the `#dfhack IRC channel on freenode `_ -- the :forums:`Bay12 DFHack thread <139553>` +- the `Bay12 DFHack thread `_ - the `/r/dwarffortress `_ questions thread - the thread for the mod or Starter Pack you're using (if any) From 7a9be264c6464a482418b92a8578d158c1789a0d Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:46:09 -0400 Subject: [PATCH 25/97] Update Linux GCC docs and some ninja notes --- docs/Compile.rst | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index 3e7f9701e..f61d89e2e 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -112,7 +112,8 @@ Generator The ``Ninja`` CMake build generator is the prefered build method on Linux and macOS, instead of ``Unix Makefiles``, which is the default. You can select Ninja by passing ``-G Ninja`` to CMake. Incremental builds using Unix Makefiles can be -much slower than Ninja builds. +much slower than Ninja builds. Note that you will probably need to install +Ninja; see the platform-specific sections for details. :: @@ -178,13 +179,18 @@ DFHack is meant to be installed into an existing DF folder, so get one ready. We assume that any Linux platform will have ``git`` available (though it may need to be installed with your package manager.) -To build DFHack you need GCC version 4.8 or later. GCC 4.8 is easiest to work -with due to avoiding libstdc++ issues (see below), but any version from 4.8 -onwards (including 5.x) will work. +To build DFHack, you need GCC 4.8 or newer. GCC 4.8 has the benefit of avoiding +`libstdc++ compatibility issues `, but can be hard +to obtain on modern distributions, and working around these issues is done +automatically by the ``dfhack`` launcher script. As long as your system-provided +GCC is new enough, it should work. Note that extremely new GCC versions may not +have been used to build DFHack yet, so if you run into issues with these, please +let us know (e.g. by opening a GitHub issue). Before you can build anything, you'll also need ``cmake``. It is advisable to also get ``ccmake`` on distributions that split the cmake package into multiple -parts. +parts. As mentioned above, ``ninja`` is recommended (many distributions call +this package ``ninja-build``). You will need pthread; most systems should have this already. Note that older CMake versions may have trouble detecting pthread, so if you run into @@ -260,29 +266,32 @@ This will show a curses-based interface that lets you set all of the extra options. You can also use a cmake-friendly IDE like KDevelop 4 or the cmake-gui program. +.. _linux-incompatible-libstdcxx: + Incompatible libstdc++ ~~~~~~~~~~~~~~~~~~~~~~ -When compiling dfhack yourself, it builds against your system libstdc++. When -Dwarf Fortress runs, it uses a libstdc++ shipped with the binary, which comes -from GCC 4.8 and is incompatible with code compiled with newer GCC versions. If -you compile DFHack with a GCC version newer than 4.8, you will see an error -message such as:: +When compiling DFHack yourself, it builds against your system libstdc++. When +Dwarf Fortress runs, it uses a libstdc++ shipped in the ``libs`` folder, which +comes from GCC 4.8 and is incompatible with code compiled with newer GCC +versions. As of DFHack 0.42.05-alpha1, the ``dfhack`` launcher script attempts +to fix this by automatically removing the DF-provided libstdc++ on startup. +In rare cases, this may fail and cause errors such as:: ./libs/Dwarf_Fortress: /pathToDF/libs/libstdc++.so.6: version `GLIBCXX_3.4.18' not found (required by ./hack/libdfhack.so) -To fix this you can compile with GCC 4.8 or remove the libstdc++ shipped with +The easiest way to fix this is generally removing the libstdc++ shipped with DF, which causes DF to use your system libstdc++ instead:: cd /path/to/DF/ rm libs/libstdc++.so.6 -Note that distributing binaries compiled with newer GCC versions requires end- -users to delete libstdc++ themselves and have a libstdc++ on their system from -the same GCC version or newer. For this reason, distributing anything compiled -with GCC versions newer than 4.8 is discouraged. In the future we may start -bundling a later libstdc++ as part of the DFHack package, so as to enable -compilation-for-distribution with a GCC newer than 4.8. +Note that distributing binaries compiled with newer GCC versions may result in +the opposite compatibily issue: users with *older* GCC versions may encounter +similar errors. This is why DFHack distributes both GCC 4.8 and GCC 7 builds. If +you are planning on distributing binaries to other users, we recommend using an +older GCC (but still at least 4.8) version if possible. + Mac OS X ======== From d189b91f4997b35fa691dcf2e809ee0860fcba9e Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 13 May 2020 22:58:32 -0400 Subject: [PATCH 26/97] Update macOS -> OS X, tweak a bit and add more anchors --- docs/Compile.rst | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index f61d89e2e..6ebaa8c1f 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -168,6 +168,9 @@ your build folder or by running ``ccmake`` (or another CMake GUI). Most DFHack-specific settings begin with ``BUILD_`` and control which parts of DFHack are built. + +.. _compile-linux: + Linux ===== On Linux, DFHack acts as a library that shadows parts of the SDL API using LD_PRELOAD. @@ -293,13 +296,15 @@ you are planning on distributing binaries to other users, we recommend using an older GCC (but still at least 4.8) version if possible. -Mac OS X -======== -DFHack functions similarly on OS X and Linux, and the majority of the -information above regarding the build process (cmake and ninja) applies here +.. _compile-macos: + +macOS +===== +DFHack functions similarly on macOS and Linux, and the majority of the +information above regarding the build process (CMake and Ninja) applies here as well. -DFHack can officially be built on OS X with GCC 4.8 or 7. Anything newer than 7 +DFHack can officially be built on macOS only with GCC 4.8 or 7. Anything newer than 7 will require you to perform extra steps to get DFHack to run (see `osx-new-gcc-notes`), and your build will likely not be redistributable. @@ -352,7 +357,7 @@ Dependencies and system set-up cleaner, quicker, and smarter. For example, installing MacPort's GCC will install more than twice as many dependencies as Homebrew's will, and all in both 32-bit and 64-bit variants. Homebrew also doesn't require constant use - of sudo. + of ``sudo``. Using `Homebrew `_ (recommended):: @@ -421,9 +426,9 @@ Building export CC=gcc-7 export CXX=g++-7 - etc. + (adjust as needed for different GCC installations) -* Build dfhack:: +* Build DFHack:: mkdir build-osx cd build-osx @@ -433,6 +438,7 @@ Building should be a path to a copy of Dwarf Fortress, of the appropriate version for the DFHack you are building. + .. _compile-windows: Windows @@ -725,9 +731,9 @@ It may be installed in a directory such as ``~/.local/bin/``, so after pip install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``. -Mac OS X --------- -OS X has Python 2.7 installed by default, but it does not have the pip package manager. +macOS +----- +macOS has Python 2.7 installed by default, but it does not have the pip package manager. You can install Homebrew's Python 3, which includes pip, and then install the latest Sphinx using pip:: @@ -739,11 +745,11 @@ Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew:: brew install sphinx-doc -This will install Sphinx for OS X's system Python 2.7, without needing pip. +This will install Sphinx for macOS's system Python 2.7, without needing pip. Either method works; if you plan to use Python for other purposes, it might best to install Homebrew's Python 3 so that you have the latest Python as well as pip. -If not, just installing sphinx-doc for OS X's system Python 2.7 is fine. +If not, just installing sphinx-doc for macOS's system Python 2.7 is fine. Windows @@ -810,7 +816,7 @@ files alphabetically, so all the files you need should be next to each other. It is recommended that you create a build folder and run CMake to verify that you have downloaded everything at this point, assuming your download machine has CMake installed. This involves running a "generate" batch script on Windows, or -a command starting with ``cmake .. -G Ninja`` on Linux and OS X, following the +a command starting with ``cmake .. -G Ninja`` on Linux and macOS, following the instructions in the sections above. CMake should automatically locate files that you placed in ``CMake/downloads``, and use them instead of attempting to download them. From 7dd4b6d9685cbbbb9dda44129967e34b9b5970e8 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 14 May 2020 13:07:11 -0500 Subject: [PATCH 27/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index efc5c2610..cbeed8b52 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit efc5c2610bd0345e9102279be0a9f154e5862050 +Subproject commit cbeed8b52482ce749fc2ff13a9db9733931e7da0 From f20446534bb7f39425e102bd70daec46e328004f Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 14 May 2020 14:01:05 -0500 Subject: [PATCH 28/97] fix DfOtherVectors computed size --- library/include/BitArray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/include/BitArray.h b/library/include/BitArray.h index 5858897a3..1f558221a 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -574,7 +574,7 @@ namespace DFHack { std::vector & operator[](O other_id) { - CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(*this) / sizeof(std::vector)); + CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(T) / sizeof(std::vector)); auto vectors = reinterpret_cast *>(this); return vectors[other_id]; From 3db490ee9e31c7a0f950f242b8c338a161a63fcc Mon Sep 17 00:00:00 2001 From: Rose Date: Fri, 22 May 2020 08:01:34 -0700 Subject: [PATCH 29/97] Fix a problem where the block coordinates would sometimes not get copied. --- plugins/remotefortressreader/remotefortressreader.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/remotefortressreader/remotefortressreader.cpp b/plugins/remotefortressreader/remotefortressreader.cpp index e94c3f832..8e9ac8d45 100644 --- a/plugins/remotefortressreader/remotefortressreader.cpp +++ b/plugins/remotefortressreader/remotefortressreader.cpp @@ -949,9 +949,6 @@ static command_result GetGrowthList(color_ostream &stream, const EmptyMessage *i void CopyBlock(df::map_block * DfBlock, RemoteFortressReader::MapBlock * NetBlock, MapExtras::MapCache * MC, DFCoord pos) { - NetBlock->set_map_x(DfBlock->map_pos.x); - NetBlock->set_map_y(DfBlock->map_pos.y); - NetBlock->set_map_z(DfBlock->map_pos.z); MapExtras::Block * block = MC->BlockAtTile(DfBlock->map_pos); @@ -1465,7 +1462,12 @@ static command_result GetBlockList(color_ostream &stream, const BlockRequest *in bool flows = block->flows.size() > 0; RemoteFortressReader::MapBlock *net_block = nullptr; if (tileChanged || desChanged || spatterChanged || firstBlock || itemsChanged || flows) + { net_block = out->add_map_blocks(); + net_block->set_map_x(block->map_pos.x); + net_block->set_map_y(block->map_pos.y); + net_block->set_map_z(block->map_pos.z); + } if (tileChanged) { CopyBlock(block, net_block, &MC, pos); From ad3735421186f56e2097e8324610b9aa77969722 Mon Sep 17 00:00:00 2001 From: Rose Date: Sun, 24 May 2020 15:42:59 -0700 Subject: [PATCH 30/97] Bump version number to make sure the updater works. --- plugins/remotefortressreader/remotefortressreader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/remotefortressreader/remotefortressreader.cpp b/plugins/remotefortressreader/remotefortressreader.cpp index 4ab540a5e..5a8957ddf 100644 --- a/plugins/remotefortressreader/remotefortressreader.cpp +++ b/plugins/remotefortressreader/remotefortressreader.cpp @@ -1,5 +1,5 @@ #include "df_version_int.h" -#define RFR_VERSION "0.20.3" +#define RFR_VERSION "0.21.0" #include #include From 2fc7fa7ac2e56bffa5775e33fc3a3a8cafd6f364 Mon Sep 17 00:00:00 2001 From: lethosor Date: Thu, 28 May 2020 23:11:08 -0400 Subject: [PATCH 31/97] Update xml, scripts See #1576 --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index cbeed8b52..995caf761 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit cbeed8b52482ce749fc2ff13a9db9733931e7da0 +Subproject commit 995caf76159ae84046f723ebb4547f9a9eae372c diff --git a/scripts b/scripts index 2079b9fb6..47d9b3c93 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 2079b9fb69b8b4db48aa35ec54a96f5cca7cc8ef +Subproject commit 47d9b3c9390d21ecc49ddaba477b16071884ee87 From 5d05cfc7ccadbad5dceab045f8e66bb064a2c43c Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 2 Jun 2020 01:15:33 -0500 Subject: [PATCH 32/97] Fix several functions in the Units module ignoring unit caste. Fixes #1583. --- docs/changelog.txt | 1 + library/modules/Units.cpp | 52 +++++++++++---------------------------- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index 1aa7f916e..90492cdfc 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -42,6 +42,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: ## Fixes - Fixed a segfault when attempting to start a headless session with a graphical PRINT_MODE setting - `labormanager`: fixed handling of new jobs in 0.47 +- Fixed ``Units::isEggLayer``, ``Units::isGrazer``, ``Units::isMilkable``, ``Units::isTrainableHunting``, ``Units::isTrainableWar``, and ``Units::isTamable`` ignoring the unit's caste ## Ruby - Updated ``item_find`` and ``building_find`` to use centralized logic that works on more screens diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index c4b9e60fa..422060803 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -636,74 +636,50 @@ bool Units::isEggLayer(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if ((*caste)->flags.is_set(caste_raw_flags::LAYS_EGGS) - || (*caste)->flags.is_set(caste_raw_flags::LAYS_UNUSUAL_EGGS)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::LAYS_EGGS) + || caste->flags.is_set(caste_raw_flags::LAYS_UNUSUAL_EGGS); } bool Units::isGrazer(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::GRAZER)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::GRAZER); } bool Units::isMilkable(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::MILKABLE)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::MILKABLE); } bool Units::isTrainableWar(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::TRAINABLE_WAR)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::TRAINABLE_WAR); } bool Units::isTrainableHunting(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::TRAINABLE_HUNTING)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::TRAINABLE_HUNTING); } bool Units::isTamable(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::PET) || - (*caste)->flags.is_set(caste_raw_flags::PET_EXOTIC)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::PET) + || caste->flags.is_set(caste_raw_flags::PET_EXOTIC); } bool Units::isMale(df::unit* unit) From bd216dbc117bdb197fe12febaf1285f5a67c57d7 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 15 Jun 2020 23:47:35 -0500 Subject: [PATCH 33/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 995caf761..b80a661f1 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 995caf76159ae84046f723ebb4547f9a9eae372c +Subproject commit b80a661f1139eb8535e6f76f8a07b697eeb9c9de From 97a20acf1b3bcb83e84e85e0ced0eeb4e33f2269 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 17 Jun 2020 20:32:30 -0400 Subject: [PATCH 34/97] Add current year to docs copyright message --- conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf.py b/conf.py index 2f47d5644..c36e4f8aa 100644 --- a/conf.py +++ b/conf.py @@ -15,6 +15,7 @@ serve to show the default. # pylint:disable=redefined-builtin +import datetime from io import open import os import re @@ -222,7 +223,7 @@ master_doc = 'index' # General information about the project. project = 'DFHack' -copyright = '2015, The DFHack Team' +copyright = '2015-%d, The DFHack Team' % datetime.datetime.now().year author = 'The DFHack Team' # The version info for the project you're documenting, acts as replacement for From 861c8c81482e87e7bc07c4faaaa103505f8876f0 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 17 Jun 2020 20:36:07 -0400 Subject: [PATCH 35/97] Remove -q from docs/build.sh to make -v work --- docs/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build.sh b/docs/build.sh index 6745db4d5..48e1bfd4a 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -17,4 +17,4 @@ if [ -z "$JOBS" ]; then JOBS=2 fi -"$sphinx" -a -E -q -b html . ./docs/html -w ./docs/_sphinx-warnings.txt -j "$JOBS" "$@" +"$sphinx" -a -E -b html . ./docs/html -w ./docs/_sphinx-warnings.txt -j "$JOBS" "$@" From 5073bd31c4545e91f78623d7c28b0a61fa813d4a Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 19 Jun 2020 19:37:44 -0500 Subject: [PATCH 36/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index b80a661f1..bc42fe311 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b80a661f1139eb8535e6f76f8a07b697eeb9c9de +Subproject commit bc42fe3114a7943850b32231559adf54bb1d0c60 From aee4fcd1a6fd708f706e03bee01d4287e0d656f9 Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 20 Jun 2020 12:02:13 -0400 Subject: [PATCH 37/97] Add confirmation for convicting dwarves of crimes Closes #1593 --- docs/changelog.txt | 3 +++ plugins/confirm.cpp | 2 ++ plugins/lua/confirm.lua | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index 90492cdfc..c165ff855 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -44,6 +44,9 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: - `labormanager`: fixed handling of new jobs in 0.47 - Fixed ``Units::isEggLayer``, ``Units::isGrazer``, ``Units::isMilkable``, ``Units::isTrainableHunting``, ``Units::isTrainableWar``, and ``Units::isTamable`` ignoring the unit's caste +## Misc Improvements +- `confirm`: added a confirmation dialog for convicting dwarves of crimes + ## Ruby - Updated ``item_find`` and ``building_find`` to use centralized logic that works on more screens diff --git a/plugins/confirm.cpp b/plugins/confirm.cpp index c616ac493..07aee4640 100644 --- a/plugins/confirm.cpp +++ b/plugins/confirm.cpp @@ -18,6 +18,7 @@ #include "df/general_ref.h" #include "df/general_ref_contained_in_itemst.h" #include "df/viewscreen_dwarfmodest.h" +#include "df/viewscreen_justicest.h" #include "df/viewscreen_layer_militaryst.h" #include "df/viewscreen_locationsst.h" #include "df/viewscreen_tradegoodsst.h" @@ -479,6 +480,7 @@ DEFINE_CONFIRMATION(uniform_delete, viewscreen_layer_militaryst); DEFINE_CONFIRMATION(note_delete, viewscreen_dwarfmodest); DEFINE_CONFIRMATION(route_delete, viewscreen_dwarfmodest); DEFINE_CONFIRMATION(location_retire, viewscreen_locationsst); +DEFINE_CONFIRMATION(convict, viewscreen_justicest); DFhackCExport command_result plugin_init (color_ostream &out, vector &commands) { diff --git a/plugins/lua/confirm.lua b/plugins/lua/confirm.lua index 4312ab4be..b8a815662 100644 --- a/plugins/lua/confirm.lua +++ b/plugins/lua/confirm.lua @@ -204,6 +204,21 @@ end location_retire.title = "Retire location" location_retire.message = "Are you sure you want to retire this location?" +convict = defconf('convict') +convict.title = "Confirm conviction" +function convict.intercept_key(key) + return key == keys.SELECT and + screen.cur_column == df.viewscreen_justicest.T_cur_column.ConvictChoices +end +function convict.get_message() + name = dfhack.TranslateName(screen.convict_choices[screen.cursor_right].name) + if name == "" then + name = "this creature" + end + return "Are you sure you want to convict " .. name .. "?\n" .. + "This action is irreversible." +end + -- End of confirmation definitions function check() From 7b4ae9b6a3b0cd289dfcea61ef34b5d3e9eae57b Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 20 Jun 2020 23:08:55 -0500 Subject: [PATCH 38/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index bc42fe311..cd0e5ec3c 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit bc42fe3114a7943850b32231559adf54bb1d0c60 +Subproject commit cd0e5ec3c4588310e5d20c6f28ee1deed2cd33f6 From 85855a2d88ed7ee1bd4457084ea0b617006f0663 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sun, 21 Jun 2020 16:16:16 -0500 Subject: [PATCH 39/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index cd0e5ec3c..6f7f33ceb 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit cd0e5ec3c4588310e5d20c6f28ee1deed2cd33f6 +Subproject commit 6f7f33cebbbefefa7c6f920b2fc197b56a3e8d1c From e2301ecae7d7c76778430545c32e0e5e452eebe1 Mon Sep 17 00:00:00 2001 From: Quietust Date: Tue, 23 Jun 2020 13:31:27 -0600 Subject: [PATCH 40/97] Update structures, fix reaction_productst::produce() parms --- library/modules/Items.cpp | 4 ++-- library/xml | 2 +- plugins/add-spatter.cpp | 4 ++-- plugins/createitem.cpp | 4 ++-- plugins/diggingInvaders/assignJob.cpp | 4 ++-- plugins/eventful.cpp | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/library/modules/Items.cpp b/library/modules/Items.cpp index 05f174ddf..0e02043dc 100644 --- a/library/modules/Items.cpp +++ b/library/modules/Items.cpp @@ -1477,9 +1477,9 @@ int32_t Items::createItem(df::item_type item_type, int16_t item_subtype, int16_t df::enums::game_type::game_type type = *df::global::gametype; prod->produce(unit, &out_products, &out_items, &in_reag, &in_items, 1, job_skill::NONE, - df::historical_entity::find(unit->civ_id), 0, + 0, df::historical_entity::find(unit->civ_id), ((type == df::enums::game_type::DWARF_MAIN) || (type == df::enums::game_type::DWARF_RECLAIM)) ? df::world_site::find(df::global::ui->site_id) : NULL, - 0); + NULL); if ( out_items.size() != 1 ) return -1; diff --git a/library/xml b/library/xml index 6f7f33ceb..542d1bd28 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 6f7f33cebbbefefa7c6f920b2fc197b56a3e8d1c +Subproject commit 542d1bd283051186b2d4943c9acb546af1cc7b8b diff --git a/plugins/add-spatter.cpp b/plugins/add-spatter.cpp index 3d2d89dd1..459e28f25 100644 --- a/plugins/add-spatter.cpp +++ b/plugins/add-spatter.cpp @@ -249,7 +249,7 @@ struct product_hook : improvement_product { std::vector *in_reag, std::vector *in_items, int32_t quantity, df::job_skill skill, - df::historical_entity *entity, int32_t unk, df::world_site *site, void* unk2) + int32_t quality, df::historical_entity *entity, df::world_site *site, std::vector *unk2) ) { if (auto product = products[this]) { @@ -295,7 +295,7 @@ struct product_hook : improvement_product { return; } - INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, entity, unk, site, unk2); + INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, quality, entity, site, unk2); } }; diff --git a/plugins/createitem.cpp b/plugins/createitem.cpp index 939afebc2..4ef1c853c 100644 --- a/plugins/createitem.cpp +++ b/plugins/createitem.cpp @@ -86,8 +86,8 @@ bool makeItem (df::reaction_product_itemst *prod, df::unit *unit, bool second_it building = df::building::find(dest_building); prod->produce(unit, &out_products, &out_items, &in_reag, &in_items, 1, job_skill::NONE, - df::historical_entity::find(unit->civ_id), 0, - (World::isFortressMode()) ? df::world_site::find(ui->site_id) : NULL, 0); + 0, df::historical_entity::find(unit->civ_id), + (World::isFortressMode()) ? df::world_site::find(ui->site_id) : NULL, NULL); if (!out_items.size()) return false; // if we asked to make shoes and we got twice as many as we asked, then we're okay diff --git a/plugins/diggingInvaders/assignJob.cpp b/plugins/diggingInvaders/assignJob.cpp index b253e7833..7e3dfaf8c 100644 --- a/plugins/diggingInvaders/assignJob.cpp +++ b/plugins/diggingInvaders/assignJob.cpp @@ -256,8 +256,8 @@ int32_t assignJob(color_ostream& out, Edge firstImportantEdge, unordered_map in_reag; vector in_items; prod->produce(firstInvader, &out_products, &out_items, &in_reag, &in_items, 1, df::job_skill::NONE, - df::historical_entity::find(firstInvader->civ_id), 0, - df::world_site::find(df::global::ui->site_id), 0); + 0, df::historical_entity::find(firstInvader->civ_id), + df::world_site::find(df::global::ui->site_id), NULL); if ( out_items.size() != 1 ) { out.print("%s, %d: wrong size: %zu.\n", __FILE__, __LINE__, out_items.size()); diff --git a/plugins/eventful.cpp b/plugins/eventful.cpp index 5d8dfdb59..42cfbdf69 100644 --- a/plugins/eventful.cpp +++ b/plugins/eventful.cpp @@ -287,12 +287,12 @@ struct product_hook : item_product { std::vector *in_reag, std::vector *in_items, int32_t quantity, df::job_skill skill, - df::historical_entity *entity, int32_t unk, df::world_site *site, void* unk2) + int32_t quality, df::historical_entity *entity, df::world_site *site, std::vector *unk2) ) { color_ostream_proxy out(Core::getInstance().getConsole()); auto product = products[this]; if ( !product ) { - INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, entity, unk, site, unk2); + INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, quality, entity, site, unk2); return; } df::reaction* this_reaction=product->react; @@ -304,7 +304,7 @@ struct product_hook : item_product { size_t out_item_count = out_items->size(); - INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, entity, unk, site, unk2); + INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, quality, entity, site, unk2); if ( out_items->size() == out_item_count ) return; //if it produced something, call the scripts From ce3cc24228258afd76c8b7193b605eb32bc5db54 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 24 Jun 2020 00:11:54 -0400 Subject: [PATCH 41/97] Split docs on documentation into a separate file and reorganize/expand a bit --- docs/Compile.rst | 125 +-------------------------- docs/Documentation.rst | 191 +++++++++++++++++++++++++++++++++++++++++ docs/changelog.txt | 12 +-- index.rst | 1 + 4 files changed, 197 insertions(+), 132 deletions(-) create mode 100644 docs/Documentation.rst diff --git a/docs/Compile.rst b/docs/Compile.rst index 6ebaa8c1f..650e762e3 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -656,129 +656,8 @@ Then build the ``INSTALL`` target listed under ``CMakePredefinedTargets``. Building the documentation ========================== -DFHack documentation, like the file you are reading now, is created as .rst files, -which are in `reStructuredText (reST) `_ format. -This is a documenation format that has come from the Python community. It is very -similar in concept - and in syntax - to Markdown, as found on GitHub and many other -places. However it is more advanced than Markdown, with more features available when -compiled to HTML, such as automatic tables of contents, cross-linking, special -external links (forum, wiki, etc) and more. The documentation is compiled by a -Python tool, `Sphinx `_. - -The DFHack build process will compile the documentation but this has been disabled -by default. You only need to build the docs if you're changing them, or perhaps -if you want a local HTML copy; otherwise, read them easily online at -`ReadTheDoc's DFHack hosted documentation `_. - -(Note that even if you do want a local copy, it is certainly not necesesary to -compile the documentation in order to read it. Like Markdown, reST documents are -designed to be just as readable in a plain-text editor as they are in HTML format. -The main thing you lose in plain text format is hyperlinking.) - - -Enabling documentation building -------------------------------- -First, make sure you have followed all the necessary steps for your platform as -outlined in the rest of this document. - -To compile documentation with DFHack, add the following flag to your ``cmake`` command:: - - -DBUILD_DOCS:bool=ON - -For example:: - - cmake .. -DCMAKE_BUILD_TYPE:string=Release -DBUILD_DOCS:bool=ON -DCMAKE_INSTALL_PREFIX= - -Alternatively you can use the CMake GUI which allows options to be changed easily. - -On Windows you should either use ``generate-msvc-gui.bat`` and set the option -through the GUI, or else if you want to use an alternate file, such as -``generate-msvc-all.bat``, you will need to edit it to add the flag. -Or you could just run ``cmake`` on the command line like in other platforms. - -Required dependencies ---------------------- -In order to build the documentation, you must have Python with Sphinx -version 1.3.1 or later. Both Python 2.x and 3.x are supported. - -When installing Sphinx from OS package managers, be aware that there is -another program called Sphinx, completely unrelated to documentation management. -Be sure you are installing the right Sphinx; it may be called ``python-sphinx``, -for example. To avoid doubt, ``pip`` can be used instead as detailed below. - - -Linux ------ -Most Linux distributions will include Python as standard. - -Check your package manager to see if Sphinx 1.3.1 or later is available, -but at the time of writing Ubuntu for example only has 1.2.x. - -You can instead install Sphinx with the pip package manager. This may need -to be installed from your OS package manager; this is the case on Ubuntu. -On Ubuntu/Debian, use the following to first install pip:: - - sudo apt-get install python-pip - -Once pip is available, you can then install the Python Sphinx module with:: - - pip install sphinx - -If you run this as a normal user it will install a local copy for your user only. -Run it with sudo if you want a system-wide install. Either is fine for DFHack, -however if installing locally do check that ``sphinx-build`` is in your path. -It may be installed in a directory such as ``~/.local/bin/``, so after pip -install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``. - - -macOS ------ -macOS has Python 2.7 installed by default, but it does not have the pip package manager. - -You can install Homebrew's Python 3, which includes pip, and then install the -latest Sphinx using pip:: - - brew install python3 - pip3 install sphinx - -Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew:: - - brew install sphinx-doc - -This will install Sphinx for macOS's system Python 2.7, without needing pip. - -Either method works; if you plan to use Python for other purposes, it might best -to install Homebrew's Python 3 so that you have the latest Python as well as pip. -If not, just installing sphinx-doc for macOS's system Python 2.7 is fine. - - -Windows -------- -Use the Chocolatey package manager to install Python and pip, -then use pip to install Sphinx. - -Run the following commands from an elevated (Admin) ``cmd.exe``, after installing -Chocolatey as outlined in the `Windows section `:: - - choco install python pip -y - -Then close that Admin ``cmd.exe``, re-open another Admin ``cmd.exe``, and run:: - - pip install sphinx - -.. _build-changelog: - -Building the changelogs ------------------------ -If you have Python installed, but do not want to build all of the documentation, -you can build the changelogs with the ``docs/gen_changelog.py`` script. - -All changes should be listed in ``changelog.txt``. A description of this file's -format follows: - -.. include:: /docs/changelog.txt - :start-after: ===help - :end-before: ===end +The steps above will not build DFHack's documentation by default. If you are +editing documentation, see `documentation` for details on how to build it. Misc. Notes =========== diff --git a/docs/Documentation.rst b/docs/Documentation.rst new file mode 100644 index 000000000..f7cb9ef92 --- /dev/null +++ b/docs/Documentation.rst @@ -0,0 +1,191 @@ +.. _documentation: + +#################### +DFHack Documentation +#################### + + +DFHack documentation, like the file you are reading now, is created as ``.rst`` files, +which are in `reStructuredText (reST) `_ format. +This is a documentation format common in the Python community. It is very +similar in concept - and in syntax - to Markdown, as found on GitHub and many other +places. However it is more advanced than Markdown, with more features available when +compiled to HTML, such as automatic tables of contents, cross-linking, special +external links (forum, wiki, etc) and more. The documentation is compiled by a +Python tool, `Sphinx `_. + +The DFHack build process will compile the documentation, but this is disabled +by default due to the additional Python and Sphinx requirements. You typically +only need to build the docs if you're changing them, or perhaps +if you want a local HTML copy; otherwise, you can read an +`online version hosted by ReadTheDocs `_. + +(Note that even if you do want a local copy, it is certainly not necessary to +compile the documentation in order to read it. Like Markdown, reST documents are +designed to be just as readable in a plain-text editor as they are in HTML format. +The main thing you lose in plain text format is hyperlinking.) + +.. contents:: + + +Required dependencies +===================== +In order to build the documentation, you must have Python with Sphinx +version 1.3.1 or later. Both Python 2.x and 3.x are supported. + +When installing Sphinx from OS package managers, be aware that there is +another program called Sphinx, completely unrelated to documentation management. +Be sure you are installing the right Sphinx; it may be called ``python-sphinx``, +for example. To avoid doubt, ``pip`` can be used instead as detailed below. + +For more detailed platform-specific instructions, see the sections below: + + +Linux +----- +Most Linux distributions will include Python as standard. + +Check your package manager to see if Sphinx 1.3.1 or later is available, +but at the time of writing Ubuntu for example only has 1.2.x. + +You can instead install Sphinx with the pip package manager. This may need +to be installed from your OS package manager; this is the case on Ubuntu. +On Ubuntu/Debian, use the following to first install pip:: + + sudo apt-get install python-pip + +Once pip is available, you can then install the Python Sphinx module with:: + + pip install sphinx + +If you run this as a normal user it will install a local copy for your user only. +Run it with sudo if you want a system-wide install. Either is fine for DFHack, +however if installing locally do check that ``sphinx-build`` is in your path. +It may be installed in a directory such as ``~/.local/bin/``, so after pip +install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``. + + +macOS +----- +macOS has Python 2.7 installed by default, but it does not have the pip package manager. + +You can install Homebrew's Python 3, which includes pip, and then install the +latest Sphinx using pip:: + + brew install python3 + pip3 install sphinx + +Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew:: + + brew install sphinx-doc + +This will install Sphinx for macOS's system Python 2.7, without needing pip. + +Either method works; if you plan to use Python for other purposes, it might best +to install Homebrew's Python 3 so that you have the latest Python as well as pip. +If not, just installing sphinx-doc for macOS's system Python 2.7 is fine. + + +Windows +------- +Python for Windows can be downloaded `from python.org `_. +The latest version of Python 3 is recommended, as it includes pip already. + +You can also install Python and pip through the Chocolatey package manager. +After installing Chocolatey as outlined in the `Windows compilation instructions `, +run the following command from an elevated (admin) command prompt (e.g. ``cmd.exe``):: + + choco install python pip -y + +Once you have pip available, you can install Sphinx with the following command:: + + pip install sphinx + +Note that this may require opening a new (admin) command prompt if you just +installed pip from the same command prompt. + + +Building the documentation +========================== + +Once you have installed Sphinx, ``sphinx-build --version`` should report the +version of Sphinx that you have installed. If this works, CMake should also be +able to find Sphinx. + +Using CMake +----------- + +Enabling the ``BUILD_DOCS`` CMake option will cause the documentation to be built +whenever it changes as part of the normal DFHack build process. There are several +ways to do this: + +* When initially running CMake, add ``-DBUILD_DOCS:bool=ON`` to your ``cmake`` + command. For example:: + + cmake .. -DCMAKE_BUILD_TYPE:string=Release -DBUILD_DOCS:bool=ON -DCMAKE_INSTALL_PREFIX= + +* If you have already run CMake, you can simply run it again from your build + folder to update your configuration:: + + cmake .. -DBUILD_DOCS:bool=ON + +* You can edit the ``BUILD_DOCS`` setting in CMakeCache.txt directly + +* You can use the CMake GUI to change ``BUILD_DOCS`` + +* On Windows, if you prefer to use the batch scripts, you can run + ``generate-msvc-gui.bat`` and set ``BUILD_DOCS`` through the GUI. If you are + running another file, such as ``generate-msvc-all.bat``, you will need to edit + it to add the flag. You can also run ``cmake`` on the command line, similar to + other platforms. + +Running Sphinx manually +----------------------- + +You can also build the documentation without going through CMake, which may be +faster. There is a ``docs/build.sh`` script available for Linux and macOS that +will run essentially the same command that CMake runs - see the script for +options. + +To build the documentation with default options, run the following command from +the root DFHack folder:: + + sphinx-build . docs/html + +Sphinx has many options to enable clean builds, parallel builds, logging, and +more - run ``sphinx-build --help`` for details. + +.. _build-changelog: + +Building the changelogs +======================= +If you have Python installed, but do not want to build all of the documentation, +you can build the changelogs with the ``docs/gen_changelog.py`` script. This +script provides additional options, including one to build individual changelogs +for all DFHack versions - run ``python docs/gen_changelog.py --help`` for details. + +Changelog entries are obtained from ``changelog.txt`` files in multiple repos. +This allows changes to be listed in the same repo where they were made. These +changelogs are combined as part of the changelog build process: + +* ``docs/changelog.txt`` for changes in the main ``dfhack`` repo +* ``scripts/changelog.txt`` for changes made to scripts in the ``scripts`` repo +* ``library/xml/changelog.txt`` for changes made in the ``df-structures`` repo + +Building the changelogs generates two files: ``docs/_auto/news.rst`` and +``docs/_auto/news-dev.rst``. These correspond to `changelog` and `dev-changelog` +and contain changes organized by stable and development DFHack releases, +respectively. For example, an entry listed under "0.44.05-alpha1" in +changelog.txt will be listed under that version in the development changelog as +well, but under "0.44.05-r1" in the stable changelog (assuming that is the +closest stable release after 0.44.05-alpha1). An entry listed under a stable +release like "0.44.05-r1" in changelog.txt will be listed under that release in +both the stable changelog and the development changelog. + + +Changelog syntax +---------------- + +.. include:: /docs/changelog.txt + :start-after: ===help + :end-before: ===end diff --git a/docs/changelog.txt b/docs/changelog.txt index c165ff855..7201bfd23 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -1,16 +1,10 @@ === Scroll down for changes ===[[[ -===help +The text below is included in docs/Documentation.rst - see that file for more details on the changelog setup. +This is kept in this file as a quick syntax reference. -Entries in docs/NEWS.rst and docs/NEWS-dev.rst are generated from -docs/changelog.txt. NEWS.rst groups entries by stable releases, and NEWS-dev.rst -groups them by all releases (stable and development). For example, an entry -listed under "0.44.05-alpha1" in changelog.txt will be listed under that in -NEWS-dev.rst as well, but under "0.44.05-r1" in NEWS.rst (assuming that is the -closest stable release after 0.44.05-alpha1). An entry listed under a stable -release in changelog.txt will be listed under that release in both NEWS.rst and -NEWS-dev.rst. +===help changelog.txt uses a syntax similar to RST, with a few special sequences: diff --git a/index.rst b/index.rst index d4bd223c5..855b31a73 100644 --- a/index.rst +++ b/index.rst @@ -62,6 +62,7 @@ For Developers /docs/Compile /docs/NEWS-dev /docs/Lua API + /docs/Documentation /library/xml/SYNTAX /library/xml/how-to-update /docs/Binpatches From 8a7e8f4facf50757f54a049fb18f67b8b81a3837 Mon Sep 17 00:00:00 2001 From: lethosor Date: Thu, 25 Jun 2020 01:12:18 -0400 Subject: [PATCH 42/97] Update xml --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 542d1bd28..2ed5fae13 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 542d1bd283051186b2d4943c9acb546af1cc7b8b +Subproject commit 2ed5fae13f721b73b71bc31a062fb66abd1f3920 From ee56d8157f62e5161f6248d9329d30e0c0ab314e Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 27 Jun 2020 00:15:48 -0400 Subject: [PATCH 43/97] Update xml --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 2ed5fae13..97084e556 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 2ed5fae13f721b73b71bc31a062fb66abd1f3920 +Subproject commit 97084e556119ce4445c3745151180467338a746a From a9f219baf9dda6d8cc300251fc040420e84bbc49 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 29 Jun 2020 16:16:44 -0500 Subject: [PATCH 44/97] update for DFHack/df-structures#7dfea1f1d4e453b97d5529785701db46360596ce --- library/modules/Kitchen.cpp | 4 ++-- library/modules/MapCache.cpp | 8 ++++---- library/modules/Materials.cpp | 2 +- library/xml | 2 +- plugins/autochop.cpp | 2 +- plugins/getplants.cpp | 10 +++++----- plugins/stonesense | 2 +- scripts | 2 +- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/library/modules/Kitchen.cpp b/library/modules/Kitchen.cpp index 63372d564..fa4d1dea7 100644 --- a/library/modules/Kitchen.cpp +++ b/library/modules/Kitchen.cpp @@ -95,7 +95,7 @@ void Kitchen::denyPlantSeedCookery(t_materialIndex materialIndex) { ui->kitchen.item_types.push_back(item_type::SEEDS); ui->kitchen.item_subtypes.push_back(organicSubtype); - ui->kitchen.mat_types.push_back(type->material_defs.type_seed); + ui->kitchen.mat_types.push_back(type->material_defs.type[plant_material_def::seed]); ui->kitchen.mat_indices.push_back(materialIndex); ui->kitchen.exc_types.push_back(df::kitchen_exc_type::Cook); } @@ -103,7 +103,7 @@ void Kitchen::denyPlantSeedCookery(t_materialIndex materialIndex) { ui->kitchen.item_types.push_back(item_type::PLANT); ui->kitchen.item_subtypes.push_back(organicSubtype); - ui->kitchen.mat_types.push_back(type->material_defs.type_basic_mat); + ui->kitchen.mat_types.push_back(type->material_defs.type[plant_material_def::basic_mat]); ui->kitchen.mat_indices.push_back(materialIndex); ui->kitchen.exc_types.push_back(df::kitchen_exc_type::Cook); } diff --git a/library/modules/MapCache.cpp b/library/modules/MapCache.cpp index b61ba375e..ce3039ecf 100644 --- a/library/modules/MapCache.cpp +++ b/library/modules/MapCache.cpp @@ -899,8 +899,8 @@ t_matpair MapExtras::BlockInfo::getBaseMaterial(df::tiletype tt, df::coord2d pos { if (auto raw = df::plant_raw::find(plant->material)) { - rv.mat_type = raw->material_defs.type_basic_mat; - rv.mat_index = raw->material_defs.idx_basic_mat; + rv.mat_type = raw->material_defs.type[plant_material_def::basic_mat]; + rv.mat_index = raw->material_defs.idx[plant_material_def::basic_mat]; } } break; @@ -912,8 +912,8 @@ t_matpair MapExtras::BlockInfo::getBaseMaterial(df::tiletype tt, df::coord2d pos rv.mat_type = MaterialInfo::PLANT_BASE; if (auto raw = df::plant_raw::find(grass[x][y])) { - rv.mat_type = raw->material_defs.type_basic_mat; - rv.mat_index = raw->material_defs.idx_basic_mat; + rv.mat_type = raw->material_defs.type[plant_material_def::basic_mat]; + rv.mat_index = raw->material_defs.idx[plant_material_def::basic_mat]; } break; diff --git a/library/modules/Materials.cpp b/library/modules/Materials.cpp index 51d717d05..c341ea99e 100644 --- a/library/modules/Materials.cpp +++ b/library/modules/Materials.cpp @@ -261,7 +261,7 @@ bool MaterialInfo::findPlant(const std::string &token, const std::string &subtok // As a special exception, return the structural material with empty subtoken if (subtoken.empty()) - return decode(p->material_defs.type_basic_mat, p->material_defs.idx_basic_mat); + return decode(p->material_defs.type[plant_material_def::basic_mat], p->material_defs.idx[plant_material_def::basic_mat]); for (size_t j = 0; j < p->material.size(); j++) if (p->material[j]->id == subtoken) diff --git a/library/xml b/library/xml index 97084e556..7dfea1f1d 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 97084e556119ce4445c3745151180467338a746a +Subproject commit 7dfea1f1d4e453b97d5529785701db46360596ce diff --git a/plugins/autochop.cpp b/plugins/autochop.cpp index e7f3874d3..8f2225f72 100644 --- a/plugins/autochop.cpp +++ b/plugins/autochop.cpp @@ -258,7 +258,7 @@ static bool skip_plant(const df::plant * plant, bool *restricted) const df::plant_raw *plant_raw = df::plant_raw::find(plant->material); // Skip fruit trees if set. - if (skip.fruit_trees && plant_raw->material_defs.type_drink != -1) + if (skip.fruit_trees && plant_raw->material_defs.type[plant_material_def::drink] != -1) { if (restricted) *restricted = true; diff --git a/plugins/getplants.cpp b/plugins/getplants.cpp index d1ee79a04..b4b7a493b 100644 --- a/plugins/getplants.cpp +++ b/plugins/getplants.cpp @@ -70,7 +70,7 @@ enum class selectability { //selectability selectablePlant(color_ostream &out, const df::plant_raw *plant, bool farming) selectability selectablePlant(const df::plant_raw *plant, bool farming) { - const DFHack::MaterialInfo basic_mat = DFHack::MaterialInfo(plant->material_defs.type_basic_mat, plant->material_defs.idx_basic_mat); + const DFHack::MaterialInfo basic_mat = DFHack::MaterialInfo(plant->material_defs.type[plant_material_def::basic_mat], plant->material_defs.idx[plant_material_def::basic_mat]); bool outOfSeason = false; selectability result = selectability::Nonselectable; @@ -92,7 +92,7 @@ selectability selectablePlant(const df::plant_raw *plant, bool farming) return selectability::Grass; } - if (farming && plant->material_defs.type_seed == -1) + if (farming && plant->material_defs.type[plant_material_def::seed] == -1) { return selectability::Nonselectable; } @@ -163,8 +163,8 @@ selectability selectablePlant(const df::plant_raw *plant, bool farming) { for (size_t k = 0; growth_mat.material->reaction_product.material.mat_type.size(); k++) { - if (growth_mat.material->reaction_product.material.mat_type[k] == plant->material_defs.type_seed && - growth_mat.material->reaction_product.material.mat_index[k] == plant->material_defs.idx_seed) + if (growth_mat.material->reaction_product.material.mat_type[k] == plant->material_defs.type[plant_material_def::seed] && + growth_mat.material->reaction_product.material.mat_index[k] == plant->material_defs.idx[plant_material_def::seed]) { seedSource = true; break; @@ -193,7 +193,7 @@ selectability selectablePlant(const df::plant_raw *plant, bool farming) } /* else if (plant->growths[i]->behavior.bits.has_seed) // This code designates beans, etc. when DF doesn't, but plant gatherers still fail to collect anything, so it's useless: bug #0006940. { - const DFHack::MaterialInfo seed_mat = DFHack::MaterialInfo(plant->material_defs.type_seed, plant->material_defs.idx_seed); + const DFHack::MaterialInfo seed_mat = DFHack::MaterialInfo(plant->material_defs.type[plant_material_def::seed], plant->material_defs.idx[plant_material_def::seed]); if (seed_mat.material->flags.is_set(material_flags::EDIBLE_RAW) || seed_mat.material->flags.is_set(material_flags::EDIBLE_COOKED)) diff --git a/plugins/stonesense b/plugins/stonesense index 5b7e7743a..bdab71c99 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit 5b7e7743a1372b929acb2ccb75080e139ac11691 +Subproject commit bdab71c99a0a7cc268ca517a0cd3f0a5fb41042a diff --git a/scripts b/scripts index 47d9b3c93..792ae44ab 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 47d9b3c9390d21ecc49ddaba477b16071884ee87 +Subproject commit 792ae44ab14afd3a2b3e850a6fcb463d714b0a44 From d13e5a3f3157b02fd552a90c2722046745626791 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 29 Jun 2020 20:20:37 -0500 Subject: [PATCH 45/97] update structures handle renaming of caste_raw::gender --- library/xml | 2 +- plugins/remotefortressreader/remotefortressreader.cpp | 2 +- plugins/tweak/tweaks/tradereq-pet-gender.h | 2 +- scripts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/library/xml b/library/xml index 7dfea1f1d..b11433eea 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 7dfea1f1d4e453b97d5529785701db46360596ce +Subproject commit b11433eea0a1117edc2a4a51f7a30b01594cb52b diff --git a/plugins/remotefortressreader/remotefortressreader.cpp b/plugins/remotefortressreader/remotefortressreader.cpp index e94c3f832..1a0d045dc 100644 --- a/plugins/remotefortressreader/remotefortressreader.cpp +++ b/plugins/remotefortressreader/remotefortressreader.cpp @@ -2685,7 +2685,7 @@ static command_result GetPartialCreatureRaws(color_ostream &stream, const ListRe send_caste->add_child_name(orig_caste->child_name[0]); send_caste->add_child_name(orig_caste->child_name[1]); - send_caste->set_gender(orig_caste->gender); + send_caste->set_gender(orig_caste->sex); for (size_t partIndex = 0; partIndex < orig_caste->body_info.body_parts.size(); partIndex++) { diff --git a/plugins/tweak/tweaks/tradereq-pet-gender.h b/plugins/tweak/tweaks/tradereq-pet-gender.h index ca786bef2..c62a56f67 100644 --- a/plugins/tweak/tweaks/tradereq-pet-gender.h +++ b/plugins/tweak/tweaks/tradereq-pet-gender.h @@ -20,7 +20,7 @@ struct pet_gender_hook : df::viewscreen_topicmeeting_takerequestsst { vector& castes = entity->resources.animals.pet_castes; for (int i = (good_idx / 17) * 17, y = 4; i < (good_idx / 17) * 17 + 17 && size_t(i) < races.size(); i++, y++) { int x = 30 + 1 + world->raws.creatures.all[races[i]]->caste[castes[i]]->caste_name[0].size(); - bool male = (bool)world->raws.creatures.all[races[i]]->caste[castes[i]]->gender; + bool male = world->raws.creatures.all[races[i]]->caste[castes[i]]->sex == pronoun_type::he; OutputString((i == good_idx) ? COLOR_WHITE : COLOR_GREY, x, y, male ? "\013" : "\014"); } diff --git a/scripts b/scripts index 792ae44ab..91060d6d2 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 792ae44ab14afd3a2b3e850a6fcb463d714b0a44 +Subproject commit 91060d6d2c79196f6b332b9b88492eb0e7ef32da From 0f58d95ea890a7333131547bba95a43e582a3f78 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 22:02:53 -0400 Subject: [PATCH 46/97] Use workflow commands to set up environment variables https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable --- .github/workflows/build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 17af62cb1..7da428d32 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,11 +25,11 @@ jobs: submodules: true - name: Set up environment run: | - echo export DF_VERSION="$(sh travis/get-df-version.sh)" >> "$HOME/.df-env" - echo export DF_FOLDER="$HOME/DF/$DF_VERSION/df_linux" >> "$HOME/.df-env" + DF_VERSION="$(sh travis/get-df-version.sh)" + echo "::set-env name=DF_VERSION::${DF_VERSION}" + echo "::set-env name=DF_FOLDER::${HOME}/DF/${DF_VERSION}/df_linux" - name: Download DF run: | - source "$HOME/.df-env" sh travis/download-df.sh - name: Build docs run: | @@ -41,7 +41,6 @@ jobs: path: docs/html - name: Build DFHack run: | - source "$HOME/.df-env" cmake \ -S . \ -B build-ci \ @@ -53,7 +52,6 @@ jobs: ninja -C build-ci install - name: Run tests run: | - source "$HOME/.df-env" export TERM=dumb mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init script -qe -c "python travis/run-tests.py --headless --keep-status \"$DF_FOLDER\"" From c6283ea262f401845c2d0f6e645a7dc18fff79c3 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 22:04:33 -0400 Subject: [PATCH 47/97] Make download-df.sh fail sooner --- travis/download-df.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis/download-df.sh b/travis/download-df.sh index 834aa4da7..6918a596a 100755 --- a/travis/download-df.sh +++ b/travis/download-df.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -e + tardest="df.tar.bz2" which md5sum && alias md5=md5sum From 9def098ef914a9c8e0f7121889b17cf550e879a8 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 29 Jun 2020 21:30:22 -0500 Subject: [PATCH 48/97] add enum-attrs to pronoun_type --- library/xml | 2 +- plugins/manipulator.cpp | 4 +++- scripts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/library/xml b/library/xml index b11433eea..b67b3fa4b 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b11433eea0a1117edc2a4a51f7a30b01594cb52b +Subproject commit b67b3fa4b02f834dc76a2e1eae21136e0be089f7 diff --git a/plugins/manipulator.cpp b/plugins/manipulator.cpp index e9206dc0f..a013df335 100644 --- a/plugins/manipulator.cpp +++ b/plugins/manipulator.cpp @@ -2032,7 +2032,9 @@ void viewscreen_unitlaborsst::render() int x = 1, y = 3 + num_rows + 2; Screen::Pen white_pen(' ', 15, 0); - Screen::paintString(white_pen, x, y, (cur->unit && cur->unit->sex) ? "\x0b" : "\x0c"); + auto symbol = cur->unit ? ENUM_ATTR(pronoun_type, symbol, cur->unit->sex) : nullptr; + if (symbol) + Screen::paintString(white_pen, x, y, symbol); x += 2; Screen::paintString(white_pen, x, y, cur->transname); x += cur->transname.length(); diff --git a/scripts b/scripts index 91060d6d2..3987d0314 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 91060d6d2c79196f6b332b9b88492eb0e7ef32da +Subproject commit 3987d031469420d64aff23097d9548676f1ab752 From b4a55110e3a9572b32df10c841e7b4d06ad0a9da Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 22:35:23 -0400 Subject: [PATCH 49/97] Also try downloading DF from files.dfhack.org --- travis/download-df.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/travis/download-df.sh b/travis/download-df.sh index 6918a596a..49dcedea7 100755 --- a/travis/download-df.sh +++ b/travis/download-df.sh @@ -4,7 +4,6 @@ set -e tardest="df.tar.bz2" -which md5sum && alias md5=md5sum selfmd5=$(openssl md5 < "$0") echo $selfmd5 @@ -30,7 +29,20 @@ if [ ! -f receipt ]; then patch=$(echo "$DF_VERSION" | cut -d. -f3) url="http://www.bay12games.com/dwarves/df_${minor}_${patch}_linux.tar.bz2" echo Downloading - wget "$url" -O "$tardest" + while read url; do + echo "Attempting download: ${url}" + if wget -v "$url" -O "$tardest"; then + break + fi + done < Date: Mon, 29 Jun 2020 22:35:36 -0400 Subject: [PATCH 50/97] Attempt to cache DF --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7da428d32..6ad88dc27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,6 +28,11 @@ jobs: DF_VERSION="$(sh travis/get-df-version.sh)" echo "::set-env name=DF_VERSION::${DF_VERSION}" echo "::set-env name=DF_FOLDER::${HOME}/DF/${DF_VERSION}/df_linux" + - name: Fetch DF cache + uses: actions/cache@v2 + with: + path: ~/DF + key: ${DF_VERSION} - name: Download DF run: | sh travis/download-df.sh From e33a3ffff9f68a61aa39597bb8339d211e5574c6 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 22:40:58 -0400 Subject: [PATCH 51/97] Fix cache key --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ad88dc27..fdea2180e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,15 +24,17 @@ jobs: with: submodules: true - name: Set up environment + id: env_setup run: | DF_VERSION="$(sh travis/get-df-version.sh)" echo "::set-env name=DF_VERSION::${DF_VERSION}" + echo "::set-output name=df_version::${DF_VERSION}" echo "::set-env name=DF_FOLDER::${HOME}/DF/${DF_VERSION}/df_linux" - name: Fetch DF cache uses: actions/cache@v2 with: path: ~/DF - key: ${DF_VERSION} + key: ${{ steps.env_setup.outputs.df_version }} - name: Download DF run: | sh travis/download-df.sh From e26a091b7d02eacb3b0d4855ded6151ebff0fc01 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 22:52:44 -0400 Subject: [PATCH 52/97] Clean up cached DF --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fdea2180e..aadb1a15f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,12 @@ jobs: with: name: test-artifacts path: artifacts + - name: Clean up DF folder + # to prevent DFHack-generated files from ending up in the cache + # (download-df.sh also removes them, this is just to save cache space) + if: success() || failure() + run: | + rm -rf "$DF_FOLDER" lint: runs-on: ubuntu-18.04 steps: From c4aebfdf65acf63293648316144f7da2f11c5f58 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 22:53:05 -0400 Subject: [PATCH 53/97] Use setup-python --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aadb1a15f..4205e5e2a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,10 @@ jobs: build: runs-on: ubuntu-18.04 steps: + - name: Set up Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3 - name: Install dependencies run: | sudo apt-get update @@ -18,7 +22,7 @@ jobs: lua5.3 \ ninja-build \ zlib1g-dev - sudo pip3 install --system sphinx + pip install sphinx - name: Clone DFHack uses: actions/checkout@v1 with: From 6b2887d31b0e84b29b1121fabab96b5610e9678e Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 23:00:20 -0400 Subject: [PATCH 54/97] Upgrade to checkout@v2 --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4205e5e2a..612302397 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,8 +24,9 @@ jobs: zlib1g-dev pip install sphinx - name: Clone DFHack - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: + fetch-depth: 0 # unlimited - we need past tags submodules: true - name: Set up environment id: env_setup From e39150ac36934e54b5cdbdbdcb5c17e4f5faacf1 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 23:05:03 -0400 Subject: [PATCH 55/97] Update authors (dfhack/scripts#157) --- docs/Authors.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Authors.rst b/docs/Authors.rst index 6821a84b0..e816389cd 100644 --- a/docs/Authors.rst +++ b/docs/Authors.rst @@ -25,6 +25,7 @@ belal jimhester Ben Lubar BenLubar Ben Rosser TC01 billw2012 billw2012 +BrickViking brickviking brndd brndd burneddi Bumber Bumber64 Caldfir caldfir From e95edc095c395d4fca41dd3838dba5fccc8013a5 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 29 Jun 2020 23:58:47 -0400 Subject: [PATCH 56/97] Use workflow commands to annotate linter errors --- travis/lint.py | 83 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/travis/lint.py b/travis/lint.py index 7d8df7246..53abc4cbc 100644 --- a/travis/lint.py +++ b/travis/lint.py @@ -18,12 +18,49 @@ def valid_file(filename): not len(list(filter(lambda path: path.replace('\\', '/') in filename.replace('\\', '/'), path_blacklist))) success = True -def error(msg): +def error(msg=None): global success success = False - sys.stderr.write(msg + '\n') + if msg: + sys.stderr.write(msg + '\n') -class LinterError(Exception): pass +def format_lines(lines, total): + if len(lines) == total - 1: + return 'entire file' + if not len(lines): + # should never happen + return 'nowhere' + if len(lines) == 1: + return 'line %i' % lines[0] + s = 'lines ' + range_start = range_end = lines[0] + for i, line in enumerate(lines): + if line > range_end + 1: + if range_start == range_end: + s += ('%i, ' % range_end) + else: + s += ('%i-%i, ' % (range_start, range_end)) + range_start = range_end = line + if i == len(lines) - 1: + s += ('%i' % line) + else: + range_end = line + if i == len(lines) - 1: + s += ('%i-%i, ' % (range_start, range_end)) + return s.rstrip(' ').rstrip(',') + +class LinterError(Exception): + def __init__(self, message, lines, total_lines): + self.message = message + self.lines = lines + self.total_lines = total_lines + + def __str__(self): + return '%s: %s' % (self.message, format_lines(self.lines, self.total_lines)) + + def github_actions_workflow_command(self, filename): + first_line = self.lines[0] if self.lines else 1 + return '::error file=%s,line=%i::%s' % (filename, first_line, self) class Linter(object): ignore = False @@ -33,36 +70,12 @@ class Linter(object): if not self.check_line(line): failures.append(i + 1) if len(failures): - raise LinterError('%s: %s' % (self.msg, self.display_lines(failures, len(lines)))) + raise LinterError(self.msg, failures, len(lines)) def fix(self, lines): for i in range(len(lines)): lines[i] = self.fix_line(lines[i]) - def display_lines(self, lines, total): - if len(lines) == total - 1: - return 'entire file' - if not len(lines): - # should never happen - return 'nowhere' - if len(lines) == 1: - return 'line %i' % lines[0] - s = 'lines ' - range_start = range_end = lines[0] - for i, line in enumerate(lines): - if line > range_end + 1: - if range_start == range_end: - s += ('%i, ' % range_end) - else: - s += ('%i-%i, ' % (range_start, range_end)) - range_start = range_end = line - if i == len(lines) - 1: - s += ('%i' % line) - else: - range_end = line - if i == len(lines) - 1: - s += ('%i-%i, ' % (range_start, range_end)) - return s.rstrip(' ').rstrip(',') class NewlineLinter(Linter): msg = 'Contains DOS-style newlines' @@ -91,6 +104,7 @@ class TabLinter(Linter): linters = [cls() for cls in Linter.__subclasses__() if not cls.ignore] def main(): + is_github_actions = os.environ.get('GITHUB_ACTIONS') root_path = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else '.') if not os.path.exists(root_path): print('Nonexistent path: %s' % root_path) @@ -112,13 +126,22 @@ def main(): try: lines[i] = line.decode('utf-8') except UnicodeDecodeError: - error('%s:%i: Invalid UTF-8 (other errors will be ignored)' % (rel_path, i + 1)) + msg_params = (rel_path, i + 1, 'Invalid UTF-8 (other errors will be ignored)') + if is_github_actions: + error() + print('::error file=%s,line=%i::%s' % msg_params) + else: + error('%s:%i: %s' % msg_params) lines[i] = '' for linter in linters: try: linter.check(lines) except LinterError as e: - error('%s: %s' % (rel_path, e)) + if is_github_actions: + error() + print(e.github_actions_workflow_command(rel_path)) + else: + error('%s: %s' % (rel_path, e)) if fix: linter.fix(lines) contents = '\n'.join(lines) From fedf68cc7ccd0787ad7231085a9695c29844445d Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 00:24:03 -0400 Subject: [PATCH 57/97] Annotate script syntax errors --- .github/workflows/build.yml | 4 ++-- travis/script-syntax.py | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 612302397..d60a428bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -106,10 +106,10 @@ jobs: python travis/script-docs.py - name: Check Lua syntax run: | - python travis/script-syntax.py --ext=lua --cmd="luac5.3 -p" + python travis/script-syntax.py --ext=lua --cmd="luac5.3 -p" --github-actions - name: Check Ruby syntax run: | - python travis/script-syntax.py --ext=rb --cmd="ruby -c" + python travis/script-syntax.py --ext=rb --cmd="ruby -c" --github-actions check-pr: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/travis/script-syntax.py b/travis/script-syntax.py index b3a264f04..e33a11f1d 100644 --- a/travis/script-syntax.py +++ b/travis/script-syntax.py @@ -3,6 +3,24 @@ import os import subprocess import sys + +def print_stderr(stderr, args): + if not args.github_actions: + sys.stderr.write(stderr + '\n') + return + + for line in stderr.split('\n'): + parts = list(map(str.strip, line.split(':'))) + # e.g. luac prints "luac:" in front of messages, so find the first part + # containing the actual filename + for i in range(len(parts) - 1): + if parts[i].endswith('.' + args.ext) and parts[i + 1].isdigit(): + print('::error file=%s,line=%s::%s' % (parts[i], parts[i + 1], ':'.join(parts[i + 2:]))) + break + else: + print(line) + + def main(args): root_path = os.path.abspath(args.path) cmd = args.cmd.split(' ') @@ -19,7 +37,13 @@ def main(args): continue full_path = os.path.join(cur, filename) try: - subprocess.check_output(cmd + [full_path]) + p = subprocess.Popen(cmd + [full_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = p.communicate() + stderr = stderr.decode('utf-8', errors='ignore') + if stderr: + print_stderr(stderr, args) + if p.returncode != 0: + err = True except subprocess.CalledProcessError: err = True except IOError: @@ -34,5 +58,7 @@ if __name__ == '__main__': parser.add_argument('--path', default='.', help='Root directory') parser.add_argument('--ext', help='Script extension', required=True) parser.add_argument('--cmd', help='Command', required=True) + parser.add_argument('--github-actions', action='store_true', + help='Enable GitHub Actions workflow command output') args = parser.parse_args() main(args) From c333059316c2455416dbfacd1edf60477eb27a36 Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 00:29:30 -0400 Subject: [PATCH 58/97] Switch CI scripts to Python 3 --- .github/workflows/build.yml | 5 +++++ travis/all.py | 1 + travis/authors-rst.py | 1 + travis/check-rpc.py | 1 + travis/lint.py | 1 + travis/run-tests.py | 1 + travis/script-docs.py | 1 + travis/script-syntax.py | 1 + 8 files changed, 12 insertions(+) mode change 100644 => 100755 travis/all.py mode change 100644 => 100755 travis/authors-rst.py mode change 100644 => 100755 travis/check-rpc.py mode change 100644 => 100755 travis/lint.py mode change 100644 => 100755 travis/run-tests.py mode change 100644 => 100755 travis/script-docs.py mode change 100644 => 100755 travis/script-syntax.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d60a428bf..37a306abd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -82,9 +82,14 @@ jobs: if: success() || failure() run: | rm -rf "$DF_FOLDER" + lint: runs-on: ubuntu-18.04 steps: + - name: Set up Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3 - name: Install dependencies run: | sudo apt-get update diff --git a/travis/all.py b/travis/all.py old mode 100644 new mode 100755 index 281911e7a..edd2eee18 --- a/travis/all.py +++ b/travis/all.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse, os, sys, time parser = argparse.ArgumentParser() diff --git a/travis/authors-rst.py b/travis/authors-rst.py old mode 100644 new mode 100755 index 2d7274219..e8db817f6 --- a/travis/authors-rst.py +++ b/travis/authors-rst.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """ Overly-complicated script to check formatting/sorting in Authors.rst """ import re, sys diff --git a/travis/check-rpc.py b/travis/check-rpc.py old mode 100644 new mode 100755 index ecddafa9f..aba3e3811 --- a/travis/check-rpc.py +++ b/travis/check-rpc.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import glob import sys diff --git a/travis/lint.py b/travis/lint.py old mode 100644 new mode 100755 index 53abc4cbc..51117201c --- a/travis/lint.py +++ b/travis/lint.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import re, os, sys valid_extensions = ['c', 'cpp', 'h', 'hpp', 'mm', 'lua', 'rb', 'proto', diff --git a/travis/run-tests.py b/travis/run-tests.py old mode 100644 new mode 100755 index 70ab6cf20..f4c173af2 --- a/travis/run-tests.py +++ b/travis/run-tests.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import enum import json diff --git a/travis/script-docs.py b/travis/script-docs.py old mode 100644 new mode 100755 index 33b2818c8..04728e101 --- a/travis/script-docs.py +++ b/travis/script-docs.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from __future__ import print_function from io import open import os diff --git a/travis/script-syntax.py b/travis/script-syntax.py old mode 100644 new mode 100755 index e33a11f1d..1f6eca2eb --- a/travis/script-syntax.py +++ b/travis/script-syntax.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import os import subprocess From 324bc515408fec1b9d0ca848329e88ac956b51a9 Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 00:34:40 -0400 Subject: [PATCH 59/97] Upgrade to checkout@v2 --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37a306abd..829ab1c14 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,9 +97,10 @@ jobs: lua5.3 \ ruby - name: Clone DFHack - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: submodules: true + # don't need tags here - name: Check whitespace run: | python travis/lint.py From 504d2ec65b0d3a5823789519535ed7eb792153f0 Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 00:35:01 -0400 Subject: [PATCH 60/97] Stop linter steps from stopping each other --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 829ab1c14..3533bd1ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -105,17 +105,22 @@ jobs: run: | python travis/lint.py - name: Check Authors.rst + if: success() || failure() run: | python travis/authors-rst.py - name: Check for missing documentation + if: success() || failure() run: | python travis/script-docs.py - name: Check Lua syntax + if: success() || failure() run: | python travis/script-syntax.py --ext=lua --cmd="luac5.3 -p" --github-actions - name: Check Ruby syntax + if: success() || failure() run: | python travis/script-syntax.py --ext=rb --cmd="ruby -c" --github-actions + check-pr: runs-on: ubuntu-latest if: github.event_name == 'pull_request' From 1df7c0be122f1751b858b5c78a9992e86bdf16be Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 00:41:02 -0400 Subject: [PATCH 61/97] Upgrade Ruby --- .github/workflows/build.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3533bd1ee..fb98bfa0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,12 +90,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3 - - name: Install dependencies + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Install Lua run: | sudo apt-get update - sudo apt-get install \ - lua5.3 \ - ruby + sudo apt-get install lua5.3 - name: Clone DFHack uses: actions/checkout@v2 with: From 13f411d9fa650cc4ca443f606d2c7119fc5d59bf Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 00:44:27 -0400 Subject: [PATCH 62/97] Always print error lines to make GitHub logs more readable --- travis/script-syntax.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/travis/script-syntax.py b/travis/script-syntax.py index 1f6eca2eb..53891f762 100755 --- a/travis/script-syntax.py +++ b/travis/script-syntax.py @@ -18,8 +18,7 @@ def print_stderr(stderr, args): if parts[i].endswith('.' + args.ext) and parts[i + 1].isdigit(): print('::error file=%s,line=%s::%s' % (parts[i], parts[i + 1], ':'.join(parts[i + 2:]))) break - else: - print(line) + print(line) def main(args): From 20993f9a429a2011d1d696d504b8ea1e6fac0375 Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 30 Jun 2020 14:03:05 -0400 Subject: [PATCH 63/97] script-docs.py: add support for printing errors in GitHub actions format --- travis/lint.py | 2 +- travis/script-docs.py | 40 +++++++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/travis/lint.py b/travis/lint.py index 51117201c..e1db140c2 100755 --- a/travis/lint.py +++ b/travis/lint.py @@ -105,7 +105,7 @@ class TabLinter(Linter): linters = [cls() for cls in Linter.__subclasses__() if not cls.ignore] def main(): - is_github_actions = os.environ.get('GITHUB_ACTIONS') + is_github_actions = bool(os.environ.get('GITHUB_ACTIONS')) root_path = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else '.') if not os.path.exists(root_path): print('Nonexistent path: %s' % root_path) diff --git a/travis/script-docs.py b/travis/script-docs.py index 04728e101..b2c8063bf 100755 --- a/travis/script-docs.py +++ b/travis/script-docs.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -from __future__ import print_function -from io import open import os from os.path import basename, dirname, join, splitext import sys SCRIPT_PATH = sys.argv[1] if len(sys.argv) > 1 else 'scripts' +IS_GITHUB_ACTIONS = bool(os.environ.get('GITHUB_ACTIONS')) def expected_cmd(path): """Get the command from the name of a script.""" @@ -20,42 +19,61 @@ def check_ls(fname, line): line = line.strip() comment = '--' if fname.endswith('.lua') else '#' if '[====[' in line or not line.startswith(comment): - print('Error: no leading comment in ' + fname) + print_error('missing leading comment (requred for `ls`)', fname) return 1 return 0 +def print_error(message, filename, line=None): + if not isinstance(line, int): + line = 1 + print('Error: %s:%i: %s' % (filename, line, message)) + if IS_GITHUB_ACTIONS: + print('::error file=%s,line=%i::%s' % (filename, line, message)) + + def check_file(fname): errors, doclines = 0, [] tok1, tok2 = ('=begin', '=end') if fname.endswith('.rb') else \ ('[====[', ']====]') + doc_start_line = None with open(fname, errors='ignore') as f: lines = f.readlines() + if not lines: + print_error('empty file', fname) + return 1 errors += check_ls(fname, lines[0]) - for l in lines: + for i, l in enumerate(lines): if doclines or l.strip().endswith(tok1): + if not doclines: + doc_start_line = i + 1 doclines.append(l.rstrip()) if l.startswith(tok2): break else: if doclines: - print('Error: docs start but not end: ' + fname) + print_error('docs start but do not end', fname, doc_start_line) else: - print('Error: no documentation in: ' + fname) + print_error('no documentation found', fname) return 1 if not doclines: - print('Error: missing or malformed documentation in: ' + fname) + print_error('missing or malformed documentation', fname) return 1 title, underline = [d for d in doclines if d and '=begin' not in d and '[====[' not in d][:2] - if underline != '=' * len(title): - print('Error: title/underline mismatch:', fname, title, underline) + title_line = doc_start_line + doclines.index(title) + expected_underline = '=' * len(title) + if underline != expected_underline: + print_error('title/underline mismatch: expected {!r}, got {!r}'.format( + expected_underline, underline), + fname, title_line + 1) errors += 1 if title != expected_cmd(fname): - print('Warning: expected script title {}, got {}'.format( - expected_cmd(fname), title)) + print_error('expected script title {!r}, got {!r}'.format( + expected_cmd(fname), title), + fname, title_line) errors += 1 return errors From 142a7aa287daba0b1c652d67b94a6d4577d602de Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 1 Jul 2020 21:58:54 -0400 Subject: [PATCH 64/97] authors-rst.py: add support for printing errors in GitHub actions format --- travis/authors-rst.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travis/authors-rst.py b/travis/authors-rst.py index e8db817f6..076f7f226 100755 --- a/travis/authors-rst.py +++ b/travis/authors-rst.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Overly-complicated script to check formatting/sorting in Authors.rst """ -import re, sys +import os, re, sys def main(): success = [True] @@ -10,6 +10,8 @@ def main(): for k in kwargs: info += ' %s %s:' % (k, kwargs[k]) print('line %i:%s %s' % (line, info, msg)) + if os.environ.get('GITHUB_ACTIONS'): + print('::error file=docs/Authors.rst,line=%i::%s %s' % (line, info.lstrip(), msg)) success[0] = False with open('docs/Authors.rst', 'rb') as f: lines = list(map(lambda line: line.decode('utf8').replace('\n', ''), f.readlines())) From ae0f57f0e061d769e1189e58a54b4471ff4eb865 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 1 Jul 2020 23:54:31 -0400 Subject: [PATCH 65/97] Update submodules --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index b67b3fa4b..ec0f06754 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b67b3fa4b02f834dc76a2e1eae21136e0be089f7 +Subproject commit ec0f06754f0e03655611fe29d388f0a3ffad21e4 diff --git a/scripts b/scripts index 3987d0314..e4b5898ac 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 3987d031469420d64aff23097d9548676f1ab752 +Subproject commit e4b5898acb9cae806ef99416e1f9265df3abeeae From 27ee6a30d491773a423693cf4622590cd1c627dc Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 1 Jul 2020 23:56:41 -0400 Subject: [PATCH 66/97] Add additional path to strawberry perl docs From http://www.bay12forums.com/smf/index.php?topic=164123.msg8141934#msg8141934 --- docs/Compile.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index 650e762e3..5bb51a01f 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -545,23 +545,25 @@ to your binary search PATH so the tool can be later run from anywhere. Perl / Strawberry Perl ^^^^^^^^^^^^^^^^^^^^^^ -For the code generation parts you'll need Perl 5 with XML::LibXML and XML::LibXSLT. -`Strawberry Perl `_ is recommended as it includes -all of the required packages in a single, easy install. +For the code generation stage of the build process, you'll need Perl 5 with +XML::LibXML and XML::LibXSLT. `Strawberry Perl `_ is +recommended as it includes all of the required packages in a single, easy +install. After install, ensure Perl is in your user's PATH. This can be edited from ``Control Panel -> System -> Advanced System Settings -> Environment Variables``. -The following three directories must be in PATH, in this order: +The following directories must be in your PATH, in this order: * ``\c\bin`` * ``\perl\site\bin`` * ``\perl\bin`` +* ``\perl\vendor\lib\auto\XML\LibXML`` (may only be required on some systems) Be sure to close and re-open any existing ``cmd.exe`` windows after updating your PATH. -If you already have a different version of Perl (for example the one from Cygwin), +If you already have a different version of Perl installed (for example, from Cygwin), you can run into some trouble. Either remove the other Perl install from PATH, or install XML::LibXML and XML::LibXSLT for it using CPAN. From 72ce52e0044263e851dc59cca09dcd22e0324d8a Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Wed, 1 Jul 2020 23:10:13 -0500 Subject: [PATCH 67/97] update structures --- library/modules/Translation.cpp | 11 +---------- library/xml | 2 +- plugins/strangemood.cpp | 8 ++++---- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/library/modules/Translation.cpp b/library/modules/Translation.cpp index e05154b44..282039c02 100644 --- a/library/modules/Translation.cpp +++ b/library/modules/Translation.cpp @@ -69,16 +69,7 @@ bool Translation::copyName(df::language_name * source, df::language_name * targe if (source == target) return true; - target->first_name = source->first_name; - target->nickname = source->nickname; - for (int i = 0; i < 7; i++) - { - target->words[i] = source->words[i]; - target->parts_of_speech[i] = source->parts_of_speech[i]; - } - target->language = source->language; - target->unknown = source->unknown; - target->has_name = source->has_name; + *target = *source; return true; } diff --git a/library/xml b/library/xml index ec0f06754..b439f2287 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit ec0f06754f0e03655611fe29d388f0a3ffad21e4 +Subproject commit b439f22876634725f246703e3ac8a7a0b1a01d63 diff --git a/plugins/strangemood.cpp b/plugins/strangemood.cpp index e8a1046df..f37400cbb 100644 --- a/plugins/strangemood.cpp +++ b/plugins/strangemood.cpp @@ -153,7 +153,7 @@ void selectWord (const df::language_word_table &table, int32_t &word, df::part_o } } -void generateName(df::language_name &output, int language, int mode, const df::language_word_table &table1, const df::language_word_table &table2) +void generateName(df::language_name &output, int language, df::language_name_type mode, const df::language_word_table &table1, const df::language_word_table &table2) { for (int i = 0; i < 100; i++) { @@ -162,7 +162,7 @@ void generateName(df::language_name &output, int language, int mode, const df::l output = df::language_name(); if (language == -1) language = rng.df_trandom(world->raws.language.translations.size()); - output.unknown = mode; + output.type = mode; output.language = language; } output.has_name = 1; @@ -1316,10 +1316,10 @@ command_result df_strangemood (color_ostream &out, vector & parameters) // Generate the artifact's name if (type == mood_type::Fell || type == mood_type::Macabre) - generateName(unit->status.artifact_name, unit->name.language, 1, world->raws.language.word_table[0][2], world->raws.language.word_table[1][2]); + generateName(unit->status.artifact_name, unit->name.language, language_name_type::Artifact, world->raws.language.word_table[0][2], world->raws.language.word_table[1][2]); else { - generateName(unit->status.artifact_name, unit->name.language, 1, world->raws.language.word_table[0][1], world->raws.language.word_table[1][1]); + generateName(unit->status.artifact_name, unit->name.language, language_name_type::Artifact, world->raws.language.word_table[0][1], world->raws.language.word_table[1][1]); if (!rng.df_trandom(100)) unit->status.artifact_name = unit->name; } From 25f1b14a8882ee7cabaaccb31ca8bfe903972543 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Wed, 1 Jul 2020 23:12:45 -0500 Subject: [PATCH 68/97] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index b439f2287..2d67b03a1 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b439f22876634725f246703e3ac8a7a0b1a01d63 +Subproject commit 2d67b03a1c5c701e65fb483b9d6c1d3067f990cb From 30c8db811dc2c5ee99b857200460b52afaadd8c5 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 3 Jul 2020 16:50:52 -0400 Subject: [PATCH 69/97] Add note referring to gui/liquids, update scripts (docs) --- docs/Plugins.rst | 6 +++++- scripts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/Plugins.rst b/docs/Plugins.rst index 421e5e81b..5c855ddd3 100644 --- a/docs/Plugins.rst +++ b/docs/Plugins.rst @@ -2356,11 +2356,15 @@ dfhack command line and can't be used from a hotkey. Settings will be remembered as long as dfhack runs. Intended for use in combination with the command ``liquids-here`` (which can be bound to a hotkey). See also :issue:`80`. -.. note:: +.. warning:: Spawning and deleting liquids can mess up pathing data and temperatures (creating heat traps). You've been warned. +.. note:: + + `gui/liquids` is an in-game UI for this script. + Settings will be remembered until you quit DF. You can call `liquids-here` to execute the last configured action, which is useful in combination with keybindings. diff --git a/scripts b/scripts index e4b5898ac..d265de74d 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit e4b5898acb9cae806ef99416e1f9265df3abeeae +Subproject commit d265de74d4c28460961f8e646b1b574997756ea2 From 9a3747d307e48f053e246c6bf8f23d00a0da1b67 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 3 Jul 2020 20:42:23 -0400 Subject: [PATCH 70/97] Make sidebar fixed --- conf.py | 1 + docs/styles/dfhack.css | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/conf.py b/conf.py index c36e4f8aa..9f497fb57 100644 --- a/conf.py +++ b/conf.py @@ -297,6 +297,7 @@ html_theme_options = { 'github_repo': 'dfhack', 'github_button': False, 'travis_button': False, + 'fixed_sidebar': True, } # The name for this set of Sphinx documents. If None, it defaults to diff --git a/docs/styles/dfhack.css b/docs/styles/dfhack.css index d12348c31..bfe37720d 100644 --- a/docs/styles/dfhack.css +++ b/docs/styles/dfhack.css @@ -46,3 +46,12 @@ div.body p, div.body dd, div.body li, div.body blockquote { -webkit-hyphens: manual; hyphens: manual; } + +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} + +div.sphinxsidebar h3 > a { + border-bottom: 1px dotted #999; +} From fa910cd74f45f3fd1838dc3c5c8c11b3b3b0a8c3 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 3 Jul 2020 22:43:21 -0400 Subject: [PATCH 71/97] Include custom CSS separately instead of overriding theme CSS May help with https://github.com/DFHack/dfhack/issues/1578#issuecomment-635512324 --- conf.py | 7 +++++-- docs/styles/dfhack.css | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf.py b/conf.py index 9f497fb57..b3f2b5848 100644 --- a/conf.py +++ b/conf.py @@ -187,7 +187,7 @@ all_keybinds_documented() # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.3' +needs_sphinx = '1.8' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -286,7 +286,6 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'alabaster' -html_style = 'dfhack.css' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -337,6 +336,10 @@ html_domain_indices = False # If false, no index is generated. html_use_index = False +html_css_files = [ + 'dfhack.css', +] + # -- Options for LaTeX output --------------------------------------------- # Grouping the document tree into LaTeX files. List of tuples diff --git a/docs/styles/dfhack.css b/docs/styles/dfhack.css index bfe37720d..e7a608460 100644 --- a/docs/styles/dfhack.css +++ b/docs/styles/dfhack.css @@ -1,6 +1,3 @@ -/* make sure to sync this with the base theme's css filename */ -@import url("alabaster.css"); - /* Keybinding CSS from the DF wiki; applies to :kbd:`` directives. * Use this directive for all keypresses, to make them look like keys. */ From f32757fa0910c178bf3174c1d9daf03de3a86c49 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 3 Jul 2020 22:49:48 -0400 Subject: [PATCH 72/97] Update scripts --- plugins/buildingplan-lib.cpp | 1 - scripts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/buildingplan-lib.cpp b/plugins/buildingplan-lib.cpp index 4e7cfd3fd..0c29284db 100644 --- a/plugins/buildingplan-lib.cpp +++ b/plugins/buildingplan-lib.cpp @@ -9,7 +9,6 @@ void debug(const string &msg) color_ostream_proxy out(Core::getInstance().getConsole()); out << "DEBUG (" << PLUGIN_VERSION << "): " << msg << endl; } -#define dbg Core::getInstance().getConsole() void enable_quickfort_fn(pair& pair) { pair.second = true; } diff --git a/scripts b/scripts index d265de74d..e8de92efb 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit d265de74d4c28460961f8e646b1b574997756ea2 +Subproject commit e8de92efb73d5ef4d0b52df000d60d3350f07a37 From f81f77352cbca122cf72c589b2f5d160e0db660a Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 4 Jul 2020 02:15:05 -0400 Subject: [PATCH 73/97] Restructure changelog generation as a Sphinx extension --- conf.py | 10 ++++++---- docs/sphinx_extensions/dfhack/__init__.py | 0 .../dfhack/changelog.py} | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 docs/sphinx_extensions/dfhack/__init__.py rename docs/{gen_changelog.py => sphinx_extensions/dfhack/changelog.py} (97%) diff --git a/conf.py b/conf.py index b3f2b5848..47d1c0c58 100644 --- a/conf.py +++ b/conf.py @@ -22,8 +22,6 @@ import re import shlex # pylint:disable=unused-import import sys -sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs')) -from gen_changelog import generate_changelog # -- Support :dfhack-keybind:`command` ------------------------------------ # this is a custom directive that pulls info from dfhack.init-example @@ -180,19 +178,23 @@ def all_keybinds_documented(): # Actually call the docs generator and run test -generate_changelog() write_script_docs() all_keybinds_documented() # -- General configuration ------------------------------------------------ +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs', 'sphinx_extensions')) + # If your documentation needs a minimal Sphinx version, state it here. needs_sphinx = '1.8' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.extlinks'] +extensions = [ + 'sphinx.ext.extlinks', + 'dfhack.changelog', +] # This config value must be a dictionary of external sites, mapping unique # short alias names to a base URL and a prefix. diff --git a/docs/sphinx_extensions/dfhack/__init__.py b/docs/sphinx_extensions/dfhack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/gen_changelog.py b/docs/sphinx_extensions/dfhack/changelog.py similarity index 97% rename from docs/gen_changelog.py rename to docs/sphinx_extensions/dfhack/changelog.py index 8b1a6771e..4c0f30427 100644 --- a/docs/gen_changelog.py +++ b/docs/sphinx_extensions/dfhack/changelog.py @@ -283,3 +283,17 @@ if __name__ == '__main__': print('stable missing: ' + description) if description not in content_dev: print('dev missing: ' + description) + + +def sphinx_entrypoint(app, config): + generate_changelog() + + +def setup(app): + app.connect('config-inited', sphinx_entrypoint) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } From b0fe96f6c7c81c800e7cf2e9df148637feeb6392 Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 4 Jul 2020 02:35:55 -0400 Subject: [PATCH 74/97] Add gen_changelog.py compatibility shim, fix paths when run under cmake --- docs/gen_changelog.py | 5 +++++ docs/sphinx_extensions/dfhack/changelog.py | 19 ++++++++++--------- docs/sphinx_extensions/dfhack/util.py | 7 +++++++ 3 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 docs/gen_changelog.py create mode 100644 docs/sphinx_extensions/dfhack/util.py diff --git a/docs/gen_changelog.py b/docs/gen_changelog.py new file mode 100644 index 000000000..3b4452f62 --- /dev/null +++ b/docs/gen_changelog.py @@ -0,0 +1,5 @@ +import os, sys +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sphinx_extensions')) + +from dfhack.changelog import cli_entrypoint +cli_entrypoint() diff --git a/docs/sphinx_extensions/dfhack/changelog.py b/docs/sphinx_extensions/dfhack/changelog.py index 4c0f30427..99f0ea299 100644 --- a/docs/sphinx_extensions/dfhack/changelog.py +++ b/docs/sphinx_extensions/dfhack/changelog.py @@ -4,11 +4,14 @@ import itertools import os import sys +from dfhack.util import DFHACK_ROOT, DOCS_ROOT + CHANGELOG_PATHS = ( 'docs/changelog.txt', 'scripts/changelog.txt', 'library/xml/changelog.txt', ) +CHANGELOG_PATHS = (os.path.join(DFHACK_ROOT, p) for p in CHANGELOG_PATHS) CHANGELOG_SECTIONS = [ 'New Plugins', @@ -237,8 +240,8 @@ def generate_changelog(all=False): consolidate_changelog(stable_entries) consolidate_changelog(dev_entries) - print_changelog(versions, stable_entries, 'docs/_auto/news.rst') - print_changelog(versions, dev_entries, 'docs/_auto/news-dev.rst') + print_changelog(versions, stable_entries, os.path.join(DOCS_ROOT, '_auto/news.rst')) + print_changelog(versions, dev_entries, os.path.join(DOCS_ROOT, '_auto/news-dev.rst')) if all: for version in versions: @@ -250,16 +253,16 @@ def generate_changelog(all=False): else: version_entries = {version: dev_entries[version]} print_changelog([version], version_entries, - 'docs/_changelogs/%s-github.txt' % version, + os.path.join(DOCS_ROOT, '_changelogs/%s-github.txt' % version), replace=False) print_changelog([version], version_entries, - 'docs/_changelogs/%s-reddit.txt' % version, + os.path.join(DOCS_ROOT, '_changelogs/%s-reddit.txt' % version), replace=False, prefix='> ') return entries -if __name__ == '__main__': +def cli_entrypoint(): import argparse parser = argparse.ArgumentParser() parser.add_argument('-a', '--all', action='store_true', @@ -268,14 +271,12 @@ if __name__ == '__main__': help='Check that all entries are printed') args = parser.parse_args() - os.chdir(os.path.abspath(os.path.dirname(__file__))) - os.chdir('..') entries = generate_changelog(all=args.all) if args.check: - with open('docs/_auto/news.rst') as f: + with open(os.path.join(DOCS_ROOT, '_auto/news.rst')) as f: content_stable = f.read() - with open('docs/_auto/news-dev.rst') as f: + with open(os.path.join(DOCS_ROOT, '_auto/news-dev.rst')) as f: content_dev = f.read() for entry in entries: for description in entry.children: diff --git a/docs/sphinx_extensions/dfhack/util.py b/docs/sphinx_extensions/dfhack/util.py new file mode 100644 index 000000000..71a432da4 --- /dev/null +++ b/docs/sphinx_extensions/dfhack/util.py @@ -0,0 +1,7 @@ +import os + +DFHACK_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +DOCS_ROOT = os.path.join(DFHACK_ROOT, 'docs') + +if not os.path.isdir(DOCS_ROOT): + raise ReferenceError('docs root not found: %s' % DOCS_ROOT) From b122870b6a0e5b43d962c6c9be810139e0a9aa60 Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 7 Jul 2020 01:54:59 -0400 Subject: [PATCH 75/97] Move documentation standards to Documentation.rst --- Contributing.rst | 89 +------------------------------ docs/Documentation.rst | 115 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 104 insertions(+), 100 deletions(-) diff --git a/Contributing.rst b/Contributing.rst index e358e66b3..cb9e87a0e 100644 --- a/Contributing.rst +++ b/Contributing.rst @@ -32,6 +32,7 @@ How to get new code into DFHack (i.e. not the master or develop branch of your fork). * If possible, compile on multiple platforms when changing anything that compiles * It must pass CI - run ``python travis/all.py`` to check this. +* Update documentation when applicable - see `docs-standards` for details. * Update ``changelog.txt`` and ``docs/Authors.rst`` when applicable. See `build-changelog` for more information on the changelog format. * Create a GitHub pull request once finished @@ -126,94 +127,6 @@ know what exactly is most useful. `remotefortressreader` provides a fairly comprehensive interface for visualisers such as :forums:`Armok Vision <146473>`. -Documentation Standards -======================= -DFHack documentation is built with Sphinx_, and configured automatically -through CMake. If you want to build the docs *only*, use this command:: - - sphinx-build . docs/html - -Whether you're adding new code or just fixing old documentation (and there's plenty), -there are a few important standards for completeness and consistent style. Treat -this section as a guide rather than iron law, match the surrounding text, and you'll -be fine. - -Each command should have a short (~54 character) help string, which is shown -by the `ls` command. For scripts, this is a comment on the first line -(the comment marker and whitespace is stripped). For plugins it's the second -argument to ``PluginCommand``. Please make this brief but descriptive! - -Everything should be documented! If it's not clear *where* a particular -thing should be documented, ask on IRC or in the DFHack thread on Bay12 - -as well as getting help, you'll be providing valuable feedback that -makes it easier for future readers! - -Scripts can use a custom autodoc function, based on the Sphinx ``include`` -directive - anything between the tokens is copied into the appropriate scripts -documentation page. For Ruby, we follow the built-in docstring convention -(``=begin`` and ``=end``). For Lua, the tokens are ``[====[`` and ``]====]`` -- ordinary multi-line strings. It is highly encouraged to reuse this string -as the in-console documentation by (eg.) printing it when a ``-help`` argument -is given. - -The docs **must** have a heading which exactly matches the command, underlined -with ``=====`` to the same length. For example, a lua file would have:: - - local helpstr = [====[ - - add-thought - =========== - Adds a thought or emotion to the selected unit. Can be used by other scripts, - or the gui invoked by running ``add-thought gui`` with a unit selected. - - ]====] - - -Where the heading for a section is also the name of a command, the spelling -and case should exactly match the command to enter in the DFHack command line. - -Try to keep lines within 80-100 characters, so it's readable in plain text -in the terminal - Sphinx (our documentation system) will make sure -paragraphs flow. - -If there aren't many options or examples to show, they can go in a paragraph of -text. Use double-backticks to put commands in monospaced font, like this:: - - You can use ``cleanowned scattered x`` to dump tattered or abandoned items. - -If the command takes more than three arguments, format the list as a table -called Usage. The table *only* lists arguments, not full commands. -Input values are specified in angle brackets. Example:: - - Usage: - - :arg1: A simple argument. - :arg2 : Does something based on the input value. - :Very long argument: - Is very specific. - -To demonstrate usage - useful mainly when the syntax is complicated, list the -full command with arguments in monospaced font, then indent the next line and -describe the effect:: - - ``resume all`` - Resumes all suspended constructions. - -If it would be helpful to mention another DFHack command, don't just type the -name - add a hyperlink! Specify the link target in backticks, and it will be -replaced with the corresponding title and linked: eg ```autolabor``` -=> `autolabor`. Link targets should be equivalent to the command -described (without file extension), and placed above the heading of that -section like this:: - - .. _autolabor: - - autolabor - ========= - -Add link targets if you need them, but otherwise plain headings are preferred. -Scripts have link targets created automatically. - Other ways to help ================== DFHack is a software project, but there's a lot more to it than programming. diff --git a/docs/Documentation.rst b/docs/Documentation.rst index f7cb9ef92..2fa32e16c 100644 --- a/docs/Documentation.rst +++ b/docs/Documentation.rst @@ -28,21 +28,118 @@ The main thing you lose in plain text format is hyperlinking.) .. contents:: +.. _docs-standards: + +Documentation standards +======================= + +Whether you're adding new code or just fixing old documentation (and there's plenty), +there are a few important standards for completeness and consistent style. Treat +this section as a guide rather than iron law, match the surrounding text, and you'll +be fine. + +Each command should have a short (~54 character) help string, which is shown +by the `ls` command. For scripts, this is a comment on the first line +(the comment marker and whitespace is stripped). For plugins it's the second +argument to ``PluginCommand``. Please make this brief but descriptive! + +Everything should be documented! If it's not clear *where* a particular +thing should be documented, ask on IRC or in the DFHack thread on Bay12 - +as well as getting help, you'll be providing valuable feedback that +makes it easier for future readers! + +Scripts can use a custom autodoc function, based on the Sphinx ``include`` +directive - anything between the tokens is copied into the appropriate scripts +documentation page. For Ruby, we follow the built-in docstring convention +(``=begin`` and ``=end``). For Lua, the tokens are ``[====[`` and ``]====]`` +- ordinary multi-line strings. It is highly encouraged to reuse this string +as the in-console documentation by (eg.) printing it when a ``-help`` argument +is given. + +The docs **must** have a heading which exactly matches the command, underlined +with ``=====`` to the same length. For example, a lua file would have:: + + local helpstr = [====[ + + add-thought + =========== + Adds a thought or emotion to the selected unit. Can be used by other scripts, + or the gui invoked by running ``add-thought gui`` with a unit selected. + + ]====] + + +Where the heading for a section is also the name of a command, the spelling +and case should exactly match the command to enter in the DFHack command line. + +Try to keep lines within 80-100 characters, so it's readable in plain text +in the terminal - Sphinx (our documentation system) will make sure +paragraphs flow. + +If there aren't many options or examples to show, they can go in a paragraph of +text. Use double-backticks to put commands in monospaced font, like this:: + + You can use ``cleanowned scattered x`` to dump tattered or abandoned items. + +If the command takes more than three arguments, format the list as a table +called Usage. The table *only* lists arguments, not full commands. +Input values are specified in angle brackets. Example:: + + Usage: + + :arg1: A simple argument. + :arg2 : Does something based on the input value. + :Very long argument: + Is very specific. + +To demonstrate usage - useful mainly when the syntax is complicated, list the +full command with arguments in monospaced font, then indent the next line and +describe the effect:: + + ``resume all`` + Resumes all suspended constructions. + +If it would be helpful to mention another DFHack command, don't just type the +name - add a hyperlink! Specify the link target in backticks, and it will be +replaced with the corresponding title and linked: eg ```autolabor``` +=> `autolabor`. Link targets should be equivalent to the command +described (without file extension), and placed above the heading of that +section like this:: + + .. _autolabor: + + autolabor + ========= + +Add link targets if you need them, but otherwise plain headings are preferred. +Scripts have link targets created automatically. + + +Building the documentation +========================== + Required dependencies -===================== +--------------------- In order to build the documentation, you must have Python with Sphinx -version 1.3.1 or later. Both Python 2.x and 3.x are supported. +version 1.8 or later. Both Python 2.x and 3.x are supported. When installing Sphinx from OS package managers, be aware that there is another program called Sphinx, completely unrelated to documentation management. Be sure you are installing the right Sphinx; it may be called ``python-sphinx``, for example. To avoid doubt, ``pip`` can be used instead as detailed below. +Once you have installed Sphinx, ``sphinx-build --version`` should report the +version of Sphinx that you have installed. If this works, CMake should also be +able to find Sphinx. + For more detailed platform-specific instructions, see the sections below: +.. contents:: + :local: + Linux ------ +~~~~~ Most Linux distributions will include Python as standard. Check your package manager to see if Sphinx 1.3.1 or later is available, @@ -66,7 +163,7 @@ install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH macOS ------ +~~~~~ macOS has Python 2.7 installed by default, but it does not have the pip package manager. You can install Homebrew's Python 3, which includes pip, and then install the @@ -87,7 +184,7 @@ If not, just installing sphinx-doc for macOS's system Python 2.7 is fine. Windows -------- +~~~~~~~ Python for Windows can be downloaded `from python.org `_. The latest version of Python 3 is recommended, as it includes pip already. @@ -105,13 +202,6 @@ Note that this may require opening a new (admin) command prompt if you just installed pip from the same command prompt. -Building the documentation -========================== - -Once you have installed Sphinx, ``sphinx-build --version`` should report the -version of Sphinx that you have installed. If this works, CMake should also be -able to find Sphinx. - Using CMake ----------- @@ -155,6 +245,7 @@ the root DFHack folder:: Sphinx has many options to enable clean builds, parallel builds, logging, and more - run ``sphinx-build --help`` for details. + .. _build-changelog: Building the changelogs From 512f425c65cd5394a7e36f7c49b048249ecd450e Mon Sep 17 00:00:00 2001 From: lethosor Date: Tue, 7 Jul 2020 23:49:28 -0400 Subject: [PATCH 76/97] Move Contributing.rst to docs/, add GitHub-specific version in .github/ --- .github/CONTRIBUTING.md | 6 ++++++ Contributing.rst => docs/Contributing.rst | 0 index.rst | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .github/CONTRIBUTING.md rename Contributing.rst => docs/Contributing.rst (100%) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000..2367db678 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,6 @@ +Thank you for your interest in contributing to DFHack! If you're reading this +document, you're probably viewing it on GitHub. The DFHack docs are hosted +on [ReadTheDocs](https://dfhack.readthedocs.io/) - in particular, contributing +guidelines are [here](https://docs.dfhack.org/en/latest/docs/Contributing.html). +Double-checking the style guidelines before submitting a pull request is +always appreciated. diff --git a/Contributing.rst b/docs/Contributing.rst similarity index 100% rename from Contributing.rst rename to docs/Contributing.rst diff --git a/index.rst b/index.rst index 855b31a73..8c1e9d86d 100644 --- a/index.rst +++ b/index.rst @@ -58,7 +58,7 @@ For Developers .. toctree:: :maxdepth: 1 - /Contributing + /docs/Contributing /docs/Compile /docs/NEWS-dev /docs/Lua API From 7f56faa64a63c043be96c572d8792624cb982807 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 00:08:47 -0400 Subject: [PATCH 77/97] Tweak Documentation.rst TOC --- docs/Documentation.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/Documentation.rst b/docs/Documentation.rst index 2fa32e16c..cbbf8e153 100644 --- a/docs/Documentation.rst +++ b/docs/Documentation.rst @@ -1,8 +1,8 @@ .. _documentation: -#################### -DFHack Documentation -#################### +########################### +DFHack Documentation System +########################### DFHack documentation, like the file you are reading now, is created as ``.rst`` files, @@ -136,6 +136,7 @@ For more detailed platform-specific instructions, see the sections below: .. contents:: :local: + :backlinks: none Linux From aff2944f28418bb71095daf57159d7146d9e4f62 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 00:12:05 -0400 Subject: [PATCH 78/97] Add "scripts-" prefix to generated scripts subpage anchors --- conf.py | 2 +- docs/Core.rst | 2 +- docs/Introduction.rst | 2 +- index.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf.py b/conf.py index 47d1c0c58..1c5ef9853 100644 --- a/conf.py +++ b/conf.py @@ -151,7 +151,7 @@ def write_script_docs(): 'gui': 'GUI Scripts', 'modtools': 'Scripts for Modders'} for k in head: - title = ('.. _{k}:\n\n{l}\n{t}\n{l}\n\n' + title = ('.. _scripts-{k}:\n\n{l}\n{t}\n{l}\n\n' '.. include:: /scripts/{a}about.txt\n\n' '.. contents::\n\n').format( k=k, t=head[k], diff --git a/docs/Core.rst b/docs/Core.rst index 4e65f4dda..9cad075f4 100644 --- a/docs/Core.rst +++ b/docs/Core.rst @@ -408,7 +408,7 @@ All matching init files will be executed in alphebetical order. A world being loaded can mean a fortress, an adventurer, or legends mode. These files are best used for non-persistent commands, such as setting -a `fix ` script to run on `repeat`. +a `fix ` script to run on `repeat`. .. _onUnload.init: diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 22033d528..52675d364 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -20,7 +20,7 @@ enhancements by default, and more can be enabled. There are also many tools You can even add third-party scripts and plugins to do almost anything! For modders, DFHack makes many things possible. Custom reactions, new -interactions, magic creature abilities, and more can be set through `modtools` +interactions, magic creature abilities, and more can be set through `scripts-modtools` and custom raws. Non-standard DFHack scripts and inits can be stored in the raw directory, making raws or saves fully self-contained for distribution - or for coexistence in a single DF install, even with incompatible components. diff --git a/index.rst b/index.rst index 8c1e9d86d..b3498d432 100644 --- a/index.rst +++ b/index.rst @@ -20,7 +20,7 @@ enhancements by default, and more can be enabled. There are also many tools You can even add third-party scripts and plugins to do almost anything! For modders, DFHack makes many things possible. Custom reactions, new -interactions, magic creature abilities, and more can be set through `modtools` +interactions, magic creature abilities, and more can be set through `scripts-modtools` and custom raws. Non-standard DFHack scripts and inits can be stored in the raw directory, making raws or saves fully self-contained for distribution - or for coexistence in a single DF install, even with incompatible components. From 85003a40c2f89693faed8af9185571e0b9a8194f Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 01:03:58 -0400 Subject: [PATCH 79/97] Split memory research into separate doc and expand --- docs/Binpatches.rst | 2 +- docs/Compile.rst | 2 + docs/Contributing.rst | 20 ------ docs/Memory-research.rst | 149 +++++++++++++++++++++++++++++++++++++++ index.rst | 1 + 5 files changed, 153 insertions(+), 21 deletions(-) create mode 100644 docs/Memory-research.rst diff --git a/docs/Binpatches.rst b/docs/Binpatches.rst index a0fcc6630..40ae8af32 100644 --- a/docs/Binpatches.rst +++ b/docs/Binpatches.rst @@ -24,7 +24,7 @@ There are no binary patches available for Dwarf Fortress versions after 0.34.11. This system is kept for the chance that someone will find it useful, so some hints on how to write your own follow. This will require disassembly and -decent skill in `memory research `. +decent skill in `memory research `. * The patches are expected to be encoded in text format used by IDA. diff --git a/docs/Compile.rst b/docs/Compile.rst index 5bb51a01f..c8c51e88c 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -161,6 +161,8 @@ in any case. Note that the scripts in the "build" folder on Windows will set the architecture automatically. +.. _compile-build-options: + Other settings -------------- There are a variety of other settings which you can find in CMakeCache.txt in diff --git a/docs/Contributing.rst b/docs/Contributing.rst index cb9e87a0e..6231b6f04 100644 --- a/docs/Contributing.rst +++ b/docs/Contributing.rst @@ -41,26 +41,6 @@ How to get new code into DFHack * Work on :issue:`reported problems ` will take priority over ideas or suggestions. -.. _contributing-memory-research: - -Memory research ---------------- -If you want to do memory research, you'll need some tools and some knowledge. -In general, you'll need a good memory viewer and optionally something -to look at machine code without getting crazy :) -Using publicly known information and analyzing the game's data is preferred. - -Good Windows tools include: - -* IDA Freeware 7.0 (for non-commercial use, supports 32-bit and 64-bit) -* Cheat Engine - -Good Linux tools: - -* angavrilov's df-structures gui (32-bit only, visit us on IRC for details) -* IDA Freeware 7.0 (see above) -* edb (Evan's Debugger) -* Some of the tools residing in the ``legacy`` dfhack branch. Using the library as a developer ================================ diff --git a/docs/Memory-research.rst b/docs/Memory-research.rst new file mode 100644 index 000000000..08c0c9573 --- /dev/null +++ b/docs/Memory-research.rst @@ -0,0 +1,149 @@ +.. _memory-research: + +############### +Memory research +############### + +There are a variety of tools that can be used to analyze DF memory - some are +listed here. Note that some of these may be old and unmaintained. If you aren't +sure what tool would be best for your purposes, feel free to ask for advice (on +IRC, Bay12, etc.). + +.. contents:: + :local: + + +Cross-platform tools +==================== + +Ghidra +------ + +Ghidra is a cross-platform reverse-engineering framework (written in Java) +available at https://ghidra-sre.org. It supports analyzing both 32-bit and +64-bit executables for all supported DF platforms. There are some custom DFHack +Ghidra scripts available in the `df_misc`_ repo (look for ``.java`` files). + + +IDA Freeware 7.0 +---------------- + +Available from `Hex-Rays `_. +Supports analyzing both 32-bit and 64-bit executables for all supported DF platforms. +Some ``.idc`` scripts for IDA are available in the `df_misc`_ repo. + +.. _df_misc: https://github.com/DFHack/df_misc + + +Hopper +------ + +Runs on macOS and some Linux distributions; available from https://www.hopperapp.com/. +`TWBT `_ uses this to produce some patches. + + +DFHack tools +------------ + +Plugins +~~~~~~~ + +There are a few development plugins useful for low-level memory research. They +are not built by default, but can be built by setting the ``BUILD_DEVEL`` +`CMake option `. These include: + +- ``check-structures-sanity``, which performs sanity checks on the given DF + object. Note that this will crash in several cases, some intentional, so using + this with `GDB ` is recommended. +- ``memview``, which produces a hex dump of a given memory range. It also + highlights valid pointers, and can be configured to work with `sizecheck` + to auto-detect object sizes. +- ``vectors``, which can identify instances of ``std::vector`` in a given memory range. + +Scripts +~~~~~~~ + +Several `development scripts ` can be useful for memory research. +These include (but are not limited to): + +- `devel/dump-offsets` +- `devel/find-offsets` +- `devel/lsmem` +- `devel/sc` (requires `sizecheck`) +- `devel/visualize-structure` +- Generally, any script starting with ``devel/find`` + +.. _sizecheck: + +Sizecheck +~~~~~~~~~ + +Sizecheck is a custom tool that hooks into the memory allocator and inserts a +header indicating the size of every object. The corresponding logic to check for +this header when freeing memory usually works, but is inherently not foolproof. +You should not count on DF being stable when using this. + +DFHack's implementation of sizecheck is currently only tested on Linux, although +it probably also works on macOS. It can be built with the ``BUILD_SIZECHECK`` +`CMake option `, which produces a ``libsizecheck`` +library installed in the ``hack`` folder. You will need to preload this library +manually, by setting ``PRELOAD_LIB`` on Linux (or ``LD_PRELOAD`` if editing +the ``dfhack`` launcher script directly), or by editing the ``dfhack`` +launcher script and adding the library to ``DYLD_INSERT_LIBRARIES`` on macOS. + +There is also an older sizecheck implementation by Mifki available on +`GitHub `__ (``b.cpp`` is the main +sizecheck library, and ``win_patch.cpp`` is used for Windows support). To use +this with other DFHack tools, you will likely need to edit the header's +magic number to match what is used in `devel/sc` (search for a hexadecimal +constant starting with ``0x``). + +Legacy tools +~~~~~~~~~~~~ + +Some very old DFHack tools are available in the `legacy branch on GitHub `_. +No attempt is made to support these. + + +Linux-specific tools +==================== + +.. _linux-gdb: + +GDB +--- + +`GDB `_ is technically cross-platform, but +tends to work best on Linux, and DFHack currently only offers support for using +GDB on 64-bit Linux. To start with GDB, pass ``-g`` to the DFHack launcher +script:: + + ./dfhack -g + +Some basic GDB commands: + +- ``run``: starts DF from the GDB prompt. Any arguments will be passed as + command-line arguments to DF (e.g. `load-save` may be useful). +- ``bt`` will produce a backtrace if DF crashes. + +See the `official GDB documentation `_ +for more details. + + +df-structures GUI +----------------- + +This is a tool written by Angavrilov and available on `GitHub `__. +It only supports 32-bit DF. Some assistance may be available on IRC. + + +EDB (Evan's debugger) +--------------------- + +Available on `GitHub `__. + + +Windows-specific tools +====================== + +Some people have used `Cheat Engine `__ for research in the past. diff --git a/index.rst b/index.rst index b3498d432..4c5b2e6e4 100644 --- a/index.rst +++ b/index.rst @@ -65,4 +65,5 @@ For Developers /docs/Documentation /library/xml/SYNTAX /library/xml/how-to-update + /docs/Memory-research /docs/Binpatches From 24d64175fb7f9c0c4e148841abee69fd905a09e7 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 02:13:37 -0400 Subject: [PATCH 80/97] Remove top-level links from TOCs --- conf.py | 3 ++- docs/Binpatches.rst | 3 ++- docs/Compile.rst | 5 +++-- docs/Contributing.rst | 4 ++-- docs/Core.rst | 5 +++-- docs/Documentation.rst | 4 ++-- docs/History.rst | 19 +++++++++++-------- docs/Introduction.rst | 3 ++- docs/Lua API.rst | 5 +++-- docs/Memory-research.rst | 4 ++-- docs/NEWS-dev.rst | 5 +++-- docs/NEWS.rst | 9 +++++---- docs/Plugins.rst | 5 +++-- docs/Scripts-removed.rst | 5 +++-- library/xml | 2 +- 15 files changed, 47 insertions(+), 34 deletions(-) diff --git a/conf.py b/conf.py index 1c5ef9853..aebe73bab 100644 --- a/conf.py +++ b/conf.py @@ -153,7 +153,8 @@ def write_script_docs(): for k in head: title = ('.. _scripts-{k}:\n\n{l}\n{t}\n{l}\n\n' '.. include:: /scripts/{a}about.txt\n\n' - '.. contents::\n\n').format( + '.. contents:: Contents\n' + ' :local:\n\n').format( k=k, t=head[k], l=len(head[k])*'#', a=('' if k == 'base' else k + '/') diff --git a/docs/Binpatches.rst b/docs/Binpatches.rst index 40ae8af32..b92190bc8 100644 --- a/docs/Binpatches.rst +++ b/docs/Binpatches.rst @@ -15,7 +15,8 @@ the `binpatch` command. at all possible - that way your work will work for many versions across multiple operating systems. -.. contents:: +.. contents:: Contents + :local: Getting a patch diff --git a/docs/Compile.rst b/docs/Compile.rst index c8c51e88c..26f720076 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -13,8 +13,9 @@ DFHack from source, this document will walk you through the build process. Note that some steps may be unconventional compared to other projects, so be sure to pay close attention if this is your first time compiling DFHack. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. _compile-how-to-get-the-code: diff --git a/docs/Contributing.rst b/docs/Contributing.rst index 6231b6f04..28e4e6e95 100644 --- a/docs/Contributing.rst +++ b/docs/Contributing.rst @@ -2,8 +2,8 @@ How to contribute to DFHack ########################### -.. contents:: - +.. contents:: Contents + :local: .. _contributing-code: diff --git a/docs/Core.rst b/docs/Core.rst index 9cad075f4..8f84c7b54 100644 --- a/docs/Core.rst +++ b/docs/Core.rst @@ -4,8 +4,9 @@ DFHack Core ########### -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 2 Command Implementation diff --git a/docs/Documentation.rst b/docs/Documentation.rst index cbbf8e153..8a829c0a8 100644 --- a/docs/Documentation.rst +++ b/docs/Documentation.rst @@ -25,8 +25,8 @@ compile the documentation in order to read it. Like Markdown, reST documents are designed to be just as readable in a plain-text editor as they are in HTML format. The main thing you lose in plain text format is hyperlinking.) -.. contents:: - +.. contents:: Contents + :local: .. _docs-standards: diff --git a/docs/History.rst b/docs/History.rst index 8386c4cab..d382b351e 100644 --- a/docs/History.rst +++ b/docs/History.rst @@ -1,16 +1,19 @@ :orphan: -.. _HISTORY: +.. _History: -######################## -HISTORY - old changelogs -######################## +##################### +Historical changelogs +##################### -This file is where old changelogs live, so the current `changelog` -in ``NEWS.rst`` doesn't get too long. +This file is where old changelogs live, so the `current changelog ` +doesn't get too long. Some of these changelogs are also formatted differently +from current changelogs and would be difficult for the current `changelog +generation system ` to handle. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 DFHack 0.43.05-r3 ================= diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 52675d364..dc05c8725 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -30,7 +30,8 @@ allows easier development of new tools. As an open-source project under `various copyleft licences `, contributions are welcome. -.. contents:: +.. contents:: Contents + :local: .. _installing: diff --git a/docs/Lua API.rst b/docs/Lua API.rst index d7e94b92a..3e0f22116 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -24,8 +24,9 @@ implemented by Lua files located in :file:`hack/lua/*` (:file:`library/lua/*` in the git repo). -.. contents:: - :depth: 3 +.. contents:: Contents + :local: + :depth: 2 ========================= diff --git a/docs/Memory-research.rst b/docs/Memory-research.rst index 08c0c9573..41aa437c8 100644 --- a/docs/Memory-research.rst +++ b/docs/Memory-research.rst @@ -9,8 +9,8 @@ listed here. Note that some of these may be old and unmaintained. If you aren't sure what tool would be best for your purposes, feel free to ask for advice (on IRC, Bay12, etc.). -.. contents:: - :local: +.. contents:: Contents + :local: Cross-platform tools diff --git a/docs/NEWS-dev.rst b/docs/NEWS-dev.rst index 7a9da7cb7..f0c47fe96 100644 --- a/docs/NEWS-dev.rst +++ b/docs/NEWS-dev.rst @@ -13,7 +13,8 @@ which they first appeared. See `build-changelog` for more information. See `changelog` for a list of changes grouped by stable releases. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. include:: /docs/_auto/news-dev.rst diff --git a/docs/NEWS.rst b/docs/NEWS.rst index 22f9881b9..1283b079a 100644 --- a/docs/NEWS.rst +++ b/docs/NEWS.rst @@ -13,14 +13,15 @@ appeared. See `build-changelog` for more information. See `dev-changelog` for a list of changes grouped by development releases. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. include:: /docs/_auto/news.rst Older Changelogs ================ -Are kept in a seperate file: `HISTORY` +Are kept in a seperate file: `History` -.. that's ``docs/history.rst``, if you're reading the raw text. +.. that's ``docs/History.rst``, if you're reading the raw text. diff --git a/docs/Plugins.rst b/docs/Plugins.rst index 5c855ddd3..0bd699e14 100644 --- a/docs/Plugins.rst +++ b/docs/Plugins.rst @@ -9,8 +9,9 @@ game subsystems or the entire renderer. Most commands offered by plugins are listed here, hopefully organised in a way you will find useful. -.. contents:: - :depth: 3 +.. contents:: Contents + :local: + :depth: 2 =============================== Data inspection and visualizers diff --git a/docs/Scripts-removed.rst b/docs/Scripts-removed.rst index 4490110ba..5fa3d3ec4 100644 --- a/docs/Scripts-removed.rst +++ b/docs/Scripts-removed.rst @@ -4,8 +4,9 @@ Removed scripts The following scripts were removed for various reasons. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. _warn-stuck-trees: diff --git a/library/xml b/library/xml index 2d67b03a1..847ec0c2d 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 2d67b03a1c5c701e65fb483b9d6c1d3067f990cb +Subproject commit 847ec0c2d4dc74a887e1f980eeed6f97345c195e From 858e90e6c2d06b8e4b81201273699b18878b056e Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 21:04:23 -0400 Subject: [PATCH 81/97] Move Scripts-removed.rst to Removed.rst and update a bit --- docs/Removed.rst | 17 +++++++++++++++++ docs/Scripts-removed.rst | 15 --------------- index.rst | 2 +- 3 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 docs/Removed.rst delete mode 100644 docs/Scripts-removed.rst diff --git a/docs/Removed.rst b/docs/Removed.rst new file mode 100644 index 000000000..f98915bdd --- /dev/null +++ b/docs/Removed.rst @@ -0,0 +1,17 @@ +############# +Removed tools +############# + +This page lists tools (plugins or scripts) that were previously included in +DFHack but have been removed. It exists primarily so that internal links still +work (e.g. links from the `changelog`). + +.. contents:: Contents + :local: + :depth: 1 + +.. _warn-stuck-trees: + +warn-stuck-trees +================ +The corresponding DF bug, :bug:`9252` was fixed in DF 0.44.01. diff --git a/docs/Scripts-removed.rst b/docs/Scripts-removed.rst deleted file mode 100644 index 5fa3d3ec4..000000000 --- a/docs/Scripts-removed.rst +++ /dev/null @@ -1,15 +0,0 @@ -############### -Removed scripts -############### - -The following scripts were removed for various reasons. - -.. contents:: Contents - :local: - :depth: 1 - -.. _warn-stuck-trees: - -warn-stuck-trees -================ -The corresponding DF bug, :bug:`9252` was fixed in DF 0.44.01. diff --git a/index.rst b/index.rst index 4c5b2e6e4..10424f5fe 100644 --- a/index.rst +++ b/index.rst @@ -50,7 +50,7 @@ Other Contents /docs/Authors /LICENSE /docs/NEWS - /docs/Scripts-removed + /docs/Removed For Developers ============== From 87fd32d330a50b126ec8ea3a15145e982fbd0064 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 22:52:20 -0400 Subject: [PATCH 82/97] Move general dev info to a separate doc (Dev-intro) --- docs/Contributing.rst | 65 ------------------------------------------ docs/Dev-intro.rst | 66 +++++++++++++++++++++++++++++++++++++++++++ docs/index-dev.rst | 20 +++++++++++++ index.rst | 17 +---------- 4 files changed, 87 insertions(+), 81 deletions(-) create mode 100644 docs/Dev-intro.rst create mode 100644 docs/index-dev.rst diff --git a/docs/Contributing.rst b/docs/Contributing.rst index 28e4e6e95..7ce092a8e 100644 --- a/docs/Contributing.rst +++ b/docs/Contributing.rst @@ -42,71 +42,6 @@ How to get new code into DFHack will take priority over ideas or suggestions. -Using the library as a developer -================================ -Currently, the most direct way to use the library is to write a script or plugin that can be loaded by it. -All the plugins can be found in the 'plugins' folder. There's no in-depth documentation -on how to write one yet, but it should be easy enough to copy one and just follow the pattern. -``plugins/skeleton/skeleton.cpp`` is provided for this purpose. - -Other than through plugins, it is possible to use DFHack via remote access interface, -or by writing scripts in Lua or Ruby. There are plenty of examples in the scripts folder. -The `lua-api` is quite well documented. - -The most important parts of DFHack are the Core, Console, Modules and Plugins. - -* Core acts as the centerpiece of DFHack - it acts as a filter between DF and - SDL and synchronizes the various plugins with DF. -* Console is a thread-safe console that can be used to invoke commands exported by Plugins. -* Modules actually describe the way to access information in DF's memory. You - can get them from the Core. Most modules are split into two parts: high-level - and low-level. High-level is mostly method calls, low-level publicly visible - pointers to DF's data structures. -* Plugins are the tools that use all the other stuff to make things happen. - A plugin can have a list of commands that it exports and an onupdate function - that will be called each DF game tick. - -Rudimentary API documentation can be built using doxygen (see build options -in ``CMakeCache.txt`` or with ``ccmake`` or ``cmake-gui``). The full DFHack -documentation is built with Sphinx_, which runs automatically at compile time. - -.. _Sphinx: http://www.sphinx-doc.org - -DFHack consists of variously licensed code, but invariably weak copyleft. -The main license is zlib/libpng, some bits are MIT licensed, and some are -BSD licensed. See the `license` for more information. - -Feel free to add your own extensions and plugins. Contributing back to -the DFHack repository is welcome and the right thing to do :) - -DF data structure definitions ------------------------------ -DFHack uses information about the game data structures, represented via xml files -in the ``library/xml/`` submodule. - -See https://github.com/DFHack/df-structures, and the documentation linked in the index. - -Data structure layouts are described in files following the ``df.*.xml`` name pattern. -This information is transformed by a perl script into C++ headers describing the -structures, and associated metadata for the Lua wrapper. These headers and data -are then compiled into the DFHack libraries, thus necessitating a compatibility -break every time layouts change; in return it significantly boosts the efficiency -and capabilities of DFHack code. - -Global object addresses are stored in :file:`symbols.xml`, which is copied to the dfhack -release package and loaded as data at runtime. - -Remote access interface ------------------------ -DFHack supports remote access by exchanging Google protobuf messages via a TCP -socket. Both the core and plugins can define remotely accessible methods. The -``dfhack-run`` command uses this interface to invoke ordinary console commands. - -Currently the supported set of requests is limited, because the developers don't -know what exactly is most useful. `remotefortressreader` provides a fairly -comprehensive interface for visualisers such as :forums:`Armok Vision <146473>`. - - Other ways to help ================== DFHack is a software project, but there's a lot more to it than programming. diff --git a/docs/Dev-intro.rst b/docs/Dev-intro.rst new file mode 100644 index 000000000..e43f03659 --- /dev/null +++ b/docs/Dev-intro.rst @@ -0,0 +1,66 @@ +=========================== +DFHack development overview +=========================== + +Currently, the most direct way to use the library is to write a script or plugin that can be loaded by it. +All the plugins can be found in the 'plugins' folder. There's no in-depth documentation +on how to write one yet, but it should be easy enough to copy one and just follow the pattern. +``plugins/skeleton/skeleton.cpp`` is provided for this purpose. + +Other than through plugins, it is possible to use DFHack via remote access interface, +or by writing scripts in Lua or Ruby. There are plenty of examples in the scripts folder. +The `lua-api` is quite well documented. + +The most important parts of DFHack are the Core, Console, Modules and Plugins. + +* Core acts as the centerpiece of DFHack - it acts as a filter between DF and + SDL and synchronizes the various plugins with DF. +* Console is a thread-safe console that can be used to invoke commands exported by Plugins. +* Modules actually describe the way to access information in DF's memory. You + can get them from the Core. Most modules are split into two parts: high-level + and low-level. High-level is mostly method calls, low-level publicly visible + pointers to DF's data structures. +* Plugins are the tools that use all the other stuff to make things happen. + A plugin can have a list of commands that it exports and an onupdate function + that will be called each DF game tick. + +Rudimentary API documentation can be built using doxygen (see build options +in ``CMakeCache.txt`` or with ``ccmake`` or ``cmake-gui``). The full DFHack +documentation is built with Sphinx_, which runs automatically at compile time. + +.. _Sphinx: http://www.sphinx-doc.org + +DFHack consists of variously licensed code, but invariably weak copyleft. +The main license is zlib/libpng, some bits are MIT licensed, and some are +BSD licensed. See the `license` for more information. + +Feel free to add your own extensions and plugins. Contributing back to +the DFHack repository is welcome and the right thing to do :) + +DF data structure definitions +----------------------------- +DFHack uses information about the game data structures, represented via xml files +in the ``library/xml/`` submodule. + +See https://github.com/DFHack/df-structures, and the documentation linked in the index. + +Data structure layouts are described in files following the ``df.*.xml`` name pattern. +This information is transformed by a perl script into C++ headers describing the +structures, and associated metadata for the Lua wrapper. These headers and data +are then compiled into the DFHack libraries, thus necessitating a compatibility +break every time layouts change; in return it significantly boosts the efficiency +and capabilities of DFHack code. + +Global object addresses are stored in :file:`symbols.xml`, which is copied to the dfhack +release package and loaded as data at runtime. + +Remote access interface +----------------------- +DFHack supports remote access by exchanging Google protobuf messages via a TCP +socket. Both the core and plugins can define remotely accessible methods. The +``dfhack-run`` command uses this interface to invoke ordinary console commands. + +Currently the supported set of requests is limited, because the developers don't +know what exactly is most useful. `remotefortressreader` provides a fairly +comprehensive interface for visualisers such as :forums:`Armok Vision <146473>`. + diff --git a/docs/index-dev.rst b/docs/index-dev.rst new file mode 100644 index 000000000..9134ff6b5 --- /dev/null +++ b/docs/index-dev.rst @@ -0,0 +1,20 @@ +======================== +DFHack Development Guide +======================== + +These are pages relevant to people developing for DFHack. + +.. toctree:: + :maxdepth: 1 + + /docs/Dev-intro + /docs/Contributing + /docs/Compile + /docs/NEWS-dev + /docs/Lua API + /docs/Documentation + /library/xml/SYNTAX + /library/xml/how-to-update + /docs/Memory-research + /docs/Binpatches + diff --git a/index.rst b/index.rst index 10424f5fe..df28d0778 100644 --- a/index.rst +++ b/index.rst @@ -40,6 +40,7 @@ User Manual /docs/Core /docs/Plugins /docs/Scripts + /docs/index-dev Other Contents ============== @@ -51,19 +52,3 @@ Other Contents /LICENSE /docs/NEWS /docs/Removed - -For Developers -============== - -.. toctree:: - :maxdepth: 1 - - /docs/Contributing - /docs/Compile - /docs/NEWS-dev - /docs/Lua API - /docs/Documentation - /library/xml/SYNTAX - /library/xml/how-to-update - /docs/Memory-research - /docs/Binpatches From 108e100d2206cd7710d4c2a8074b42d8264a6a58 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 23:10:55 -0400 Subject: [PATCH 83/97] Move structures overview to Structures-intro.rst and update/elaborate --- docs/Dev-intro.rst | 16 ---------------- docs/Structures-intro.rst | 34 ++++++++++++++++++++++++++++++++++ docs/index-dev.rst | 3 +-- 3 files changed, 35 insertions(+), 18 deletions(-) create mode 100644 docs/Structures-intro.rst diff --git a/docs/Dev-intro.rst b/docs/Dev-intro.rst index e43f03659..6983693e8 100644 --- a/docs/Dev-intro.rst +++ b/docs/Dev-intro.rst @@ -37,22 +37,6 @@ BSD licensed. See the `license` for more information. Feel free to add your own extensions and plugins. Contributing back to the DFHack repository is welcome and the right thing to do :) -DF data structure definitions ------------------------------ -DFHack uses information about the game data structures, represented via xml files -in the ``library/xml/`` submodule. - -See https://github.com/DFHack/df-structures, and the documentation linked in the index. - -Data structure layouts are described in files following the ``df.*.xml`` name pattern. -This information is transformed by a perl script into C++ headers describing the -structures, and associated metadata for the Lua wrapper. These headers and data -are then compiled into the DFHack libraries, thus necessitating a compatibility -break every time layouts change; in return it significantly boosts the efficiency -and capabilities of DFHack code. - -Global object addresses are stored in :file:`symbols.xml`, which is copied to the dfhack -release package and loaded as data at runtime. Remote access interface ----------------------- diff --git a/docs/Structures-intro.rst b/docs/Structures-intro.rst new file mode 100644 index 000000000..c45d03eaf --- /dev/null +++ b/docs/Structures-intro.rst @@ -0,0 +1,34 @@ +=================================== +DF data definitions (DF-structures) +=================================== + +DFHack's information about DF's data structures is stored in XML files in the +`df-structures repository `_. If you +have `obtained a local copy of the DFHack source `, +DF-structures is included as a submodule in ``library/xml``. + +Data structure layouts are described in files named with the ``df.*.xml`` +pattern. This information is transformed by a Perl script (``codegen.pl``) into +C++ headers, as well as metadata for the Lua wrapper. This ultimately allows +DFHack code to access DF data directly, with the same speed and capabilities as +DF itself, which is an advantage over the older out-of-process approach (used +by debuggers and utilities like Dwarf Therapist). The main disadvantage of this +approach is that any compiled code relying on these layouts will break when DF's +layout changes, and will need to be recompiled for every new DF version. + +Addresses of DF global objects and vtables are stored in a separate file, +:file:`symbols.xml`. Since these are only absolute addresses, they do not need +to be compiled in to DFHack code, and are instead loaded at runtime. This makes +fixes and additions to global addresses possible without recompiling DFHack. +In an installed copy of DFHack, this file can be found at the root of the +``hack`` folder. + +The following pages contain more detailed information about various aspects +of DF-structures: + +.. toctree:: + :titlesonly: + + /library/xml/SYNTAX + /library/xml/how-to-update + diff --git a/docs/index-dev.rst b/docs/index-dev.rst index 9134ff6b5..617c0c0d7 100644 --- a/docs/index-dev.rst +++ b/docs/index-dev.rst @@ -13,8 +13,7 @@ These are pages relevant to people developing for DFHack. /docs/NEWS-dev /docs/Lua API /docs/Documentation - /library/xml/SYNTAX - /library/xml/how-to-update + /docs/Structures-intro /docs/Memory-research /docs/Binpatches From 7e620b09a44b21f4a107925314a410c87262ba1b Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 23:38:18 -0400 Subject: [PATCH 84/97] Update dev-intro Doxygen was removed in 787c54d8a --- docs/Contributing.rst | 2 + docs/Dev-intro.rst | 85 +++++++++++++++++++++++++++++-------------- docs/Lua API.rst | 2 + 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/docs/Contributing.rst b/docs/Contributing.rst index 7ce092a8e..ab92d0571 100644 --- a/docs/Contributing.rst +++ b/docs/Contributing.rst @@ -1,3 +1,5 @@ +.. _contributing: + ########################### How to contribute to DFHack ########################### diff --git a/docs/Dev-intro.rst b/docs/Dev-intro.rst index 6983693e8..b1961378c 100644 --- a/docs/Dev-intro.rst +++ b/docs/Dev-intro.rst @@ -2,41 +2,72 @@ DFHack development overview =========================== -Currently, the most direct way to use the library is to write a script or plugin that can be loaded by it. -All the plugins can be found in the 'plugins' folder. There's no in-depth documentation -on how to write one yet, but it should be easy enough to copy one and just follow the pattern. -``plugins/skeleton/skeleton.cpp`` is provided for this purpose. +DFHack has various components; this page provides an overview of some. If you +are looking to develop a tool for DFHack, developing a script or plugin is +likely the most straightforward choice. -Other than through plugins, it is possible to use DFHack via remote access interface, -or by writing scripts in Lua or Ruby. There are plenty of examples in the scripts folder. -The `lua-api` is quite well documented. +Other pages that may be relevant include: -The most important parts of DFHack are the Core, Console, Modules and Plugins. +- `contributing` +- `documentation` +- `license` -* Core acts as the centerpiece of DFHack - it acts as a filter between DF and - SDL and synchronizes the various plugins with DF. -* Console is a thread-safe console that can be used to invoke commands exported by Plugins. -* Modules actually describe the way to access information in DF's memory. You - can get them from the Core. Most modules are split into two parts: high-level - and low-level. High-level is mostly method calls, low-level publicly visible - pointers to DF's data structures. -* Plugins are the tools that use all the other stuff to make things happen. - A plugin can have a list of commands that it exports and an onupdate function - that will be called each DF game tick. -Rudimentary API documentation can be built using doxygen (see build options -in ``CMakeCache.txt`` or with ``ccmake`` or ``cmake-gui``). The full DFHack -documentation is built with Sphinx_, which runs automatically at compile time. +.. contents:: Contents + :local: -.. _Sphinx: http://www.sphinx-doc.org -DFHack consists of variously licensed code, but invariably weak copyleft. -The main license is zlib/libpng, some bits are MIT licensed, and some are -BSD licensed. See the `license` for more information. +Plugins +------- -Feel free to add your own extensions and plugins. Contributing back to -the DFHack repository is welcome and the right thing to do :) +DFHack plugins are written in C++ and located in the ``plugins`` folder. +Currently, documentation on how to write plugins is somewhat sparse. There are +templates that you can get use to get started in the ``plugins/skeleton`` +folder, and the source code of existing plugins can also be helpful. +If you want to compile a plugin that you have just added, you will need to add a +call to ``DFHACK_PLUGIN`` in ``plugins/CMakeLists.txt``. + +Plugins have the ability to make one or more commands available to users of the +DFHack console. Examples include `3dveins` (which implements the ``3dveins`` +command) and `reveal` (which implements ``reveal``, ``unreveal``, and several +other commands). + +Plugins can also register handlers to run on every tick, and can interface with +the built-in `enable` and `disable` commands. For the full plugin API, see the +skeleton plugins or ``PluginManager.cpp``. + +Scripts +------- + +DFHack scripts can currently be written in Lua or Ruby. The `Lua API ` +is more complete and currently better-documented, however. Referring to existing +scripts as well as the API documentation can be helpful when developing new +scripts. + +DFHack scripts live in a separate `scripts repository `_. +This can be found in the ``scripts`` submodule if you have +`cloned DFHack `, or the ``hack/scripts`` folder +of an installed copy of DFHack. + +Core +---- + +The DFHack core has a variety of low-level functions. It is responsible for +hooking into DF (via SDL), providing a console, and providing an interface +for plugins and scripts to interact with DF. + +Modules +------- + +A lot of shared code to interact with DF in more complicated ways is exposed by +**modules**. For example, the Units module contains functions for checking +various traits of units, changing nicknames properly, and more. Generally, +code that is useful to multiple plugins and scripts should go in the appropriate +module, if there is one. + +Several modules are also `exposed to Lua `, although +some functions (and some entire modules) are currently only available in C++. Remote access interface ----------------------- diff --git a/docs/Lua API.rst b/docs/Lua API.rst index 3e0f22116..4c7163623 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -802,6 +802,8 @@ Random number generation Dimension may be 1, 2 or 3 (default). +.. _lua-cpp-func-wrappers: + C++ function wrappers ===================== From ba0f8553ed5afde61ea9bec1360a089c7fcebcba Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 8 Jul 2020 23:46:38 -0400 Subject: [PATCH 85/97] Misc updates, links --- docs/Core.rst | 12 +++++++++--- docs/Dev-intro.rst | 22 ++++++++++++++-------- docs/Plugins.rst | 2 ++ docs/Scripts.rst | 2 ++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/Core.rst b/docs/Core.rst index 8f84c7b54..6ef1d0575 100644 --- a/docs/Core.rst +++ b/docs/Core.rst @@ -166,10 +166,13 @@ right place to do it. Most such plugins or scripts support the built-in ``enable`` and ``disable`` commands. Calling them at any time without arguments prints a list of enabled and disabled plugins, and shows whether that can be changed -through the same commands. +through the same commands. Passing plugin names to these commands will enable +or disable the specified plugins. For example, to enable the `manipulator` +plugin:: -To enable or disable plugins that support this, use their names as -arguments for the command:: + enable manipulator + +It is also possible to enable or disable multiple plugins at once:: enable manipulator search @@ -272,6 +275,9 @@ something. Usage:: Allows dealing with plugins individually by name, or all at once. +Note that plugins do not maintain their enabled state if they are reloaded, so +you may need to use `enable` to re-enable a plugin after reloading it. + .. _ls: diff --git a/docs/Dev-intro.rst b/docs/Dev-intro.rst index b1961378c..9a08533c0 100644 --- a/docs/Dev-intro.rst +++ b/docs/Dev-intro.rst @@ -37,6 +37,12 @@ Plugins can also register handlers to run on every tick, and can interface with the built-in `enable` and `disable` commands. For the full plugin API, see the skeleton plugins or ``PluginManager.cpp``. +Installed plugins live in the ``hack/plugins`` folder of a DFHack installation, +and the `load` family of commands can be used to load a recompiled plugin +without restarting DF. + +See `plugins-index` for a list of all plugins included in DFHack. + Scripts ------- @@ -45,7 +51,7 @@ is more complete and currently better-documented, however. Referring to existing scripts as well as the API documentation can be helpful when developing new scripts. -DFHack scripts live in a separate `scripts repository `_. +`Scripts included in DFHack ` live in a separate `scripts repository `_. This can be found in the ``scripts`` submodule if you have `cloned DFHack `, or the ``hack/scripts`` folder of an installed copy of DFHack. @@ -53,17 +59,17 @@ of an installed copy of DFHack. Core ---- -The DFHack core has a variety of low-level functions. It is responsible for -hooking into DF (via SDL), providing a console, and providing an interface -for plugins and scripts to interact with DF. +The `DFHack core ` has a variety of low-level functions. It is +responsible for hooking into DF (via SDL), providing a console, and providing an +interface for plugins and scripts to interact with DF. Modules ------- -A lot of shared code to interact with DF in more complicated ways is exposed by -**modules**. For example, the Units module contains functions for checking -various traits of units, changing nicknames properly, and more. Generally, -code that is useful to multiple plugins and scripts should go in the appropriate +A lot of shared code to interact with DF in more complicated ways is contained +in **modules**. For example, the Units module contains functions for checking +various traits of units, changing nicknames properly, and more. Generally, code +that is useful to multiple plugins and scripts should go in the appropriate module, if there is one. Several modules are also `exposed to Lua `, although diff --git a/docs/Plugins.rst b/docs/Plugins.rst index 0bd699e14..0185dbee5 100644 --- a/docs/Plugins.rst +++ b/docs/Plugins.rst @@ -1,3 +1,5 @@ +.. _plugins-index: + ############## DFHack Plugins ############## diff --git a/docs/Scripts.rst b/docs/Scripts.rst index 0e6ef415c..419385582 100644 --- a/docs/Scripts.rst +++ b/docs/Scripts.rst @@ -1,3 +1,5 @@ +.. _scripts-index: + ############## DFHack Scripts ############## From fdb4487a3632e11b37d82efb8caf78fe12631484 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 10 Jul 2020 02:11:03 -0400 Subject: [PATCH 86/97] Update Linux sphinx installation instructions, define substitution for minumum sphinx version --- conf.py | 7 +++++++ docs/Documentation.rst | 36 +++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/conf.py b/conf.py index aebe73bab..e70d836f1 100644 --- a/conf.py +++ b/conf.py @@ -283,6 +283,13 @@ pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False +rst_prolog = """ +.. |sphinx_min_version| replace:: {sphinx_min_version} +.. |dfhack_version| replace:: {dfhack_version} +""".format( + sphinx_min_version=needs_sphinx, + dfhack_version=version, +) # -- Options for HTML output ---------------------------------------------- diff --git a/docs/Documentation.rst b/docs/Documentation.rst index 8a829c0a8..462932b0f 100644 --- a/docs/Documentation.rst +++ b/docs/Documentation.rst @@ -121,7 +121,7 @@ Building the documentation Required dependencies --------------------- In order to build the documentation, you must have Python with Sphinx -version 1.8 or later. Both Python 2.x and 3.x are supported. +version |sphinx_min_version| or later. Both Python 2.x and 3.x are supported. When installing Sphinx from OS package managers, be aware that there is another program called Sphinx, completely unrelated to documentation management. @@ -141,27 +141,29 @@ For more detailed platform-specific instructions, see the sections below: Linux ~~~~~ -Most Linux distributions will include Python as standard. +Most Linux distributions will include Python by default. If not, start by +installing Python (preferably Python 3). On Debian-based distros:: -Check your package manager to see if Sphinx 1.3.1 or later is available, -but at the time of writing Ubuntu for example only has 1.2.x. + sudo apt install python3 -You can instead install Sphinx with the pip package manager. This may need -to be installed from your OS package manager; this is the case on Ubuntu. -On Ubuntu/Debian, use the following to first install pip:: +Check your package manager to see if Sphinx |sphinx_min_version| or later is +available. On Debian-based distros, this package is named ``python3-sphinx``. +If this package is new enough, you can install it directly. If not, or if you +want to use a newer Sphinx version (which may result in faster builds), you +can install Sphinx through the ``pip`` package manager instead. On Debian-based +distros, you can install pip with:: - sudo apt-get install python-pip + sudo apt install python3-pip -Once pip is available, you can then install the Python Sphinx module with:: +Once pip is available, you can then install Sphinx with:: - pip install sphinx - -If you run this as a normal user it will install a local copy for your user only. -Run it with sudo if you want a system-wide install. Either is fine for DFHack, -however if installing locally do check that ``sphinx-build`` is in your path. -It may be installed in a directory such as ``~/.local/bin/``, so after pip -install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``. + pip3 install sphinx +If you run this as an unpriviliged user, it may install a local copy of Sphinx +for your user only. The ``sphinx-build`` executable will typically end up in +``~/.local/bin/`` in this case. Alternatively, you can install Sphinx +system-wide by running pip with ``sudo``. In any case, you will need the folder +containing ``sphinx-build`` to be in your ``$PATH``. macOS ~~~~~ @@ -173,7 +175,7 @@ latest Sphinx using pip:: brew install python3 pip3 install sphinx -Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew:: +Alternatively, you can simply install Sphinx directly from Homebrew:: brew install sphinx-doc From 9284daaf89c4bbeb9bede091820c2bd28fc947c4 Mon Sep 17 00:00:00 2001 From: lethosor Date: Fri, 10 Jul 2020 19:59:34 -0400 Subject: [PATCH 87/97] Unset content min-width to improve mobile layout --- docs/styles/dfhack.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/styles/dfhack.css b/docs/styles/dfhack.css index e7a608460..43063cee2 100644 --- a/docs/styles/dfhack.css +++ b/docs/styles/dfhack.css @@ -52,3 +52,7 @@ div.sphinxsidebar { div.sphinxsidebar h3 > a { border-bottom: 1px dotted #999; } + +div.body { + min-width: unset; +} From c30d27cfbd6c560bd922ad01f7a8c94ce2d525f4 Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 11 Jul 2020 01:41:38 -0400 Subject: [PATCH 88/97] Silence strangemood warnings --- plugins/strangemood.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/strangemood.cpp b/plugins/strangemood.cpp index f37400cbb..67ccbf719 100644 --- a/plugins/strangemood.cpp +++ b/plugins/strangemood.cpp @@ -314,6 +314,9 @@ void generateName(df::language_name &output, int language, df::language_name_typ if (rng.df_trandom(100)) selectWord(table2, output.words[4], output.parts_of_speech[4], 4); break; + default: + // not handled yet + break; } if (output.words[2] != -1 && output.words[3] != -1 && world->raws.language.words[output.words[3]]->adj_dist < world->raws.language.words[output.words[2]]->adj_dist) From 07a2b4a9d7cfa10ab533815c9d50fc5a10305066 Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 11 Jul 2020 02:05:38 -0400 Subject: [PATCH 89/97] Use language_name_type enum item names --- library/xml | 2 +- plugins/strangemood.cpp | 48 ++++++++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/library/xml b/library/xml index 847ec0c2d..b9028b0bb 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 847ec0c2d4dc74a887e1f980eeed6f97345c195e +Subproject commit b9028b0bb9ad40d3ad4dc3f10934bb61aa16629b diff --git a/plugins/strangemood.cpp b/plugins/strangemood.cpp index 67ccbf719..b1b48f1a5 100644 --- a/plugins/strangemood.cpp +++ b/plugins/strangemood.cpp @@ -171,7 +171,9 @@ void generateName(df::language_name &output, int language, df::language_name_typ int r, r2, r3; switch (mode) { - case 0: case 9: case 10: + case language_name_type::Figure: + case language_name_type::FigureNoFirst: + case language_name_type::FigureFirstOnly: if (mode != 9) { int32_t word; df::part_of_speech part; @@ -182,7 +184,8 @@ void generateName(df::language_name &output, int language, df::language_name_typ } if (mode != 10) { - case 4: case 37: // this is not a typo + case language_name_type::Site: + case language_name_type::Unk37: // this is not a typo if (rng.df_trandom(2)) { selectWord(table2, output.words[0], output.parts_of_speech[0], 0); @@ -196,7 +199,9 @@ void generateName(df::language_name &output, int language, df::language_name_typ } break; - case 1: case 13: case 20: + case language_name_type::Artifact: + case language_name_type::Unk13: + case language_name_type::River: r = rng.df_trandom(3); if (r == 0 || r == 1) { @@ -213,7 +218,9 @@ void generateName(df::language_name &output, int language, df::language_name_typ } if (r == 1 || r == 2) { - case 3: case 8: case 11: // this is not a typo either + case language_name_type::Squad: + case language_name_type::LegendaryFigure: + case language_name_type::ArtImage: // this is not a typo either r2 = rng.df_trandom(2); if (r2) selectWord(table1, output.words[5], output.parts_of_speech[5], 2); @@ -256,9 +263,34 @@ void generateName(df::language_name &output, int language, df::language_name_typ output.parts_of_speech[5] = part_of_speech::NounPlural; break; - case 2: case 5: case 6: case 12: case 14: case 15: case 16: case 17: case 18: case 19: - case 21: case 22: case 23: case 24: case 25: case 26: case 27: case 28: case 29: case 30: - case 31: case 32: case 33: case 34: case 35: case 36: case 38: case 39: + case language_name_type::Civilization: + case language_name_type::World: + case language_name_type::Region: + case language_name_type::AdventuringGroup: + case language_name_type::SiteGovernment: + case language_name_type::Unk15: + case language_name_type::Vessel: + case language_name_type::MilitaryUnit: + case language_name_type::Religion: + case language_name_type::MountainPeak: + case language_name_type::Temple: + case language_name_type::Keep: + case language_name_type::MeadHall: + case language_name_type::Unk24: + case language_name_type::Unk25: + case language_name_type::Unk26: + case language_name_type::Market: + case language_name_type::Tavern: + case language_name_type::War: + case language_name_type::Battle: + case language_name_type::Siege: + case language_name_type::Road: + case language_name_type::Wall: + case language_name_type::Bridge: + case language_name_type::Tunnel: + case language_name_type::PretentiousEntityPosition: + case language_name_type::Tomb: + case language_name_type::MigratingGroup: selectWord(table1, output.words[5], output.parts_of_speech[5], 2); r3 = rng.df_trandom(3); if (rng.df_trandom(50)) @@ -280,7 +312,7 @@ void generateName(df::language_name &output, int language, df::language_name_typ selectWord(table2, output.words[4], output.parts_of_speech[4], 4); break; - case 7: + case language_name_type::Dungeon: r = rng.df_trandom(3); if (r == 0 || r == 1) { From 1043a4e49b2f848d39f90a17b0c5d0a2b5b638db Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 11 Jul 2020 23:50:51 -0400 Subject: [PATCH 90/97] Update intro, fix bay12 link, shorten index, add quick links section --- docs/Compile.rst | 2 ++ docs/Introduction.rst | 13 +++++++------ index.rst | 38 ++++++++++++++------------------------ 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/docs/Compile.rst b/docs/Compile.rst index 26f720076..68187e7fa 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -1,3 +1,5 @@ +.. _compile: + ################ Compiling DFHack ################ diff --git a/docs/Introduction.rst b/docs/Introduction.rst index dc05c8725..4490086f3 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -7,14 +7,15 @@ Introduction and Overview DFHack is a Dwarf Fortress memory access library, distributed with a wide variety of useful scripts and plugins. -The project is currently hosted at https://www.github.com/DFHack/dfhack, -and can be downloaded from `the releases page -`_. +The project is currently hosted `on GitHub `_, +and can be downloaded from `the releases page `_ +- see `installing` for installation instructions. This is also where the +`DFHack bug tracker `_ is hosted. -All new releases are announced in :forums:`the bay12 forums thread <139553>`, +All new releases are announced in `the Bay12 forums thread `_, which is also a good place for discussion and questions. -For users, it provides a significant suite of bugfixes and interface +For users, DFHack provides a significant suite of bugfixes and interface enhancements by default, and more can be enabled. There are also many tools (such as `workflow` or `autodump`) which can make life easier. You can even add third-party scripts and plugins to do almost anything! @@ -27,7 +28,7 @@ or for coexistence in a single DF install, even with incompatible components. For developers, DFHack unites the various ways tools access DF memory and allows easier development of new tools. As an open-source project under -`various copyleft licences `, contributions are welcome. +`various open-source licences `, contributions are welcome. .. contents:: Contents diff --git a/index.rst b/index.rst index df28d0778..d5c7fbc10 100644 --- a/index.rst +++ b/index.rst @@ -2,33 +2,23 @@ Welcome to DFHack's documentation! ################################## -Introduction -============ -DFHack is a Dwarf Fortress memory access library, distributed with -a wide variety of useful scripts and plugins. +DFHack is a memory editing library for `Dwarf Fortress `_ +that provides a unified, cross-platform environment where tools can be developed +to extend the game. The default distribution contains a variety of tools, including +bugfixes, interface improvements, automation tools, modding tools, and more. +There are also a variety of third-party tools available. -The project is currently hosted at https://www.github.com/DFHack/dfhack, -and can be downloaded from `the releases page -`_. +.. _quick-links: -All new releases are announced in :forums:`the bay12 forums thread <139553>`, -which is also a good place for discussion and questions. - -For users, it provides a significant suite of bugfixes and interface -enhancements by default, and more can be enabled. There are also many tools -(such as `workflow` or `autodump`) which can make life easier. -You can even add third-party scripts and plugins to do almost anything! - -For modders, DFHack makes many things possible. Custom reactions, new -interactions, magic creature abilities, and more can be set through `scripts-modtools` -and custom raws. Non-standard DFHack scripts and inits can be stored in the -raw directory, making raws or saves fully self-contained for distribution - -or for coexistence in a single DF install, even with incompatible components. - -For developers, DFHack unites the various ways tools access DF memory and -allows easier development of new tools. As an open-source project under -`various copyleft licences `, contributions are welcome. +Quick Links +=========== +* `Downloads `_ +* `Installation guide ` +* `Source code `_ + (**important:** read `compile` before attempting to build from source) +* `Bay 12 forums thread `_ +* `Bug tracker `_ User Manual =========== From 801961bf7853b2747a46ce553529fe73c03a422f Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 11 Jul 2020 23:55:02 -0400 Subject: [PATCH 91/97] Move changelog, license, authors into main TOC tree --- docs/index-about.rst | 13 +++++++++++++ index.rst | 12 +----------- 2 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 docs/index-about.rst diff --git a/docs/index-about.rst b/docs/index-about.rst new file mode 100644 index 000000000..3dba1e5f8 --- /dev/null +++ b/docs/index-about.rst @@ -0,0 +1,13 @@ +============ +About DFHack +============ + +These pages contain information about the general DFHack project. + +.. toctree:: + :maxdepth: 1 + + /docs/NEWS + /docs/Authors + /LICENSE + /docs/Removed diff --git a/index.rst b/index.rst index d5c7fbc10..91c883b51 100644 --- a/index.rst +++ b/index.rst @@ -30,15 +30,5 @@ User Manual /docs/Core /docs/Plugins /docs/Scripts + /docs/index-about /docs/index-dev - -Other Contents -============== - -.. toctree:: - :maxdepth: 1 - - /docs/Authors - /LICENSE - /docs/NEWS - /docs/Removed From 279d3e80f2f3691facae7cea3cc022b59251cd59 Mon Sep 17 00:00:00 2001 From: lethosor Date: Sat, 11 Jul 2020 23:56:15 -0400 Subject: [PATCH 92/97] Add kazimuth to authors --- docs/Authors.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Authors.rst b/docs/Authors.rst index e816389cd..3111b17f0 100644 --- a/docs/Authors.rst +++ b/docs/Authors.rst @@ -57,6 +57,7 @@ Hayati Ayguen hayguen Herwig Hochleitner bendlas Ian S kremlin- IndigoFenix +James Gilles kazimuth James Logsdon jlogsdon Japa JapaMala Jared Adams From a8d34bb64ec6ef6e09c611de7a8bbb14b322f0c3 Mon Sep 17 00:00:00 2001 From: lethosor Date: Sun, 12 Jul 2020 00:30:28 -0400 Subject: [PATCH 93/97] Add explicit home link to sidebar --- conf.py | 2 +- docs/styles/dfhack.css | 2 +- docs/templates/about.html | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 docs/templates/about.html diff --git a/conf.py b/conf.py index e70d836f1..a46039126 100644 --- a/conf.py +++ b/conf.py @@ -212,7 +212,7 @@ extlinks = { } # Add any paths that contain templates here, relative to this directory. -templates_path = [] +templates_path = ["docs/templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: diff --git a/docs/styles/dfhack.css b/docs/styles/dfhack.css index 43063cee2..daedd1120 100644 --- a/docs/styles/dfhack.css +++ b/docs/styles/dfhack.css @@ -49,7 +49,7 @@ div.sphinxsidebar { overflow-y: auto; } -div.sphinxsidebar h3 > a { +div.sphinxsidebar h3.logo-name a { border-bottom: 1px dotted #999; } diff --git a/docs/templates/about.html b/docs/templates/about.html new file mode 100644 index 000000000..48f98859f --- /dev/null +++ b/docs/templates/about.html @@ -0,0 +1,6 @@ + + +

From 1d2b0bb3a522cfeb92c4695d7d4ef5a6a0e9c13b Mon Sep 17 00:00:00 2001 From: Tommy R Date: Mon, 13 Jul 2020 10:04:09 -0700 Subject: [PATCH 94/97] Fix the unquarantine behavior on OSX. - Using print0 but without the parenthesis, find only outputs one filename. - Also added the .framework files to the unquarantine list. --- package/darwin/dfhack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/darwin/dfhack b/package/darwin/dfhack index 1c2c7a175..5241f1a67 100755 --- a/package/darwin/dfhack +++ b/package/darwin/dfhack @@ -13,7 +13,7 @@ fi # attempt to remove quarantine flag: https://github.com/DFHack/dfhack/issues/1465 if ! test -f hack/quarantine-removed; then - find hack/ libs/ dwarfort.exe -name '*.dylib' -or -name '*.exe' -print0 | xargs -0 xattr -d com.apple.quarantine 2>&1 | grep -iv 'no such xattr' + find hack/ libs/ dwarfort.exe \( -name '*.dylib' -or -name '*.framework' -or -name '*.exe' \) -print0 | xargs -0 xattr -d com.apple.quarantine 2>&1 | grep -iv 'no such xattr' echo "quarantine flag removed on $(date); remove this file to re-run" > hack/quarantine-removed fi From f33f9136c88031e59fbaee3105c8bac1250fe14c Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 13 Jul 2020 19:57:14 -0400 Subject: [PATCH 95/97] Update changelog + authors (#1606) --- docs/Authors.rst | 1 + docs/changelog.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/Authors.rst b/docs/Authors.rst index 3111b17f0..6701b67a0 100644 --- a/docs/Authors.rst +++ b/docs/Authors.rst @@ -161,6 +161,7 @@ Timothy Collett danaris Timur Kelman TymurGubayev Tom Jobbins TheBloke Tom Prince +Tommy R tommy TotallyGatsby TotallyGatsby Travis Hoppe thoppe orthographic-pedant txtsd txtsd diff --git a/docs/changelog.txt b/docs/changelog.txt index 7201bfd23..c6fbe840a 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -35,6 +35,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: ## Fixes - Fixed a segfault when attempting to start a headless session with a graphical PRINT_MODE setting +- Fixed an issue with the macOS launcher failing to un-quarantine some files - `labormanager`: fixed handling of new jobs in 0.47 - Fixed ``Units::isEggLayer``, ``Units::isGrazer``, ``Units::isMilkable``, ``Units::isTrainableHunting``, ``Units::isTrainableWar``, and ``Units::isTamable`` ignoring the unit's caste From ced8f4a55164da36f7b1b7b458eb1f8c789f2e97 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 13 Jul 2020 20:05:08 -0400 Subject: [PATCH 96/97] Update changelog (#1565) --- docs/changelog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index c6fbe840a..dfda78c0b 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -38,6 +38,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: - Fixed an issue with the macOS launcher failing to un-quarantine some files - `labormanager`: fixed handling of new jobs in 0.47 - Fixed ``Units::isEggLayer``, ``Units::isGrazer``, ``Units::isMilkable``, ``Units::isTrainableHunting``, ``Units::isTrainableWar``, and ``Units::isTamable`` ignoring the unit's caste +- `RemoteFortressReader`: fixed a couple crashes that could result from decoding invalid enum items (``site_realization_building_type`` and ``improvement_type``) ## Misc Improvements - `confirm`: added a confirmation dialog for convicting dwarves of crimes From 89283026ae917883f84da7a16179f207390bc540 Mon Sep 17 00:00:00 2001 From: lethosor Date: Mon, 13 Jul 2020 20:14:11 -0400 Subject: [PATCH 97/97] Improve changelog error handling a bit when called from sphinx --- docs/sphinx_extensions/dfhack/changelog.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/sphinx_extensions/dfhack/changelog.py b/docs/sphinx_extensions/dfhack/changelog.py index 99f0ea299..6f444d36f 100644 --- a/docs/sphinx_extensions/dfhack/changelog.py +++ b/docs/sphinx_extensions/dfhack/changelog.py @@ -4,6 +4,8 @@ import itertools import os import sys +from sphinx.errors import ExtensionError, SphinxError, SphinxWarning + from dfhack.util import DFHACK_ROOT, DOCS_ROOT CHANGELOG_PATHS = ( @@ -214,7 +216,7 @@ def generate_changelog(all=False): # scan for unrecognized sections for entry in entries: if entry.section not in CHANGELOG_SECTIONS: - raise RuntimeWarning('Unknown section: ' + entry.section) + raise SphinxWarning('Unknown section: ' + entry.section) # ordered versions versions = ['future'] @@ -287,7 +289,12 @@ def cli_entrypoint(): def sphinx_entrypoint(app, config): - generate_changelog() + try: + generate_changelog() + except SphinxError: + raise + except Exception as e: + raise ExtensionError(str(e), e) def setup(app):