From a9bb11c145928d5ce9d0301842b889d18509afd4 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 20 Nov 2020 17:57:54 -0500
Subject: [PATCH 01/31] Optimize Lua's internal.runCommand() when printing
 directly to the console

This also makes commands run with `run_command()` detect the console properly (notably used by `df2console()`)
---
 library/LuaApi.cpp     | 46 +++++++++++++++++++++++++++++-------------
 library/lua/dfhack.lua | 16 ++++-----------
 2 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp
index 431ab3ae5..ed214397d 100644
--- a/library/LuaApi.cpp
+++ b/library/LuaApi.cpp
@@ -2822,13 +2822,25 @@ static int internal_diffscan(lua_State *L)
 
 static int internal_runCommand(lua_State *L)
 {
-    buffered_color_ostream out;
+    color_ostream *out = NULL;
+    std::unique_ptr<buffered_color_ostream> out_buffer;
     command_result res;
     if (lua_gettop(L) == 0)
     {
         lua_pushstring(L, "");
     }
     int type_1 = lua_type(L, 1);
+    bool use_console = lua_toboolean(L, 2);
+    if (use_console)
+    {
+        out = &Core::getInstance().getConsole();
+    }
+    else
+    {
+        out_buffer.reset(new buffered_color_ostream());
+        out = out_buffer.get();
+    }
+
     if (type_1 == LUA_TTABLE)
     {
         std::string command = "";
@@ -2843,13 +2855,13 @@ static int internal_runCommand(lua_State *L)
             lua_pop(L, 1);  // remove value, leave key
         }
         CoreSuspender suspend;
-        res = Core::getInstance().runCommand(out, command, args);
+        res = Core::getInstance().runCommand(*out, command, args);
     }
     else if (type_1 == LUA_TSTRING)
     {
         std::string command = lua_tostring(L, 1);
         CoreSuspender suspend;
-        res = Core::getInstance().runCommand(out, command);
+        res = Core::getInstance().runCommand(*out, command);
     }
     else
     {
@@ -2857,22 +2869,28 @@ static int internal_runCommand(lua_State *L)
         lua_pushfstring(L, "Expected table, got %s", lua_typename(L, type_1));
         return 2;
     }
-    auto fragments = out.fragments();
+
     lua_newtable(L);
     lua_pushinteger(L, (int)res);
     lua_setfield(L, -2, "status");
-    int i = 1;
-    for (auto iter = fragments.begin(); iter != fragments.end(); iter++, i++)
+
+    if (out_buffer)
     {
-        int color = iter->first;
-        std::string output = iter->second;
-        lua_createtable(L, 2, 0);
-        lua_pushinteger(L, color);
-        lua_rawseti(L, -2, 1);
-        lua_pushstring(L, output.c_str());
-        lua_rawseti(L, -2, 2);
-        lua_rawseti(L, -2, i);
+        auto fragments = out_buffer->fragments();
+        int i = 1;
+        for (auto iter = fragments.begin(); iter != fragments.end(); iter++, i++)
+        {
+            int color = iter->first;
+            std::string output = iter->second;
+            lua_createtable(L, 2, 0);
+            lua_pushinteger(L, color);
+            lua_rawseti(L, -2, 1);
+            lua_pushstring(L, output.c_str());
+            lua_rawseti(L, -2, 2);
+            lua_rawseti(L, -2, i);
+        }
     }
+
     lua_pushvalue(L, -1);
     return 1;
 }
diff --git a/library/lua/dfhack.lua b/library/lua/dfhack.lua
index 73fd651e2..26a23db55 100644
--- a/library/lua/dfhack.lua
+++ b/library/lua/dfhack.lua
@@ -735,8 +735,7 @@ function dfhack.script_help(script_name, extension)
     return help
 end
 
-local function _run_command(...)
-    args = {...}
+local function _run_command(args, use_console)
     if type(args[1]) == 'table' then
         command = args[1]
     elseif #args > 1 and type(args[2]) == 'table' then
@@ -750,11 +749,11 @@ local function _run_command(...)
     else
         error('Invalid arguments')
     end
-    return internal.runCommand(command)
+    return internal.runCommand(command, use_console)
 end
 
 function dfhack.run_command_silent(...)
-    local result = _run_command(...)
+    local result = _run_command({...})
     local output = ""
     for i, f in pairs(result) do
         if type(f) == 'table' then
@@ -765,14 +764,7 @@ function dfhack.run_command_silent(...)
 end
 
 function dfhack.run_command(...)
-    local result = _run_command(...)
-    for i, f in pairs(result) do
-        if type(f) == 'table' then
-            dfhack.color(f[1])
-            dfhack.print(f[2])
-        end
-    end
-    dfhack.color(COLOR_RESET)
+    local result = _run_command({...}, true)
     return result.status
 end
 

From b4c040291f117d6f6fabd3cd84260b42bbc88b7d Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 20 Nov 2020 22:23:29 -0500
Subject: [PATCH 02/31] Document dfhack.run_command() and related
 functions/constants

---
 docs/Lua API.rst | 51 +++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/docs/Lua API.rst b/docs/Lua API.rst
index ffce7fd4e..02b8a324f 100644
--- a/docs/Lua API.rst	
+++ b/docs/Lua API.rst	
@@ -833,6 +833,9 @@ can be omitted.
 * ``dfhack.getCompiledDFVersion()``
 * ``dfhack.getGitDescription()``
 * ``dfhack.getGitCommit()``
+* ``dfhack.getGitXmlCommit()``
+* ``dfhack.getGitXmlExpectedCommit()``
+* ``dfhack.gitXmlMatch()``
 * ``dfhack.isRelease()``
 
   Return information about the DFHack build in use.
@@ -899,6 +902,28 @@ can be omitted.
   Note that the returned string may be longer than the input string. For
   example, ``ä`` is replaced with ``a``, and ``æ`` is replaced with ``ae``.
 
+* ``dfhack.run_command(command[, ...])``
+
+  Run an arbitrary DFHack command, with the core suspended, and send output to
+  the DFHack console. The command can be passed as a table, multiple string
+  arguments, or a single string argument (not recommended - in this case, the
+  usual DFHack console tokenization is used).
+
+  A ``command_result`` constant starting with ``CR_`` is returned, where ``CR_OK``
+  indicates success.
+
+  The following examples are equivalent::
+
+    dfhack.run_command({'ls', '-a'})
+    dfhack.run_command('ls', '-a')
+    dfhack.run_command('ls -a')  -- not recommended
+
+* ``dfhack.run_command_silent(command[, ...])``
+
+  Similar to ``run_command()``, but instead of printing to the console,
+  returns an ``output, command_result`` pair. ``output`` is a single string -
+  see ``dfhack.internal.runCommand()`` to obtain colors as well.
+
 Gui module
 ----------
 
@@ -2229,11 +2254,6 @@ Internal API
 These functions are intended for the use by dfhack developers,
 and are only documented here for completeness:
 
-* ``dfhack.internal.scripts``
-
-  The table used by ``dfhack.run_script()`` to give every script its own
-  global environment, persistent between calls to the script.
-
 * ``dfhack.internal.getPE()``
 
   Returns the PE timestamp of the DF executable (only on Windows)
@@ -2348,6 +2368,21 @@ and are only documented here for completeness:
     This requires an extension to be specified (``.lua`` or ``.rb``) - use
     ``dfhack.findScript()`` to include the ``.lua`` extension automatically.
 
+* ``dfhack.internal.runCommand(command[, use_console])``
+
+  Runs a DFHack command with the core suspended. Used internally by the
+  ``dfhack.run_command()`` family of functions.
+
+  - ``command``: either a table of strings or a single string which is parsed by
+    the default console tokenization strategy (not recommended)
+  - ``use_console``: if true, output is sent directly to the DFHack console
+
+  Returns a table with a ``status`` key set to a ``command_result`` constant
+  (``status = CR_OK`` indicates success). Additionally, if ``use_console`` is
+  not true, enumerated table entries of the form ``{color, text}`` are included,
+  e.g. ``result[1][0]`` is the color of the first piece of text printed (a
+  ``COLOR_`` constant). These entries can be iterated over with ``ipairs()``.
+
 * ``dfhack.internal.md5(string)``
 
   Returns the MD5 hash of the given string.
@@ -2507,6 +2542,12 @@ environment by the mandatory init file dfhack.lua:
   SC_WORLD_LOADED, SC_WORLD_UNLOADED, SC_MAP_LOADED,
   SC_MAP_UNLOADED, SC_VIEWSCREEN_CHANGED, SC_CORE_INITIALIZED
 
+* Command result constants (equivalent to ``command_result`` in C++), used by
+  ``dfhack.run_command()`` and related functions:
+
+  CR_OK, CR_LINK_FAILURE, CR_NEEDS_CONSOLE, CR_NOT_IMPLEMENTED, CR_FAILURE,
+  CR_WRONG_USAGE, CR_NOT_FOUND
+
 * Functions already described above
 
   safecall, qerror, mkmodule, reload

From 867d7ebfc953254e227aa4083ab0cff77e49786a Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Sat, 21 Nov 2020 10:10:32 -0500
Subject: [PATCH 03/31] Update scripts

---
 scripts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts b/scripts
index 4ac39084f..381c9f871 160000
--- a/scripts
+++ b/scripts
@@ -1 +1 @@
-Subproject commit 4ac39084f7e6fde5ec08c5c5691422859a212460
+Subproject commit 381c9f87160e4d836faab8678ebdfbb219d23ebe

From 6ceaf77b38863b1d5cb42ba6cc0824f1e98e4708 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Sat, 21 Nov 2020 10:10:47 -0500
Subject: [PATCH 04/31] Update authors (dfhack/scripts#217)

---
 docs/Authors.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/Authors.rst b/docs/Authors.rst
index e3478e486..83ce58ef0 100644
--- a/docs/Authors.rst
+++ b/docs/Authors.rst
@@ -159,6 +159,7 @@ stolencatkarma
 Stoyan Gaydarov         sgayda2
 suokko                  suokko                  shrieker
 sv-esk                  sv-esk
+Tachytaenius            wolfboyft
 Tacomagic
 thefriendlyhacker       thefriendlyhacker
 TheHologram             TheHologram

From fe0cd9a939385fa987e6ffc7960fe3a21a696491 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Thu, 3 Dec 2020 22:04:00 -0500
Subject: [PATCH 05/31] Make lineedit() fail sooner when another lineedit()
 call is active

Before, calling `dfhack.run_command('tiletypes')` from a `dfhack.timeout` callback would leave the console in a broken state, since raw mode was enabled but never disabled.

Only tested on Linux with a supported terminal.
---
 library/Console-posix.cpp   | 6 ++++--
 library/Console-windows.cpp | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/library/Console-posix.cpp b/library/Console-posix.cpp
index 595f50e97..3c8645ebf 100644
--- a/library/Console-posix.cpp
+++ b/library/Console-posix.cpp
@@ -405,6 +405,8 @@ namespace DFHack
         /// A simple line edit (raw mode)
         int lineedit(const std::string& prompt, std::string& output, recursive_mutex * lock, CommandHistory & ch)
         {
+            if(state == con_lineedit)
+                return Console::FAILURE;
             output.clear();
             reset_color();
             this->prompt = prompt;
@@ -414,7 +416,9 @@ namespace DFHack
                 fflush(dfout_C);
                 // FIXME: what do we do here???
                 //SDL_recursive_mutexV(lock);
+                state = con_lineedit;
                 std::getline(std::cin, output);
+                state = con_unclaimed;
                 //SDL_recursive_mutexP(lock);
                 return output.size();
             }
@@ -422,8 +426,6 @@ namespace DFHack
             {
                 int count;
                 if (enable_raw() == -1) return 0;
-                if(state == con_lineedit)
-                    return Console::FAILURE;
                 state = con_lineedit;
                 count = prompt_loop(lock,ch);
                 state = con_unclaimed;
diff --git a/library/Console-windows.cpp b/library/Console-windows.cpp
index d339e53a0..515d89911 100644
--- a/library/Console-windows.cpp
+++ b/library/Console-windows.cpp
@@ -376,6 +376,8 @@ namespace DFHack
         }
         int lineedit(const std::string & prompt, std::string & output, recursive_mutex * lock, CommandHistory & ch)
         {
+            if(state == con_lineedit)
+                return Console::FAILURE;
             output.clear();
             reset_color();
             int count;

From b585bca90b17d638edf10c68f016095afa0dc6b6 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Wed, 30 Dec 2020 15:04:43 -0500
Subject: [PATCH 06/31] look_menu_search: fix crash due to certain cursor keys
 not restoring list

Fixes #1737
---
 plugins/search.cpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/plugins/search.cpp b/plugins/search.cpp
index 27958f6dc..7b791dc7f 100644
--- a/plugins/search.cpp
+++ b/plugins/search.cpp
@@ -115,13 +115,16 @@ static string get_unit_description(df::unit *unit)
     return desc;
 }
 
-static bool cursor_key_pressed (std::set<df::interface_key> *input)
+static bool cursor_key_pressed (std::set<df::interface_key> *input, bool in_entry_mode)
 {
-    // give text input (e.g. "2") priority over cursor keys
-    for (auto it = input->begin(); it != input->end(); ++it)
+    if (in_entry_mode)
     {
-        if (Screen::keyToChar(*it) != -1)
-            return false;
+        // give text input (e.g. "2") priority over cursor keys
+        for (auto it = input->begin(); it != input->end(); ++it)
+        {
+            if (Screen::keyToChar(*it) != -1)
+                return false;
+        }
     }
     return
     input->count(df::interface_key::CURSOR_UP) ||
@@ -249,7 +252,7 @@ public:
                 // ENTER or ESC: leave typing mode
                 end_entry_mode();
             }
-            else if (cursor_key_pressed(input))
+            else if (cursor_key_pressed(input, entry_mode))
             {
                 // Arrow key pressed. Leave entry mode and allow screen to process key
                 end_entry_mode();
@@ -1953,7 +1956,7 @@ public:
             end_entry_mode();
             return false;
         }
-        if (cursor_key_pressed(input))
+        if (cursor_key_pressed(input, in_entry_mode()))
         {
             end_entry_mode();
             clear_search();

From 936a13181aa777372781e44ceb1677d2ed303887 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Wed, 30 Dec 2020 15:08:20 -0500
Subject: [PATCH 07/31] Update changelog

---
 docs/changelog.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index fdc7c13e7..94c187b08 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -35,6 +35,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 
 ## Fixes
 - `embark-assistant`: fixed order of factors when calculating min temperature
+- `search`: fixed crash when searching the ``k`` sidebar and navigating to another tile with certain keys, like ``<`` or ``>``
 
 # 0.47.04-r4
 

From 69ce5d9e3faed4b147ae46a3e6910f2d208dff33 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 8 Jan 2021 21:06:42 -0500
Subject: [PATCH 08/31] Send runCommand output to correct output stream

e.g. when run with dfhack-run
---
 library/LuaApi.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp
index ed214397d..d7f3b269f 100644
--- a/library/LuaApi.cpp
+++ b/library/LuaApi.cpp
@@ -2833,7 +2833,11 @@ static int internal_runCommand(lua_State *L)
     bool use_console = lua_toboolean(L, 2);
     if (use_console)
     {
-        out = &Core::getInstance().getConsole();
+        out = Lua::GetOutput(L);
+        if (!out)
+        {
+            out = &Core::getInstance().getConsole();
+        }
     }
     else
     {

From 20276be50f70e3ba02bbcd16dbd6816839d6b4a5 Mon Sep 17 00:00:00 2001
From: Myk Taylor <myk.taylor@gmail.com>
Date: Sun, 1 Nov 2020 16:05:56 -0800
Subject: [PATCH 09/31] check in unaltered version of alt_getopt

---
 library/lua/3rdparty/alt_getopt.lua | 171 ++++++++++++++++++++++++++++
 1 file changed, 171 insertions(+)
 create mode 100644 library/lua/3rdparty/alt_getopt.lua

diff --git a/library/lua/3rdparty/alt_getopt.lua b/library/lua/3rdparty/alt_getopt.lua
new file mode 100644
index 000000000..87ed20c8f
--- /dev/null
+++ b/library/lua/3rdparty/alt_getopt.lua
@@ -0,0 +1,171 @@
+-- Copyright (c) 2009 Aleksey Cheusov <vle@gmx.net>
+--
+-- Permission is hereby granted, free of charge, to any person obtaining
+-- a copy of this software and associated documentation files (the
+-- "Software"), to deal in the Software without restriction, including
+-- without limitation the rights to use, copy, modify, merge, publish,
+-- distribute, sublicense, and/or sell copies of the Software, and to
+-- permit persons to whom the Software is furnished to do so, subject to
+-- the following conditions:
+--
+-- The above copyright notice and this permission notice shall be
+-- included in all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+-- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+-- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+-- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+-- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+-- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+-- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-- based on https://github.com/LuaDist/alt-getopt/blob/master/alt_getopt.lua
+-- MIT licence
+-- modified to support non-options and to not call os.exit()
+-- intended to be used via utils.processArgs2()
+
+local _ENV = mkmodule('alt_getopt')
+
+local function convert_short2long (opts)
+   local i = 1
+   local len = #opts
+   local ret = {}
+
+   for short_opt, accept_arg in opts:gmatch("(%w)(:?)") do
+      ret[short_opt]=#accept_arg
+   end
+
+   return ret
+end
+
+local function exit_with_error (msg, exit_status)
+   io.stderr:write (msg)
+   os.exit (exit_status)
+end
+
+local function err_unknown_opt (opt)
+   exit_with_error ("Unknown option `-" ..
+		    (#opt > 1 and "-" or "") .. opt .. "'\n", 1)
+end
+
+local function canonize (options, opt)
+   if not options [opt] then
+      err_unknown_opt (opt)
+   end
+
+   while type (options [opt]) == "string" do
+      opt = options [opt]
+
+      if not options [opt] then
+	 err_unknown_opt (opt)
+      end
+   end
+
+   return opt
+end
+
+function get_ordered_opts (arg, sh_opts, long_opts)
+   local i      = 1
+   local count  = 1
+   local opts   = {}
+   local optarg = {}
+
+   local options = convert_short2long (sh_opts)
+   for k,v in pairs (long_opts) do
+      options [k] = v
+   end
+
+   while i <= #arg do
+      local a = arg [i]
+
+      if a == "--" then
+	 i = i + 1
+	 break
+
+      elseif a == "-" then
+	 break
+
+      elseif a:sub (1, 2) == "--" then
+	 local pos = a:find ("=", 1, true)
+
+	 if pos then
+	    local opt = a:sub (3, pos-1)
+
+	    opt = canonize (options, opt)
+
+	    if options [opt] == 0 then
+	       exit_with_error ("Bad usage of option `" .. a .. "'\n", 1)
+	    end
+
+	    optarg [count] = a:sub (pos+1)
+	    opts [count] = opt
+	 else
+	    local opt = a:sub (3)
+
+	    opt = canonize (options, opt)
+
+	    if options [opt] == 0 then
+	       opts [count] = opt
+	    else
+	       if i == #arg then
+		  exit_with_error ("Missed value for option `" .. a .. "'\n", 1)
+	       end
+
+	       optarg [count] = arg [i+1]
+	       opts [count] = opt
+	       i = i + 1
+	    end
+	 end
+	 count = count + 1
+
+      elseif a:sub (1, 1) == "-" then
+	 local j
+	 for j=2,a:len () do
+	    local opt = canonize (options, a:sub (j, j))
+
+	    if options [opt] == 0 then
+	       opts [count] = opt
+	       count = count + 1
+	    elseif a:len () == j then
+	       if i == #arg then
+		  exit_with_error ("Missed value for option `-" .. opt .. "'\n", 1)
+	       end
+
+	       optarg [count] = arg [i+1]
+	       opts [count] = opt
+	       i = i + 1
+	       count = count + 1
+	       break
+	    else
+	       optarg [count] = a:sub (j+1)
+	       opts [count] = opt
+	       count = count + 1
+	       break
+	    end
+	 end
+      else
+	 break
+      end
+
+      i = i + 1
+   end
+
+   return opts,i,optarg
+end
+
+function get_opts (arg, sh_opts, long_opts)
+   local ret = {}
+
+   local opts,optind,optarg = get_ordered_opts (arg, sh_opts, long_opts)
+   for i,v in ipairs (opts) do
+      if optarg [i] then
+	 ret [v] = optarg [i]
+      else
+	 ret [v] = 1
+      end
+   end
+
+   return ret,optind
+end
+
+return _ENV

From 3b45878d41480a9fcabfb3b58782d86e8c52bc43 Mon Sep 17 00:00:00 2001
From: Myk Taylor <myk.taylor@gmail.com>
Date: Tue, 3 Nov 2020 16:19:40 -0800
Subject: [PATCH 10/31] allow non-options in commandlines and return them

also call qerror() on error, not os.exit
---
 library/lua/3rdparty/alt_getopt.lua | 254 ++++++++++++++--------------
 1 file changed, 126 insertions(+), 128 deletions(-)

diff --git a/library/lua/3rdparty/alt_getopt.lua b/library/lua/3rdparty/alt_getopt.lua
index 87ed20c8f..a356552aa 100644
--- a/library/lua/3rdparty/alt_getopt.lua
+++ b/library/lua/3rdparty/alt_getopt.lua
@@ -21,151 +21,149 @@
 
 -- based on https://github.com/LuaDist/alt-getopt/blob/master/alt_getopt.lua
 -- MIT licence
--- modified to support non-options and to not call os.exit()
--- intended to be used via utils.processArgs2()
+-- modified to support aggregation of non-options and to call qerror instead of
+-- os.exit() on error. can be used directly or via the utils.processArgs2()
+-- wrapper.
+
+-- sh_opts should be in standard getopt format: a string of letters that
+-- represent options, each followed by a colon if that option takes an argument.
+-- e.g.: 'ak:hv' has three flags (options with no arguments): 'a', 'h', and 'v'
+-- and one option that takes an argument: 'k'.
+--
+-- Options passed to the module to parse can be in any of the following formats:
+--   -kVALUE, -k VALUE, --key=VALUE, --key VALUE
+--   -abcd is equivalent to -a -b -c -d if none of them accept arguments.
+--   -abckVALUE and -abck VALUE are also acceptable (where k is the only option
+--      in the string that takes a value).
 
-local _ENV = mkmodule('alt_getopt')
+local _ENV = mkmodule('3rdparty.alt_getopt')
 
-local function convert_short2long (opts)
-   local i = 1
-   local len = #opts
-   local ret = {}
+local function get_opt_map(opts)
+    local i = 1
+    local len = #opts
+    local options = {}
 
-   for short_opt, accept_arg in opts:gmatch("(%w)(:?)") do
-      ret[short_opt]=#accept_arg
-   end
+    for short_opt, accept_arg in opts:gmatch('(%w)(:?)') do
+        options[short_opt] = #accept_arg
+    end
 
-   return ret
+    return options
 end
 
-local function exit_with_error (msg, exit_status)
-   io.stderr:write (msg)
-   os.exit (exit_status)
+local function err_unknown_opt(opt)
+    qerror(string.format('Unknown option "-%s%s"', #opt > 1 and '-' or '', opt))
 end
 
-local function err_unknown_opt (opt)
-   exit_with_error ("Unknown option `-" ..
-		    (#opt > 1 and "-" or "") .. opt .. "'\n", 1)
-end
+-- resolve aliases into their canonical forms
+local function canonicalize(options, opt)
+    if not options[opt] then
+        err_unknown_opt(opt)
+    end
 
-local function canonize (options, opt)
-   if not options [opt] then
-      err_unknown_opt (opt)
-   end
+    while type(options[opt]) == 'string' do
+        opt = options[opt]
 
-   while type (options [opt]) == "string" do
-      opt = options [opt]
+        if not options[opt] then
+             err_unknown_opt(opt)
+        end
+    end
 
-      if not options [opt] then
-	 err_unknown_opt (opt)
-      end
-   end
+    if type(options[opt]) ~= 'number' then
+        qerror(string.format(
+                'Option "%s" resolves to non-number for has_arg flag', opt))
+    end
 
-   return opt
+    return opt
 end
 
-function get_ordered_opts (arg, sh_opts, long_opts)
-   local i      = 1
-   local count  = 1
-   local opts   = {}
-   local optarg = {}
-
-   local options = convert_short2long (sh_opts)
-   for k,v in pairs (long_opts) do
-      options [k] = v
-   end
-
-   while i <= #arg do
-      local a = arg [i]
-
-      if a == "--" then
-	 i = i + 1
-	 break
-
-      elseif a == "-" then
-	 break
-
-      elseif a:sub (1, 2) == "--" then
-	 local pos = a:find ("=", 1, true)
-
-	 if pos then
-	    local opt = a:sub (3, pos-1)
-
-	    opt = canonize (options, opt)
-
-	    if options [opt] == 0 then
-	       exit_with_error ("Bad usage of option `" .. a .. "'\n", 1)
-	    end
-
-	    optarg [count] = a:sub (pos+1)
-	    opts [count] = opt
-	 else
-	    local opt = a:sub (3)
-
-	    opt = canonize (options, opt)
-
-	    if options [opt] == 0 then
-	       opts [count] = opt
-	    else
-	       if i == #arg then
-		  exit_with_error ("Missed value for option `" .. a .. "'\n", 1)
-	       end
-
-	       optarg [count] = arg [i+1]
-	       opts [count] = opt
-	       i = i + 1
-	    end
-	 end
-	 count = count + 1
-
-      elseif a:sub (1, 1) == "-" then
-	 local j
-	 for j=2,a:len () do
-	    local opt = canonize (options, a:sub (j, j))
-
-	    if options [opt] == 0 then
-	       opts [count] = opt
-	       count = count + 1
-	    elseif a:len () == j then
-	       if i == #arg then
-		  exit_with_error ("Missed value for option `-" .. opt .. "'\n", 1)
-	       end
-
-	       optarg [count] = arg [i+1]
-	       opts [count] = opt
-	       i = i + 1
-	       count = count + 1
-	       break
-	    else
-	       optarg [count] = a:sub (j+1)
-	       opts [count] = opt
-	       count = count + 1
-	       break
-	    end
-	 end
-      else
-	 break
-      end
-
-      i = i + 1
-   end
-
-   return opts,i,optarg
+local function has_arg(options, opt)
+    return options[canonicalize(options, opt)] == 1
 end
 
-function get_opts (arg, sh_opts, long_opts)
-   local ret = {}
-
-   local opts,optind,optarg = get_ordered_opts (arg, sh_opts, long_opts)
-   for i,v in ipairs (opts) do
-      if optarg [i] then
-	 ret [v] = optarg [i]
-      else
-	 ret [v] = 1
-      end
-   end
+-- returns vectors for opts, optargs, and nonoptions
+function get_ordered_opts(args, sh_opts, long_opts)
+    local optind, count, opts, optargs, nonoptions = 1, 1, {}, {}, {}
+
+    local options = get_opt_map(sh_opts)
+    for k,v in pairs(long_opts) do
+        options[k] = v
+    end
+
+    while optind <= #args do
+        local a = args[optind]
+        if a == '--' then
+            optind = optind + 1
+        elseif a:sub(1, 2) == '--' then
+            local pos = a:find('=', 1, true)
+            if pos then
+                local opt = a:sub(3, pos-1)
+                if not has_arg(options, opt) then
+                    qerror(string.format('Bad usage of option "%s"', a))
+                end
+                opts[count] = opt
+                optargs[count] = a:sub(pos+1)
+            else
+                local opt = a:sub(3)
+                opts[count] = opt
+                if has_arg(options, opt) then
+                    if i == #args then
+                        qerror(string.format(
+                                'Missing value for option "%s"', a))
+                    end
+                    optargs[count] = args[optind+1]
+                    optind = optind + 1
+                end
+            end
+            count = count + 1
+        elseif a:sub(1, 1) == '-' then
+            local j
+            for j=2,#a do
+                local opt = canonicalize(options, a:sub(j, j))
+                if not has_arg(options, opt) then
+                    opts[count] = opt
+                    count = count + 1
+                elseif j == #a then
+                    if optind == #args then
+                        qerror(string.format(
+                                'Missing value for option "-%s"', opt))
+                    end
+                    opts[count] = opt
+                    optargs[count] = args[optind+1]
+                    optind = optind + 1
+                    count = count + 1
+                else
+                    opts[count] = opt
+                    optargs[count] = a:sub(j+1)
+                    count = count + 1
+                    break
+                end
+            end
+        else
+            table.insert(nonoptions, args[optind])
+        end
+        optind = optind + 1
+    end
+    for i=optind,#args do
+        table.insert(nonoptions, args[i])
+    end
+    return opts, optargs, nonoptions
+end
 
-   return ret,optind
+-- returns a map of options to their optargs (or 1 if the option doesn't take an
+-- argument), and a vector for nonoptions
+function get_opts(args, sh_opts, long_opts)
+    local ret = {}
+
+    local opts,optargs,nonoptions = get_ordered_opts(args, sh_opts, long_opts)
+    for i,v in ipairs(opts) do
+        if optarg[i] then
+            ret[v] = optarg[i]
+        else
+            ret[v] = 1
+        end
+    end
+
+    return ret, nonoptions
 end
 
 return _ENV

From 175776812e32e93ec9c85efb5576e4b10663aadf Mon Sep 17 00:00:00 2001
From: Myk Taylor <myk.taylor@gmail.com>
Date: Tue, 3 Nov 2020 16:19:50 -0800
Subject: [PATCH 11/31] implement a nice API wrapper for getopt

---
 library/lua/utils.lua | 49 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/library/lua/utils.lua b/library/lua/utils.lua
index 43dcdcd60..ee2cadd29 100644
--- a/library/lua/utils.lua
+++ b/library/lua/utils.lua
@@ -613,6 +613,55 @@ function processArgs(args, validArgs)
     return result
 end
 
+-- processes commandline options according to optionActions and returns all
+-- argument strings that are not options.
+--
+-- optionActions is a vector with elements in the following format:
+-- {shortOptionName, longOptionAlias, hasArg=boolean, handler=fn}
+-- shortOptionName and handler are required. If the option takes an argument,
+-- it will be passed to the handler function.
+-- longOptionAlias is optional.
+-- hasArgument defaults to false.
+--
+-- example usage:
+--
+-- local filename = nil
+-- local open_readonly = false
+-- local nonoptions = processArgs2(args, {
+--   {'r', handler=function() open_readonly = true end},
+--   {'f', 'filename', hasArg=true,
+--    handler=function(optarg) filename = optarg end}
+-- })
+--
+-- when args is {'first', '-f', 'fname', 'second'}
+-- then filename will be fname and nonoptions will contain {'first', 'second'}
+function processArgs2(args, optionActions)
+    local sh_opts, long_opts = '', {}
+    local handlers = {}
+    for _,optionAction in ipairs(optionActions) do
+        local sh_opt,long_opt = optionAction[1], optionAction[2]
+        if not sh_opt or type(sh_opt) ~= 'string' or #sh_opt ~= 1 then
+            qerror('optionAction missing option letter at index 1')
+        end
+        if not optionAction.handler then
+            qerror(string.format('handler missing for option "%s"', sh_opt))
+        end
+        sh_opts = sh_opts .. sh_opt
+        if optionAction.hasArg then sh_opts = sh_opts .. ':' end
+        handlers[sh_opt] = optionAction.handler
+        if long_opt then
+            long_opts[long_opt] = sh_opt
+            handlers[long_opt] = optionAction.handler
+        end
+    end
+    local getopt = require('3rdparty.alt_getopt').get_ordered_opts
+    local opts, optargs, nonoptions = getopt(args, sh_opts, long_opts)
+    for i,v in ipairs(opts) do
+        handlers[v](optargs[i])
+    end
+    return nonoptions
+end
+
 function fillTable(table1,table2)
     for k,v in pairs(table2) do
         table1[k] = v

From 4e1c70bc06d93913c70b6ca46de22220546a240d Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Sun, 10 Jan 2021 17:10:44 -0800
Subject: [PATCH 12/31] update changelog

---
 docs/changelog.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index fdc7c13e7..0dc119742 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -36,6 +36,9 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 ## Fixes
 - `embark-assistant`: fixed order of factors when calculating min temperature
 
+## API
+- ``processArgs2()`` added to utils.lua, providing a callback interface for parameter parsing and getopt-like flexibility for parameter ordering and combination (see docs in ``library/lua/utils.lua`` and ``library/lua/3rdparty/alt_getopt.lua`` for details).
+
 # 0.47.04-r4
 
 ## Fixes

From 4300c185d733d8085e161923976810147d2df93f Mon Sep 17 00:00:00 2001
From: Myk <myk002@yahoo.com>
Date: Tue, 12 Jan 2021 22:46:18 -0800
Subject: [PATCH 13/31] Update docs/changelog.txt

Co-authored-by: Alan <lethosor@users.noreply.github.com>
---
 docs/changelog.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 0dc119742..65413f9f1 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -36,7 +36,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 ## Fixes
 - `embark-assistant`: fixed order of factors when calculating min temperature
 
-## API
+## Lua
 - ``processArgs2()`` added to utils.lua, providing a callback interface for parameter parsing and getopt-like flexibility for parameter ordering and combination (see docs in ``library/lua/utils.lua`` and ``library/lua/3rdparty/alt_getopt.lua`` for details).
 
 # 0.47.04-r4
@@ -1171,4 +1171,3 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 - The ``ui_menu_width`` global is now a 2-byte array; the second item is the former ``ui_area_map_width`` global, which is now removed
 - The former ``announcements`` global is now a field in ``d_init``
 - ``world`` fields formerly beginning with ``job_`` are now fields of ``world.jobs``, e.g. ``world.job_list`` is now ``world.jobs.list``
-

From 39d274b00e7474033c2e6f644ca18486f1009364 Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Tue, 12 Jan 2021 23:42:53 -0800
Subject: [PATCH 14/31] address review comments

---
 LICENSE.rst           |  2 ++
 library/lua/utils.lua | 20 +++++++++++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/LICENSE.rst b/LICENSE.rst
index f1e07d111..48bcfbe95 100644
--- a/LICENSE.rst
+++ b/LICENSE.rst
@@ -34,6 +34,7 @@ tinythread_     Zlib          \(c\) 2010, Marcus Geelnard
 tinyxml_        Zlib          \(c\) 2000-2006, Lee Thomason
 UTF-8-decoder_  MIT           \(c\) 2008-2010, Bjoern Hoehrmann
 xlsxio_         MIT           \(c\) 2016-2020, Brecht Sanders
+alt-getopt_     MIT           \(c\) 2009 Aleksey Cheusov
 =============== ============= =================================================
 
 .. _DFHack: https://github.com/DFHack/dfhack
@@ -52,6 +53,7 @@ xlsxio_         MIT           \(c\) 2016-2020, Brecht Sanders
 .. _tinyxml: http://www.sourceforge.net/projects/tinyxml
 .. _UTF-8-decoder: http://bjoern.hoehrmann.de/utf-8/decoder/dfa
 .. _xlsxio: https://github.com/brechtsanders/xlsxio
+.. _alt-getopt: https://github.com/LuaDist/alt-getopt
 
 .. _CC-BY-SA: http://creativecommons.org/licenses/by/3.0/deed.en_US
 
diff --git a/library/lua/utils.lua b/library/lua/utils.lua
index ee2cadd29..c652e54d4 100644
--- a/library/lua/utils.lua
+++ b/library/lua/utils.lua
@@ -1,6 +1,7 @@
 local _ENV = mkmodule('utils')
 
 local df = df
+local getopt = require('3rdparty.alt_getopt')
 
 -- Comparator function
 function compare(a,b)
@@ -614,7 +615,10 @@ function processArgs(args, validArgs)
 end
 
 -- processes commandline options according to optionActions and returns all
--- argument strings that are not options.
+-- argument strings that are not options. Options and non-option strings can
+-- appear in any order, and single-letter options that do not take arguments
+-- can be combined into a single option string (e.g. '-abc' is the same as
+-- '-a -b -c' if options 'a' and 'b' do not take arguments.
 --
 -- optionActions is a vector with elements in the following format:
 -- {shortOptionName, longOptionAlias, hasArg=boolean, handler=fn}
@@ -627,15 +631,17 @@ end
 --
 -- local filename = nil
 -- local open_readonly = false
--- local nonoptions = processArgs2(args, {
+-- local nonoptions = processArgsGetopt(args, {
 --   {'r', handler=function() open_readonly = true end},
 --   {'f', 'filename', hasArg=true,
 --    handler=function(optarg) filename = optarg end}
 -- })
 --
--- when args is {'first', '-f', 'fname', 'second'}
--- then filename will be fname and nonoptions will contain {'first', 'second'}
-function processArgs2(args, optionActions)
+-- when args is {'first', '-f', 'fname', 'second'} or, equivalently,
+-- {'first', '--filename', 'fname', 'second'} (note the double dash in front of
+-- the long option alias), then filename will be fname and nonoptions will
+-- contain {'first', 'second'}.
+function processArgsGetopt(args, optionActions)
     local sh_opts, long_opts = '', {}
     local handlers = {}
     for _,optionAction in ipairs(optionActions) do
@@ -654,8 +660,8 @@ function processArgs2(args, optionActions)
             handlers[long_opt] = optionAction.handler
         end
     end
-    local getopt = require('3rdparty.alt_getopt').get_ordered_opts
-    local opts, optargs, nonoptions = getopt(args, sh_opts, long_opts)
+    local opts, optargs, nonoptions =
+            getopt.get_ordered_opts(args, sh_opts, long_opts)
     for i,v in ipairs(opts) do
         handlers[v](optargs[i])
     end

From cc687673367d7b9ea913e73deaf6deaa9d9fa478 Mon Sep 17 00:00:00 2001
From: bseiller <github_public_commit@bseiller.de>
Date: Mon, 18 Jan 2021 23:23:50 +0100
Subject: [PATCH 15/31] created struct mid_level_tile_incursion_base that only
 contains attributes that are relevant for incursion processing

- def.h: make attributes/fields of mid_level_tile_incursion_base available in mid_level_tile by inheriting from mid_level_tile_incursion_base which also allows treating mid_level_tile as a mid_level_tile_incursion_base
---
 plugins/embark-assistant/defs.h | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/plugins/embark-assistant/defs.h b/plugins/embark-assistant/defs.h
index 0f34b288a..99db1f7a7 100644
--- a/plugins/embark-assistant/defs.h
+++ b/plugins/embark-assistant/defs.h
@@ -68,24 +68,29 @@ namespace embark_assist {
             Woodland,
             Heavily_Forested
         };
-
-        struct mid_level_tile {
+		
+		// only contains those attributes that are being handled during incursion processing
+		struct mid_level_tile_incursion_base {
             uint8_t aquifer = Clear_Aquifer_Bits;
             bool clay = false;
             bool sand = false;
+            int8_t soil_depth;
+            int16_t elevation;
+            int8_t biome_offset;
+            tree_levels trees;
+            uint8_t savagery_level;  // 0 - 2
+            uint8_t evilness_level;  // 0 - 2
+        };
+
+		// contains all attributes (some by inheritance), used for regular survey/matching
+        struct mid_level_tile : public mid_level_tile_incursion_base {
             bool flux = false;
             bool coal = false;
-            int8_t soil_depth;
             int8_t offset;
-            int16_t elevation;
             river_sizes river_size = river_sizes::None;
             int16_t river_elevation = 100;
             int8_t adamantine_level;  // -1 = none, 0 .. 3 = cavern 1 .. magma sea. Currently not used beyond present/absent.
             int8_t magma_level;  // -1 = none, 0 .. 3 = cavern 3 .. surface/volcano
-            int8_t biome_offset;
-            tree_levels trees;
-            uint8_t savagery_level;  // 0 - 2
-            uint8_t evilness_level;  // 0 - 2
             std::vector<bool> metals;
             std::vector<bool> economics;
             std::vector<bool> minerals;

From bdfd50cc6551645dff6339161b14f83591ffda2f Mon Sep 17 00:00:00 2001
From: bseiller <github_public_commit@bseiller.de>
Date: Mon, 18 Jan 2021 23:50:14 +0100
Subject: [PATCH 16/31] using mid_level_tile_incursion_base instead of
 mid_level_tile for incursion processing

- matcher.cpp, survey.cpp: adapting signatures to use new struct that only contains incursion specific attributes
---
 plugins/embark-assistant/matcher.cpp | 4 ++--
 plugins/embark-assistant/survey.cpp  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/embark-assistant/matcher.cpp b/plugins/embark-assistant/matcher.cpp
index 2b5e4728b..36a43d9b3 100644
--- a/plugins/embark-assistant/matcher.cpp
+++ b/plugins/embark-assistant/matcher.cpp
@@ -68,7 +68,7 @@ namespace embark_assist {
 
         void process_embark_incursion(matcher_info *result,
             embark_assist::defs::world_tile_data *survey_results,
-            embark_assist::defs::mid_level_tile *mlt,  // Note this is a single tile, as opposed to most usages of this variable name.
+            embark_assist::defs::mid_level_tile_incursion_base *mlt,  // Note this is a single tile, as opposed to most usages of this variable name.
             embark_assist::defs::finders *finder,
             int16_t elevation,
             uint16_t x,
@@ -2699,7 +2699,7 @@ namespace embark_assist {
 
         void merge_incursion_into_world_tile(embark_assist::defs::region_tile_datum* current,
             embark_assist::defs::region_tile_datum* target_tile,
-            embark_assist::defs::mid_level_tile* target_mlt) {
+            embark_assist::defs::mid_level_tile_incursion_base* target_mlt) {
             df::world_data* world_data = world->world_data;
 
             current->aquifer |= target_mlt->aquifer;
diff --git a/plugins/embark-assistant/survey.cpp b/plugins/embark-assistant/survey.cpp
index e0deb5e58..300750a29 100644
--- a/plugins/embark-assistant/survey.cpp
+++ b/plugins/embark-assistant/survey.cpp
@@ -455,7 +455,7 @@ namespace embark_assist {
 
         void process_embark_incursion(embark_assist::defs::site_infos *site_info,
             embark_assist::defs::world_tile_data *survey_results,
-            embark_assist::defs::mid_level_tile *mlt,  // Note this is a single tile, as opposed to most usages of this variable name.
+            embark_assist::defs::mid_level_tile_incursion_base *mlt,  // Note this is a single tile, as opposed to most usages of this variable name.
             int16_t elevation,
             uint16_t x,
             uint16_t y) {

From e99c8faa24b948fb56365a868d2791568c6c644c Mon Sep 17 00:00:00 2001
From: bseiller <github_public_commit@bseiller.de>
Date: Tue, 19 Jan 2021 00:01:54 +0100
Subject: [PATCH 17/31] switching to mid_level_tile_incursion_base to store
 incursion data of world tile edges

- defs.h: using mid_level_tile_incursion_base in region_tile_datum to store incursion data of world tile edges
- survey.cpp: commented out "not used" blocks of assignment in survey_mid_level_tile that no longer make sense now
---
 plugins/embark-assistant/defs.h     |  8 +--
 plugins/embark-assistant/survey.cpp | 80 ++++++++++++++---------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/plugins/embark-assistant/defs.h b/plugins/embark-assistant/defs.h
index 99db1f7a7..a67660545 100644
--- a/plugins/embark-assistant/defs.h
+++ b/plugins/embark-assistant/defs.h
@@ -146,10 +146,10 @@ namespace embark_assist {
             std::vector<bool> minerals;
             std::vector<int16_t> neighbors;  //  entity_raw indices
             uint8_t necro_neighbors;
-            mid_level_tile north_row[16];
-            mid_level_tile south_row[16];
-            mid_level_tile west_column[16];
-            mid_level_tile east_column[16];
+            mid_level_tile_incursion_base north_row[16];
+            mid_level_tile_incursion_base south_row[16];
+            mid_level_tile_incursion_base west_column[16];
+            mid_level_tile_incursion_base east_column[16];
             uint8_t north_corner_selection[16]; //  0 - 3. For some reason DF stores everything needed for incursion
             uint8_t west_corner_selection[16];  //  detection in 17:th row/colum data in the region details except
                                                 //  this info, so we have to go to neighboring world tiles to fetch it.
diff --git a/plugins/embark-assistant/survey.cpp b/plugins/embark-assistant/survey.cpp
index 300750a29..fac5af267 100644
--- a/plugins/embark-assistant/survey.cpp
+++ b/plugins/embark-assistant/survey.cpp
@@ -1381,50 +1381,50 @@ void embark_assist::survey::survey_mid_level_tile(embark_assist::defs::geo_data
         tile->west_column[i].sand = mlt->at(0).at(i).sand;
         tile->east_column[i].sand = mlt->at(15).at(i).sand;
 
-        tile->north_row[i].flux = mlt->at(i).at(0).flux;  //  Not used
-        tile->south_row[i].flux = mlt->at(i).at(15).flux;
-        tile->west_column[i].flux = mlt->at(0).at(i).flux;
-        tile->east_column[i].flux = mlt->at(15).at(i).flux;
+        // tile->north_row[i].flux = mlt->at(i).at(0).flux;  //  Not used
+        // tile->south_row[i].flux = mlt->at(i).at(15).flux;
+        // tile->west_column[i].flux = mlt->at(0).at(i).flux;
+        // tile->east_column[i].flux = mlt->at(15).at(i).flux;
 
-        tile->north_row[i].coal = mlt->at(i).at(0).coal; //  Not used
-        tile->south_row[i].coal = mlt->at(i).at(15).coal;
-        tile->west_column[i].coal = mlt->at(0).at(i).coal;
-        tile->east_column[i].coal = mlt->at(15).at(i).coal;
+        // tile->north_row[i].coal = mlt->at(i).at(0).coal; //  Not used
+        // tile->south_row[i].coal = mlt->at(i).at(15).coal;
+        // tile->west_column[i].coal = mlt->at(0).at(i).coal;
+        // tile->east_column[i].coal = mlt->at(15).at(i).coal;
 
         tile->north_row[i].soil_depth = mlt->at(i).at(0).soil_depth;
         tile->south_row[i].soil_depth = mlt->at(i).at(15).soil_depth;
         tile->west_column[i].soil_depth = mlt->at(0).at(i).soil_depth;
         tile->east_column[i].soil_depth = mlt->at(15).at(i).soil_depth;
 
-        tile->north_row[i].offset = mlt->at(i).at(0).offset; //  Not used
-        tile->south_row[i].offset = mlt->at(i).at(15).offset;
-        tile->west_column[i].offset = mlt->at(0).at(i).offset;
-        tile->east_column[i].offset = mlt->at(15).at(i).offset;
+        // tile->north_row[i].offset = mlt->at(i).at(0).offset; //  Not used
+        // tile->south_row[i].offset = mlt->at(i).at(15).offset;
+        // tile->west_column[i].offset = mlt->at(0).at(i).offset;
+        // tile->east_column[i].offset = mlt->at(15).at(i).offset;
 
         tile->north_row[i].elevation = mlt->at(i).at(0).elevation;
         tile->south_row[i].elevation = mlt->at(i).at(15).elevation;
         tile->west_column[i].elevation = mlt->at(0).at(i).elevation;
         tile->east_column[i].elevation = mlt->at(15).at(i).elevation;
 
-        tile->north_row[i].river_size = mlt->at(i).at(0).river_size; //  Not used
-        tile->south_row[i].river_size = mlt->at(i).at(15).river_size;
-        tile->west_column[i].river_size = mlt->at(0).at(i).river_size;
-        tile->east_column[i].river_size = mlt->at(15).at(i).river_size;
+        // tile->north_row[i].river_size = mlt->at(i).at(0).river_size; //  Not used
+        // tile->south_row[i].river_size = mlt->at(i).at(15).river_size;
+        // tile->west_column[i].river_size = mlt->at(0).at(i).river_size;
+        // tile->east_column[i].river_size = mlt->at(15).at(i).river_size;
 
-        tile->north_row[i].river_elevation = mlt->at(i).at(0).river_elevation; //  Not used
-        tile->south_row[i].river_elevation = mlt->at(i).at(15).river_elevation;
-        tile->west_column[i].river_elevation = mlt->at(0).at(i).river_elevation;
-        tile->east_column[i].river_elevation = mlt->at(15).at(i).river_elevation;
+        // tile->north_row[i].river_elevation = mlt->at(i).at(0).river_elevation; //  Not used
+        // tile->south_row[i].river_elevation = mlt->at(i).at(15).river_elevation;
+        // tile->west_column[i].river_elevation = mlt->at(0).at(i).river_elevation;
+        // tile->east_column[i].river_elevation = mlt->at(15).at(i).river_elevation;
 
-        tile->north_row[i].adamantine_level = mlt->at(i).at(0).adamantine_level; //  Not used
-        tile->south_row[i].adamantine_level = mlt->at(i).at(15).adamantine_level;
-        tile->west_column[i].adamantine_level = mlt->at(0).at(i).adamantine_level;
-        tile->east_column[i].adamantine_level = mlt->at(15).at(i).adamantine_level;
+        // tile->north_row[i].adamantine_level = mlt->at(i).at(0).adamantine_level; //  Not used
+        // tile->south_row[i].adamantine_level = mlt->at(i).at(15).adamantine_level;
+        // tile->west_column[i].adamantine_level = mlt->at(0).at(i).adamantine_level;
+        // tile->east_column[i].adamantine_level = mlt->at(15).at(i).adamantine_level;
 
-        tile->north_row[i].magma_level = mlt->at(i).at(0).magma_level; //  Not used
-        tile->south_row[i].magma_level = mlt->at(i).at(15).magma_level;
-        tile->west_column[i].magma_level = mlt->at(0).at(i).magma_level;
-        tile->east_column[i].magma_level = mlt->at(15).at(i).magma_level;
+        // tile->north_row[i].magma_level = mlt->at(i).at(0).magma_level; //  Not used
+        // tile->south_row[i].magma_level = mlt->at(i).at(15).magma_level;
+        // tile->west_column[i].magma_level = mlt->at(0).at(i).magma_level;
+        // tile->east_column[i].magma_level = mlt->at(15).at(i).magma_level;
 
         tile->north_row[i].biome_offset = mlt->at(i).at(0).biome_offset;
         tile->south_row[i].biome_offset = mlt->at(i).at(15).biome_offset;
@@ -1446,20 +1446,20 @@ void embark_assist::survey::survey_mid_level_tile(embark_assist::defs::geo_data
         tile->west_column[i].evilness_level = mlt->at(0).at(i).evilness_level;
         tile->east_column[i].evilness_level = mlt->at(15).at(i).evilness_level;
 
-        tile->north_row[i].metals.resize(0);  //  Not used
-        tile->south_row[i].metals.resize(0);
-        tile->west_column[i].metals.resize(0);
-        tile->east_column[i].metals.resize(0);
+        // tile->north_row[i].metals.resize(0);  //  Not used
+        // tile->south_row[i].metals.resize(0);
+        // tile->west_column[i].metals.resize(0);
+        // tile->east_column[i].metals.resize(0);
 
-        tile->north_row[i].economics.resize(0);  //  Not used
-        tile->south_row[i].economics.resize(0);
-        tile->west_column[i].economics.resize(0);
-        tile->east_column[i].economics.resize(0);
+        // tile->north_row[i].economics.resize(0);  //  Not used
+        // tile->south_row[i].economics.resize(0);
+        // tile->west_column[i].economics.resize(0);
+        // tile->east_column[i].economics.resize(0);
 
-        tile->north_row[i].minerals.resize(0);  //  Not used
-        tile->south_row[i].minerals.resize(0);
-        tile->west_column[i].minerals.resize(0);
-        tile->east_column[i].minerals.resize(0);
+        // tile->north_row[i].minerals.resize(0);  //  Not used
+        // tile->south_row[i].minerals.resize(0);
+        // tile->west_column[i].minerals.resize(0);
+        // tile->east_column[i].minerals.resize(0);
 
         tile->north_corner_selection[i] = world_data->region_details[0]->edges.biome_corner[i][0];
         tile->west_corner_selection[i] = world_data->region_details[0]->edges.biome_corner[0][i];

From cb496c3f59c8f78b40b89eaaa4ff590585806eca Mon Sep 17 00:00:00 2001
From: bseiller <github_public_commit@bseiller.de>
Date: Tue, 19 Jan 2021 00:07:28 +0100
Subject: [PATCH 18/31] fixing indention/whitespaces

- defs.h: replacing tab with space/blank for indents
---
 plugins/embark-assistant/defs.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plugins/embark-assistant/defs.h b/plugins/embark-assistant/defs.h
index a67660545..9634d7c48 100644
--- a/plugins/embark-assistant/defs.h
+++ b/plugins/embark-assistant/defs.h
@@ -68,9 +68,9 @@ namespace embark_assist {
             Woodland,
             Heavily_Forested
         };
-		
-		// only contains those attributes that are being handled during incursion processing
-		struct mid_level_tile_incursion_base {
+        
+        // only contains those attributes that are being handled during incursion processing
+        struct mid_level_tile_incursion_base {
             uint8_t aquifer = Clear_Aquifer_Bits;
             bool clay = false;
             bool sand = false;
@@ -82,7 +82,7 @@ namespace embark_assist {
             uint8_t evilness_level;  // 0 - 2
         };
 
-		// contains all attributes (some by inheritance), used for regular survey/matching
+        // contains all attributes (some by inheritance), used for regular survey/matching
         struct mid_level_tile : public mid_level_tile_incursion_base {
             bool flux = false;
             bool coal = false;

From 9b9373be4fee372e90ddb5ca0188154c5fe18c8c Mon Sep 17 00:00:00 2001
From: bseiller <github_public_commit@bseiller.de>
Date: Tue, 19 Jan 2021 12:06:36 +0100
Subject: [PATCH 19/31] removing commented out assigments

---
 plugins/embark-assistant/survey.cpp | 50 -----------------------------
 1 file changed, 50 deletions(-)

diff --git a/plugins/embark-assistant/survey.cpp b/plugins/embark-assistant/survey.cpp
index fac5af267..3691976f2 100644
--- a/plugins/embark-assistant/survey.cpp
+++ b/plugins/embark-assistant/survey.cpp
@@ -1381,51 +1381,16 @@ void embark_assist::survey::survey_mid_level_tile(embark_assist::defs::geo_data
         tile->west_column[i].sand = mlt->at(0).at(i).sand;
         tile->east_column[i].sand = mlt->at(15).at(i).sand;
 
-        // tile->north_row[i].flux = mlt->at(i).at(0).flux;  //  Not used
-        // tile->south_row[i].flux = mlt->at(i).at(15).flux;
-        // tile->west_column[i].flux = mlt->at(0).at(i).flux;
-        // tile->east_column[i].flux = mlt->at(15).at(i).flux;
-
-        // tile->north_row[i].coal = mlt->at(i).at(0).coal; //  Not used
-        // tile->south_row[i].coal = mlt->at(i).at(15).coal;
-        // tile->west_column[i].coal = mlt->at(0).at(i).coal;
-        // tile->east_column[i].coal = mlt->at(15).at(i).coal;
-
         tile->north_row[i].soil_depth = mlt->at(i).at(0).soil_depth;
         tile->south_row[i].soil_depth = mlt->at(i).at(15).soil_depth;
         tile->west_column[i].soil_depth = mlt->at(0).at(i).soil_depth;
         tile->east_column[i].soil_depth = mlt->at(15).at(i).soil_depth;
 
-        // tile->north_row[i].offset = mlt->at(i).at(0).offset; //  Not used
-        // tile->south_row[i].offset = mlt->at(i).at(15).offset;
-        // tile->west_column[i].offset = mlt->at(0).at(i).offset;
-        // tile->east_column[i].offset = mlt->at(15).at(i).offset;
-
         tile->north_row[i].elevation = mlt->at(i).at(0).elevation;
         tile->south_row[i].elevation = mlt->at(i).at(15).elevation;
         tile->west_column[i].elevation = mlt->at(0).at(i).elevation;
         tile->east_column[i].elevation = mlt->at(15).at(i).elevation;
 
-        // tile->north_row[i].river_size = mlt->at(i).at(0).river_size; //  Not used
-        // tile->south_row[i].river_size = mlt->at(i).at(15).river_size;
-        // tile->west_column[i].river_size = mlt->at(0).at(i).river_size;
-        // tile->east_column[i].river_size = mlt->at(15).at(i).river_size;
-
-        // tile->north_row[i].river_elevation = mlt->at(i).at(0).river_elevation; //  Not used
-        // tile->south_row[i].river_elevation = mlt->at(i).at(15).river_elevation;
-        // tile->west_column[i].river_elevation = mlt->at(0).at(i).river_elevation;
-        // tile->east_column[i].river_elevation = mlt->at(15).at(i).river_elevation;
-
-        // tile->north_row[i].adamantine_level = mlt->at(i).at(0).adamantine_level; //  Not used
-        // tile->south_row[i].adamantine_level = mlt->at(i).at(15).adamantine_level;
-        // tile->west_column[i].adamantine_level = mlt->at(0).at(i).adamantine_level;
-        // tile->east_column[i].adamantine_level = mlt->at(15).at(i).adamantine_level;
-
-        // tile->north_row[i].magma_level = mlt->at(i).at(0).magma_level; //  Not used
-        // tile->south_row[i].magma_level = mlt->at(i).at(15).magma_level;
-        // tile->west_column[i].magma_level = mlt->at(0).at(i).magma_level;
-        // tile->east_column[i].magma_level = mlt->at(15).at(i).magma_level;
-
         tile->north_row[i].biome_offset = mlt->at(i).at(0).biome_offset;
         tile->south_row[i].biome_offset = mlt->at(i).at(15).biome_offset;
         tile->west_column[i].biome_offset = mlt->at(0).at(i).biome_offset;
@@ -1446,21 +1411,6 @@ void embark_assist::survey::survey_mid_level_tile(embark_assist::defs::geo_data
         tile->west_column[i].evilness_level = mlt->at(0).at(i).evilness_level;
         tile->east_column[i].evilness_level = mlt->at(15).at(i).evilness_level;
 
-        // tile->north_row[i].metals.resize(0);  //  Not used
-        // tile->south_row[i].metals.resize(0);
-        // tile->west_column[i].metals.resize(0);
-        // tile->east_column[i].metals.resize(0);
-
-        // tile->north_row[i].economics.resize(0);  //  Not used
-        // tile->south_row[i].economics.resize(0);
-        // tile->west_column[i].economics.resize(0);
-        // tile->east_column[i].economics.resize(0);
-
-        // tile->north_row[i].minerals.resize(0);  //  Not used
-        // tile->south_row[i].minerals.resize(0);
-        // tile->west_column[i].minerals.resize(0);
-        // tile->east_column[i].minerals.resize(0);
-
         tile->north_corner_selection[i] = world_data->region_details[0]->edges.biome_corner[i][0];
         tile->west_corner_selection[i] = world_data->region_details[0]->edges.biome_corner[0][i];
         tile->north_row_biome_x[i] = world_data->region_details[0]->edges.biome_x[i][0];

From 03719f58df35526c8440bc13f427c2bc4bfd7b7e Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Sun, 24 Jan 2021 08:52:34 -0800
Subject: [PATCH 20/31] avoid infinite parentage when linking rooms

---
 docs/changelog.txt            | 1 +
 library/modules/Buildings.cpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 758a54fb5..735c0716e 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -36,6 +36,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 ## Fixes
 - `embark-assistant`: fixed order of factors when calculating min temperature
 - `embark-assistant`: improved performance of surveying
+- `quickfort`: creating zones no longer causes eventual crashes
 
 ## Misc Improvements
 - `buildingplan`: set global settings from the ``DFHack#`` prompt: e.g.  ``buildingplan set boulders false``
diff --git a/library/modules/Buildings.cpp b/library/modules/Buildings.cpp
index 5a0cd37dc..bd8dcb9de 100644
--- a/library/modules/Buildings.cpp
+++ b/library/modules/Buildings.cpp
@@ -885,7 +885,7 @@ static void linkRooms(df::building *bld)
             continue;
 
         df::building_extents_type *pext = getExtentTile(room->room, df::coord2d(bld->x1, bld->y1));
-        if (!pext || !*pext)
+        if (!pext || !*pext || room == bld)
             continue;
 
         changed = true;

From 32a0363f3185f49af12f06a6cae6301d38475813 Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Sun, 24 Jan 2021 08:57:56 -0800
Subject: [PATCH 21/31] move identity check up a bit to avoid useless work

---
 library/modules/Buildings.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/library/modules/Buildings.cpp b/library/modules/Buildings.cpp
index bd8dcb9de..3dab347d5 100644
--- a/library/modules/Buildings.cpp
+++ b/library/modules/Buildings.cpp
@@ -881,11 +881,11 @@ static void linkRooms(df::building *bld)
     for (size_t i = 0; i < vec.size(); i++)
     {
         auto room = vec[i];
-        if (!room->is_room || room->z != bld->z)
+        if (!room->is_room || room->z != bld->z || room == bld)
             continue;
 
         df::building_extents_type *pext = getExtentTile(room->room, df::coord2d(bld->x1, bld->y1));
-        if (!pext || !*pext || room == bld)
+        if (!pext || !*pext)
             continue;
 
         changed = true;

From d0fc448a39aaf6a03d8e9252c31d3b61c2141e87 Mon Sep 17 00:00:00 2001
From: Myk Taylor <myk.taylor@gmail.com>
Date: Wed, 4 Nov 2020 08:52:19 -0800
Subject: [PATCH 22/31] add 'enable all' option for buildingplan

This kind of functionality is much more important now than it used to
be since there are so many supported building types.

Also modified the 'Planning Mode' status on the building placement
screen to reflect whether we're in quickfort mode, enable all mode, or
whether just the building type is enabled.

this setting is not persisted (just like quickfort_mode is not
persisted), but it can be set from onMapLoad.init
---
 plugins/buildingplan.cpp     | 26 ++++++++++++++++++++++----
 plugins/lua/buildingplan.lua |  9 +++++++++
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/plugins/buildingplan.cpp b/plugins/buildingplan.cpp
index e68f9ca4b..afdfa80d0 100644
--- a/plugins/buildingplan.cpp
+++ b/plugins/buildingplan.cpp
@@ -27,6 +27,7 @@ REQUIRE_GLOBAL(world); // used in buildingplan library
 
 bool show_help = false;
 bool quickfort_mode = false;
+bool all_enabled = false;
 bool in_dummy_screen = false;
 std::unordered_map<BuildingTypeKey, bool, BuildingTypeKeyHash> planmode_enabled;
 
@@ -284,7 +285,7 @@ void ViewscreenChooseMaterial::render()
 //START Viewscreen Hook
 static bool is_planmode_enabled(BuildingTypeKey key)
 {
-    return planmode_enabled[key] || quickfort_mode;
+    return planmode_enabled[key] || quickfort_mode || all_enabled;
 }
 
 static std::string get_item_label(const BuildingTypeKey &key, int item_idx)
@@ -387,6 +388,7 @@ static void show_global_settings_dialog()
     lua_newtable(L);
     int ctable = lua_gettop(L);
     Lua::SetField(L, quickfort_mode, ctable, "quickfort_mode");
+    Lua::SetField(L, all_enabled, ctable, "all_enabled");
 
     for (auto & setting : planner.getGlobalSettings())
     {
@@ -629,7 +631,8 @@ struct buildingplan_place_hook : public df::viewscreen_dwarfmodest
             show_help = true;
         }
 
-        if (input->count(interface_key::CUSTOM_SHIFT_P))
+        if (!quickfort_mode && !all_enabled
+            && input->count(interface_key::CUSTOM_SHIFT_P))
         {
             planmode_enabled[key] = !planmode_enabled[key];
             if (!is_planmode_enabled(key))
@@ -765,8 +768,16 @@ struct buildingplan_place_hook : public df::viewscreen_dwarfmodest
             OutputString(COLOR_WHITE, x, y, "Use Shift-Keys here", true, left_margin);
         }
 
-        OutputToggleString(x, y, "Planning Mode", interface_key::CUSTOM_SHIFT_P,
-            planmode_enabled[key], true, left_margin, COLOR_WHITE, COLOR_LIGHTRED);
+        OutputHotkeyString(x, y, "Planning Mode", interface_key::CUSTOM_SHIFT_P);
+        OutputString(COLOR_WHITE, x, y, ": ");
+        if (quickfort_mode)
+            OutputString(COLOR_YELLOW, x, y, "Quickfort", true, left_margin);
+        else if (all_enabled)
+            OutputString(COLOR_YELLOW, x, y, "All", true, left_margin);
+        else if (planmode_enabled[key])
+            OutputString(COLOR_GREEN, x, y, "On", true, left_margin);
+        else
+            OutputString(COLOR_GREY, x, y, "Off", true, left_margin);
         OutputHotkeyString(x, y, "Global Settings", interface_key::CUSTOM_SHIFT_G,
             true, left_margin, COLOR_WHITE, COLOR_LIGHTRED);
 
@@ -963,6 +974,7 @@ static command_result buildingplan_cmd(color_ostream &out, vector <string> & par
             // display current settings
             out.print("active settings:\n");
 
+            out.print("  all_enabled = %s\n", all_enabled ? "true" : "false");
             for (auto & setting : planner.getGlobalSettings())
             {
                 out.print("  %s = %s\n", setting.first.c_str(),
@@ -1108,6 +1120,12 @@ static bool setSetting(std::string name, bool value) {
         quickfort_mode = value;
         return true;
     }
+    if (name == "all_enabled")
+    {
+        debug("setting all_enabled %d -> %d", all_enabled, value);
+        all_enabled = value;
+        return;
+    }
     return planner.setGlobalSetting(name, value);
 }
 
diff --git a/plugins/lua/buildingplan.lua b/plugins/lua/buildingplan.lua
index f640969b8..c0e69168a 100644
--- a/plugins/lua/buildingplan.lua
+++ b/plugins/lua/buildingplan.lua
@@ -229,7 +229,16 @@ end
     setting is not needed for DFHack quickfort.
 --]]
 function GlobalSettings:init()
+
     self.subviews.label:setText{
+        self:make_setting_label_token('Enable all', 'CUSTOM_E', 'all_enabled', 12),
+            self:make_setting_value_token('all_enabled'), '\n',
+        '  Enables buildingplan for all building types. Use this to avoid having\n',
+        '  to manually enable buildingplan for each building type that you want\n',
+        '  to plan. Note that DFHack quickfort will use buildingplan to manage\n',
+        '  buildings regardless of whether buildingplan is "enabled" for the\n',
+        '  building type.\n',
+        '\n',
         'Allowed types for generic, fire-safe, and magma-safe building material:\n',
         self:make_setting_label_token('Blocks', 'CUSTOM_B', 'blocks', 10),
             self:make_setting_value_token('blocks'), '\n',

From 96dfea70c398384db412c61f26c236488733fd7c Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Sun, 24 Jan 2021 09:12:49 -0800
Subject: [PATCH 23/31] update changelog

---
 docs/changelog.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 758a54fb5..b65be46c4 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -39,6 +39,8 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 
 ## Misc Improvements
 - `buildingplan`: set global settings from the ``DFHack#`` prompt: e.g.  ``buildingplan set boulders false``
+- `buildingplan`: add 'enable all' option for buildingplan (so you don't have to enable all building types individually). this setting is not persisted (just like quickfort_mode is not persisted), but it can be set from onMapLoad.init
+- `buildingplan`: modified ``Planning Mode`` status in the UI to show whether we're in quickfort mode, enable all mode, or whether just the building type is enabled.
 
 # 0.47.04-r4
 

From 10634b8ae5e7b1f5fdc77252ae24d360f185e9ff Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Sun, 24 Jan 2021 09:19:01 -0800
Subject: [PATCH 24/31] fix merge error

---
 plugins/buildingplan.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/buildingplan.cpp b/plugins/buildingplan.cpp
index afdfa80d0..6e18219f4 100644
--- a/plugins/buildingplan.cpp
+++ b/plugins/buildingplan.cpp
@@ -1124,7 +1124,7 @@ static bool setSetting(std::string name, bool value) {
     {
         debug("setting all_enabled %d -> %d", all_enabled, value);
         all_enabled = value;
-        return;
+        return true;
     }
     return planner.setGlobalSetting(name, value);
 }

From ff69665815b96d85d3886479a63c61cb33d1b59a Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Wed, 27 Jan 2021 08:25:21 -0800
Subject: [PATCH 25/31] keep stockflow from stealing input during rename

---
 docs/changelog.txt    | 1 +
 plugins/stockflow.cpp | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index c947797f7..a0ecacb18 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -37,6 +37,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 - `embark-assistant`: fixed order of factors when calculating min temperature
 - `embark-assistant`: improved performance of surveying
 - `quickfort`: creating zones no longer causes eventual crashes
+- `stockflow`: fixed 'j' character not being allowed in stockpile names
 
 ## Misc Improvements
 - `buildingplan`: set global settings from the ``DFHack#`` prompt: e.g.  ``buildingplan set boulders false``
diff --git a/plugins/stockflow.cpp b/plugins/stockflow.cpp
index 06635d8c0..22e4abdf9 100644
--- a/plugins/stockflow.cpp
+++ b/plugins/stockflow.cpp
@@ -266,6 +266,9 @@ struct stockflow_hook : public df::viewscreen_dwarfmodest {
     typedef df::viewscreen_dwarfmodest interpose_base;
 
     bool handleInput(set<df::interface_key> *input) {
+        if (Gui::inRenameBuilding())
+            return false;
+
         building_stockpilest *sp = get_selected_stockpile();
         if (!sp)
             return false;

From de6f9183fd0ea2706f7dffbb3c0c2bac029c0a44 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 29 Jan 2021 00:02:42 -0500
Subject: [PATCH 26/31] Fix changelog, replace a couple qerror calls with error

Ref #1746
---
 docs/changelog.txt    | 2 +-
 library/lua/utils.lua | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 771be6ac3..64c57495d 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -45,7 +45,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 - `quickfort`: Dreamfort blueprint set improvements: add a streamlined checklist for all required dreamfort commands and give names to stockpiles, levers, bridges, and zones
 
 ## Lua
-- ``processArgs2()`` added to utils.lua, providing a callback interface for parameter parsing and getopt-like flexibility for parameter ordering and combination (see docs in ``library/lua/utils.lua`` and ``library/lua/3rdparty/alt_getopt.lua`` for details).
+- ``processArgsGetopt()`` added to utils.lua, providing a callback interface for parameter parsing and getopt-like flexibility for parameter ordering and combination (see docs in ``library/lua/utils.lua`` and ``library/lua/3rdparty/alt_getopt.lua`` for details).
 
 # 0.47.04-r4
 
diff --git a/library/lua/utils.lua b/library/lua/utils.lua
index c652e54d4..c85560265 100644
--- a/library/lua/utils.lua
+++ b/library/lua/utils.lua
@@ -647,10 +647,10 @@ function processArgsGetopt(args, optionActions)
     for _,optionAction in ipairs(optionActions) do
         local sh_opt,long_opt = optionAction[1], optionAction[2]
         if not sh_opt or type(sh_opt) ~= 'string' or #sh_opt ~= 1 then
-            qerror('optionAction missing option letter at index 1')
+            error('optionAction missing option letter at index 1')
         end
         if not optionAction.handler then
-            qerror(string.format('handler missing for option "%s"', sh_opt))
+            error(string.format('handler missing for option "%s"', sh_opt))
         end
         sh_opts = sh_opts .. sh_opt
         if optionAction.hasArg then sh_opts = sh_opts .. ':' end

From 810d2858bba3a4df30a0053634254a6bd9759575 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 29 Jan 2021 00:02:57 -0500
Subject: [PATCH 27/31] Update scripts

dfhack/scripts#238
---
 scripts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts b/scripts
index ebe6f5d59..dd4253ec7 160000
--- a/scripts
+++ b/scripts
@@ -1 +1 @@
-Subproject commit ebe6f5d599bfafb614a791b637413e976c8870a5
+Subproject commit dd4253ec70d5a333304df2d338a8e8b9d6c047a3

From 4fbd763c1a2b1a0579d68d125a6d7ae798dd03f3 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 29 Jan 2021 00:28:32 -0500
Subject: [PATCH 28/31] Also cancel 'k' search when hotkeys (F1 - Shift-F8) are
 pressed

---
 plugins/search.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/plugins/search.cpp b/plugins/search.cpp
index 7b791dc7f..9ecae6460 100644
--- a/plugins/search.cpp
+++ b/plugins/search.cpp
@@ -1956,7 +1956,9 @@ public:
             end_entry_mode();
             return false;
         }
-        if (cursor_key_pressed(input, in_entry_mode()))
+        bool hotkey_pressed =
+            input->lower_bound(interface_key::D_HOTKEY1) != input->upper_bound(interface_key::D_HOTKEY16);
+        if (cursor_key_pressed(input, in_entry_mode()) || hotkey_pressed)
         {
             end_entry_mode();
             clear_search();

From 7f62c12a83e4ba09c0a1bd50f9ef02d7a26aed6e Mon Sep 17 00:00:00 2001
From: myk002 <myk002@yahoo.com>
Date: Thu, 28 Jan 2021 21:50:43 -0800
Subject: [PATCH 29/31] document all_enabled setting

---
 docs/Plugins.rst | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/docs/Plugins.rst b/docs/Plugins.rst
index 544a40844..54bb5213e 100644
--- a/docs/Plugins.rst
+++ b/docs/Plugins.rst
@@ -765,29 +765,32 @@ and displayed with::
 
 The available settings are:
 
-+----------------+---------+---------------------------------------+
-| Setting        | Default | Description                           |
-+================+=========+=======================================+
-| blocks         | true    | Allow blocks, boulders, logs, or bars |
-+----------------+---------+ to be matched for generic "building   |
-| boulders       | true    | material" items                       |
-+----------------+---------+                                       |
-| logs           | true    |                                       |
-+----------------+---------+                                       |
-| bars           | false   |                                       |
-+----------------+---------+---------------------------------------+
-| quickfort_mode | false   | Enable compatibility mode for the     |
-|                |         | legacy Python Quickfort (not required |
-|                |         | for DFHack quickfort)                 |
-+----------------+---------+---------------------------------------+
++----------------+---------+-----------+---------------------------------------+
+| Setting        | Default | Persisted | Description                           |
++================+=========+===========+=======================================+
+| all_enabled    | false   | no        | Enable planning mode for all building |
+|                |         |           | types.                                |
++----------------+---------+-----------+---------------------------------------+
+| blocks         | true    | yes       | Allow blocks, boulders, logs, or bars |
++----------------+---------+           | to be matched for generic "building   |
+| boulders       | true    |           | material" items                       |
++----------------+---------+           |                                       |
+| logs           | true    |           |                                       |
++----------------+---------+           |                                       |
+| bars           | false   |           |                                       |
++----------------+---------+-----------+---------------------------------------+
+| quickfort_mode | false   | no        | Enable compatibility mode for the     |
+|                |         |           | legacy Python Quickfort (not required |
+|                |         |           | for DFHack quickfort)                 |
++----------------+---------+-----------+---------------------------------------+
 
 For example, to ensure you only use blocks when a "building material" item is required, you
 could add this to your ``onMapLoad.init`` file::
 
     on-new-fortress buildingplan set boulders false; buildingplan set logs false
 
-You only need to set the settings for new fortresses since your current filter settings
-are saved with your game.
+Persisted settings (i.e. ``blocks``, ``boulders``, ``logs``, and ``bars``) are saved with
+your game, so you only need to set them to the values you want once.
 
 .. _confirm:
 

From c5ab6e6ed495b59ffc27318251364c60cf30a16f Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 29 Jan 2021 01:00:37 -0500
Subject: [PATCH 30/31] Update scripts

dfhack/scripts#246
---
 scripts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts b/scripts
index dd4253ec7..e20fa9f5c 160000
--- a/scripts
+++ b/scripts
@@ -1 +1 @@
-Subproject commit dd4253ec70d5a333304df2d338a8e8b9d6c047a3
+Subproject commit e20fa9f5ca7707377477b3d9c032ec1f3a2e6414

From 6db851a2ad74ea8fe839d1cf3d0a978d709b7593 Mon Sep 17 00:00:00 2001
From: lethosor <lethosor@gmail.com>
Date: Fri, 29 Jan 2021 20:32:33 -0500
Subject: [PATCH 31/31] Update changelog (#1717)

---
 docs/changelog.txt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 03698941b..294818c74 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -43,13 +43,15 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 - `buildingplan`: set global settings from the ``DFHack#`` prompt: e.g.  ``buildingplan set boulders false``
 - `buildingplan`: add 'enable all' option for buildingplan (so you don't have to enable all building types individually). this setting is not persisted (just like quickfort_mode is not persisted), but it can be set from onMapLoad.init
 - `buildingplan`: modified ``Planning Mode`` status in the UI to show whether we're in quickfort mode, enable all mode, or whether just the building type is enabled.
-
-## Misc Improvements
 - `quickfort`: Dreamfort blueprint set improvements: add a streamlined checklist for all required dreamfort commands and give names to stockpiles, levers, bridges, and zones
 
 ## Lua
+- ``dfhack.run_command()``: changed to interface directly with the console when possible, which allows interactive commands and commands that detect the console encoding to work properly
 - ``processArgsGetopt()`` added to utils.lua, providing a callback interface for parameter parsing and getopt-like flexibility for parameter ordering and combination (see docs in ``library/lua/utils.lua`` and ``library/lua/3rdparty/alt_getopt.lua`` for details).
 
+## Documentation
+- Added documentation for Lua's ``dfhack.run_command()`` and variants
+
 # 0.47.04-r4
 
 ## Fixes