Merge remote-tracking branch 'DFHack/develop' into remote_reader

CMakeLists.txt

@@ -89,6 +89,10 @@ if(MSVC)
     # a smaller type, and most of the time this is just conversion from 64 to 32 bits
     # for things like vector sizes, which are never that big anyway.
     add_definitions("/wd4267")
+
+    # MSVC panics if an object file contains more than 65,279 sections. this
+    # happens quite frequently with code that uses templates, such as vectors.
+    add_definitions("/bigobj")
 endif()
 
 # Automatically detect architecture based on Visual Studio generator

docs/Compile.rst

@@ -2,10 +2,16 @@
 Compiling DFHack
 ################
 
-You don't need to compile DFHack unless you're developing plugins or working on the core.
+DFHack builds are available for all supported platforms; see `installing` for
+installation instructions. If you are a DFHack end-user, modder, or plan on
+writing scripts (not plugins), it is generally recommended (and easier) to use
+these builds instead of compiling DFHack from source.
 
-For users, modders, and authors of scripts it's better to download
-and `install the latest release instead <installing>`.
+However, if you are looking to develop plugins, work on the DFHack core, make
+complex changes to DF-structures, or anything else that requires compiling
+DFHack from source, this document will walk you through the build process. Note
+that some steps may be unconventional compared to other projects, so be sure to
+pay close attention if this is your first time compiling DFHack.
 
 .. contents::
    :depth: 2
@@ -14,35 +20,60 @@ and `install the latest release instead <installing>`.
 
 How to get the code
 ===================
-DFHack doesn't have any kind of system of code snapshots in place, so you will have to
-get code from the GitHub repository using Git.  How to get Git is described under
-the instructions for each platform.
-
-To get the latest release code (master branch)::
+DFHack uses Git for source control; instructions for installing Git can be found
+in the platform-specific sections below. The code is hosted on
+`GitHub <https://github.com/DFHack/dfhack>`_, and can be downloaded with::
 
     git clone --recursive https://github.com/DFHack/dfhack
     cd dfhack
 
-If your version of Git does not support the ``--recursive`` flag, you will need to omit it and run
-``git submodule update --init`` after entering the dfhack directory.
+If your version of Git does not support the ``--recursive`` flag, you will need
+to omit it and run ``git submodule update --init`` after entering the dfhack
+directory.
 
-To get the latest development code (develop branch), clone as above and then::
+This will check out the code on the default branch of the GitHub repo, currently
+``develop``, which may be unstable. If you want code for the latest stable
+release, you can check out the ``master`` branch instead::
 
-  git checkout develop
+  git checkout master
   git submodule update
 
-Generally, you should only need to clone DFHack once.
-
-**Important note regarding submodule update after pulling or changing branches**:
-
-You must run ``git submodule update`` every time you change branches, such as
-when switching between the master and develop branches or vice versa. You also
-must run it after pulling any changes to submodules from the DFHack repo. If a
-submodule only exists on the newer branch, or if a commit you just pulled
-contains a new submodule, you need to run ``git submodule update --init``.
-Failure to do this may result in a variety of errors, including ``fatal: <path>
-does not exist`` when using Git, errors when building DFHack, and ``not a known
-DF version`` when starting DF.
+In general, a single DFHack clone is suitable for development - most Git
+operations such as switching branches can be done on an existing clone. If you
+find yourself cloning DFHack frequently as part of your development process, or
+getting stuck on anything else Git-related, feel free to reach out to us for
+assistance.
+
+.. admonition:: A note on submodules
+
+  DFHack uses submodules extensively to manage its subprojects (including the
+  ``scripts`` folder and DF-structures in ``library/xml``). Failing to keep
+  submodules in sync when switching between branches can result in build errors
+  or scripts that don't work. In general, you should always update submodules
+  whenever you switch between branches in the main DFHack repo with
+  ``git submodule update``. (If you are working on bleeding-edge DFHack and
+  have checked out the master branch of some submodules, running ``git pull``
+  in those submodules is also an option.)
+
+  Rarely, we add or remove submodules. If there are any changes to the existence
+  of submodules when you switch between branches, you should run
+  ``git submodule update --init`` instead (adding ``--init`` to the above
+  command).
+
+  Some common errors that can arise when failing to update submodules include:
+
+  * ``fatal: <some path> does not exist`` when performing Git operations
+  * Build errors, particularly referring to structures in the ``df::`` namespace
+    or the ``library/include/df`` folder
+  * ``Not a known DF version`` when starting DF
+  * ``Run 'git submodule update --init'`` when running CMake
+
+  Submodules are a particularly confusing feature of Git. The
+  `Git Book <https://git-scm.com/book/en/v2/Git-Tools-Submodules>`_ has a
+  thorough explanation of them (as well as of many other aspects of Git) and
+  is a recommended resource if you run into any issues. Other DFHack developers
+  are also able to help with any submodule-related (or Git-related) issues
+  you may encounter.
 
 **More notes**:
 
@@ -51,18 +82,21 @@ DF version`` when starting DF.
 
 Contributing to DFHack
 ======================
-If you want to get involved with the development, create an account on
-GitHub, make a clone there and then use that as your remote repository instead.
-
-We'd love that; join us on IRC_ (#dfhack channel on freenode) for discussion,
-and whenever you need help.
+To contribute to DFHack on GitHub, you will need a GitHub account. Only some
+DFHack developers can push directly to the DFHack repositories; we recommend
+making a fork of whatever repos you are interested in contributing to, making
+changes there, and submitting pull requests.  `GitHub's pull request tutorial
+<https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/proposing-changes-to-your-work-with-pull-requests>`_
+is a good resource for getting started with pull requests (some things to note:
+our work mainly happens on the ``develop`` branch, and you will need to use
+your own fork, assuming that you don't have write access to the DFHack repos).
 
-.. _IRC: https://webchat.freenode.net/?channels=dfhack
+Most development-related discussion happens on IRC or in individual GitHub
+issues and pull requests, but there are also other ways to reach out - see
+`support` for details.
 
-(Note: for submodule issues, please see the above instructions first!)
-
-For lots more details on contributing to DFHack, including pull requests, code format,
-and more, please see `contributing-code`.
+For more details on contributing to DFHack, including pull requests, code
+format, and more, please see `contributing-code`.
 
 
 Build settings
@@ -78,7 +112,8 @@ Generator
 The ``Ninja`` CMake build generator is the prefered build method on Linux and
 macOS, instead of ``Unix Makefiles``, which is the default. You can select Ninja
 by passing ``-G Ninja`` to CMake. Incremental builds using Unix Makefiles can be
-much slower than Ninja builds.
+much slower than Ninja builds. Note that you will probably need to install
+Ninja; see the platform-specific sections for details.
 
 ::
 
@@ -133,6 +168,9 @@ your build folder or by running ``ccmake`` (or another CMake GUI). Most
 DFHack-specific settings begin with ``BUILD_`` and control which parts of DFHack
 are built.
 
+
+.. _compile-linux:
+
 Linux
 =====
 On Linux, DFHack acts as a library that shadows parts of the SDL API using LD_PRELOAD.
@@ -144,13 +182,18 @@ DFHack is meant to be installed into an existing DF folder, so get one ready.
 We assume that any Linux platform will have ``git`` available (though it may
 need to be installed with your package manager.)
 
-To build DFHack you need GCC version 4.8 or later. GCC 4.8 is easiest to work
-with due to avoiding libstdc++ issues (see below), but any version from 4.8
-onwards (including 5.x) will work.
+To build DFHack, you need GCC 4.8 or newer. GCC 4.8 has the benefit of avoiding
+`libstdc++ compatibility issues <linux-incompatible-libstdcxx>`, but can be hard
+to obtain on modern distributions, and working around these issues is done
+automatically by the ``dfhack`` launcher script. As long as your system-provided
+GCC is new enough, it should work. Note that extremely new GCC versions may not
+have been used to build DFHack yet, so if you run into issues with these, please
+let us know (e.g. by opening a GitHub issue).
 
 Before you can build anything, you'll also need ``cmake``. It is advisable to
 also get ``ccmake`` on distributions that split the cmake package into multiple
-parts.
+parts. As mentioned above, ``ninja`` is recommended (many distributions call
+this package ``ninja-build``).
 
 You will need pthread; most systems should have this already. Note that older
 CMake versions may have trouble detecting pthread, so if you run into
@@ -226,37 +269,42 @@ This will show a curses-based interface that lets you set all of the
 extra options. You can also use a cmake-friendly IDE like KDevelop 4
 or the cmake-gui program.
 
+.. _linux-incompatible-libstdcxx:
+
 Incompatible libstdc++
 ~~~~~~~~~~~~~~~~~~~~~~
-When compiling dfhack yourself, it builds against your system libstdc++. When
-Dwarf Fortress runs, it uses a libstdc++ shipped with the binary, which comes
-from GCC 4.8 and is incompatible with code compiled with newer GCC versions. If
-you compile DFHack with a GCC version newer than 4.8, you will see an error
-message such as::
+When compiling DFHack yourself, it builds against your system libstdc++. When
+Dwarf Fortress runs, it uses a libstdc++ shipped in the ``libs`` folder, which
+comes from GCC 4.8 and is incompatible with code compiled with newer GCC
+versions. As of DFHack 0.42.05-alpha1, the ``dfhack`` launcher script attempts
+to fix this by automatically removing the DF-provided libstdc++ on startup.
+In rare cases, this may fail and cause errors such as::
 
    ./libs/Dwarf_Fortress: /pathToDF/libs/libstdc++.so.6: version
        `GLIBCXX_3.4.18' not found (required by ./hack/libdfhack.so)
 
-To fix this you can compile with GCC 4.8 or remove the libstdc++ shipped with
+The easiest way to fix this is generally removing the libstdc++ shipped with
 DF, which causes DF to use your system libstdc++ instead::
 
     cd /path/to/DF/
     rm libs/libstdc++.so.6
 
-Note that distributing binaries compiled with newer GCC versions requires end-
-users to delete libstdc++ themselves and have a libstdc++ on their system from
-the same GCC version or newer. For this reason, distributing anything compiled
-with GCC versions newer than 4.8 is discouraged. In the future we may start
-bundling a later libstdc++ as part of the DFHack package, so as to enable
-compilation-for-distribution with a GCC newer than 4.8.
-
-Mac OS X
-========
-DFHack functions similarly on OS X and Linux, and the majority of the
-information above regarding the build process (cmake and ninja) applies here
+Note that distributing binaries compiled with newer GCC versions may result in
+the opposite compatibily issue: users with *older* GCC versions may encounter
+similar errors. This is why DFHack distributes both GCC 4.8 and GCC 7 builds. If
+you are planning on distributing binaries to other users, we recommend using an
+older GCC (but still at least 4.8) version if possible.
+
+
+.. _compile-macos:
+
+macOS
+=====
+DFHack functions similarly on macOS and Linux, and the majority of the
+information above regarding the build process (CMake and Ninja) applies here
 as well.
 
-DFHack can officially be built on OS X with GCC 4.8 or 7. Anything newer than 7
+DFHack can officially be built on macOS only with GCC 4.8 or 7. Anything newer than 7
 will require you to perform extra steps to get DFHack to run (see `osx-new-gcc-notes`),
 and your build will likely not be redistributable.
 
@@ -309,7 +357,7 @@ Dependencies and system set-up
     cleaner, quicker, and smarter. For example, installing MacPort's GCC will
     install more than twice as many dependencies as Homebrew's will, and all in
     both 32-bit and 64-bit variants. Homebrew also doesn't require constant use
-    of sudo.
+    of ``sudo``.
 
     Using `Homebrew <http://brew.sh/>`_ (recommended)::
 
@@ -378,9 +426,9 @@ Building
     export CC=gcc-7
     export CXX=g++-7
 
-  etc.
+  (adjust as needed for different GCC installations)
 
-* Build dfhack::
+* Build DFHack::
 
     mkdir build-osx
     cd build-osx
@@ -390,6 +438,7 @@ Building
   <path to DF> should be a path to a copy of Dwarf Fortress, of the appropriate
   version for the DFHack you are building.
 
+
 .. _compile-windows:
 
 Windows
@@ -682,9 +731,9 @@ It may be installed in a directory such as ``~/.local/bin/``, so after pip
 install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``.
 
 
-Mac OS X
---------
-OS X has Python 2.7 installed by default, but it does not have the pip package manager.
+macOS
+-----
+macOS has Python 2.7 installed by default, but it does not have the pip package manager.
 
 You can install Homebrew's Python 3, which includes pip, and then install the
 latest Sphinx using pip::
@@ -696,11 +745,11 @@ Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew::
 
   brew install sphinx-doc
 
-This will install Sphinx for OS X's system Python 2.7, without needing pip.
+This will install Sphinx for macOS's system Python 2.7, without needing pip.
 
 Either method works; if you plan to use Python for other purposes, it might best
 to install Homebrew's Python 3 so that you have the latest Python as well as pip.
-If not, just installing sphinx-doc for OS X's system Python 2.7 is fine.
+If not, just installing sphinx-doc for macOS's system Python 2.7 is fine.
 
 
 Windows
@@ -767,7 +816,7 @@ files alphabetically, so all the files you need should be next to each other.
 It is recommended that you create a build folder and run CMake to verify that
 you have downloaded everything at this point, assuming your download machine has
 CMake installed. This involves running a "generate" batch script on Windows, or
-a command starting with ``cmake .. -G Ninja`` on Linux and OS X, following the
+a command starting with ``cmake .. -G Ninja`` on Linux and macOS, following the
 instructions in the sections above. CMake should automatically locate files that
 you placed in ``CMake/downloads``, and use them instead of attempting to
 download them.

docs/Introduction.rst

@@ -87,6 +87,7 @@ the console.
 
 
 .. _troubleshooting:
+.. _support:
 
 Troubleshooting
 ===============
@@ -101,7 +102,7 @@ If  the search function in this documentation isn't enough and
 :wiki:`the DF Wiki <>` hasn't helped, try asking in:
 
 - the `#dfhack IRC channel on freenode <https://webchat.freenode.net/?channels=dfhack>`_
-- the :forums:`Bay12 DFHack thread <139553>`
+- the `Bay12 DFHack thread <https://dfhack.org/bay12>`_
 - the `/r/dwarffortress <https://dwarffortress.reddit.com>`_ questions thread
 - the thread for the mod or Starter Pack you're using (if any)
 

library/LuaTypes.cpp

@@ -1422,6 +1422,28 @@ void struct_identity::build_metatable(lua_State *state)
     SetPtrMethods(state, base+1, base+2);
 }
 
+void other_vectors_identity::build_metatable(lua_State *state)
+{
+    int base = lua_gettop(state);
+    MakeFieldMetatable(state, this, meta_struct_index, meta_struct_newindex);
+
+    EnableMetaField(state, base+2, "_enum");
+
+    LookupInTable(state, index_enum, &DFHACK_TYPEID_TABLE_TOKEN);
+    lua_setfield(state, base+1, "_enum");
+
+    auto keys = &index_enum->getKeys()[-index_enum->getFirstItem()];
+
+    for (int64_t i = 0; i < index_enum->getLastItem(); i++)
+    {
+        lua_getfield(state, base+2, keys[i]);
+        lua_rawseti(state, base+2, int(i));
+    }
+
+    SetStructMethod(state, base+1, base+2, meta_struct_field_reference, "_field");
+    SetPtrMethods(state, base+1, base+2);
+}
+
 void global_identity::build_metatable(lua_State *state)
 {
     int base = lua_gettop(state);

library/include/BitArray.h

@@ -568,4 +568,16 @@ namespace DFHack
             root->next = link;
         }
     };
+
+    template<typename T, typename O, typename I>
+    struct DfOtherVectors
+    {
+        std::vector<I *> & operator[](O other_id)
+        {
+            CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(T) / sizeof(std::vector<I *>));
+
+            auto vectors = reinterpret_cast<std::vector<I *> *>(this);
+            return vectors[other_id];
+        }
+    };
 }

library/include/DataDefs.h

@@ -334,6 +334,23 @@ namespace DFHack
         virtual identity_type type() { return IDTYPE_UNION; }
     };
 
+    class DFHACK_EXPORT other_vectors_identity : public struct_identity {
+        enum_identity *index_enum;
+
+    public:
+        other_vectors_identity(size_t size, TAllocateFn alloc,
+                compound_identity *scope_parent, const char *dfhack_name,
+                struct_identity *parent, const struct_field_info *fields,
+                enum_identity *index_enum) :
+            struct_identity(size, alloc, scope_parent, dfhack_name, parent, fields),
+            index_enum(index_enum)
+        {}
+
+        enum_identity *getIndexEnum() { return index_enum; }
+
+        virtual void build_metatable(lua_State *state);
+    };
+
 #ifdef _MSC_VER
     typedef void *virtual_ptr;
 #else
@@ -465,6 +482,7 @@ namespace df
     using DFHack::global_identity;
     using DFHack::struct_identity;
     using DFHack::union_identity;
+    using DFHack::other_vectors_identity;
     using DFHack::struct_field_info;
     using DFHack::struct_field_info_extra;
     using DFHack::bitfield_item_info;
@@ -474,6 +492,7 @@ namespace df
     using DFHack::BitArray;
     using DFHack::DfArray;
     using DFHack::DfLinkedList;
+    using DFHack::DfOtherVectors;
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor"

library/xml

@@ -1 +1 @@
-Subproject commit 0686cfdfc18e9f606f2c296b617d5b3ab6b83889
+Subproject commit cbeed8b52482ce749fc2ff13a9db9733931e7da0

plugins/ruby/codegen.pl

@@ -240,6 +240,12 @@ sub render_global_class {
     }
 
     my $rbparent = ($parent ? rb_ucase($parent) : 'MemHack::Compound');
+    my $ienum;
+    if (($type->getAttribute('ld:subtype') or '') eq 'df-other-vectors-type')
+    {
+        $rbparent = 'MemHack::OtherVectors';
+        $ienum = rb_ucase($type->getAttribute('index-enum'));
+    }
     push @lines_rb, "class $rbname < $rbparent";
     indent_rb {
         my $sz = sizeof($type);
@@ -249,6 +255,8 @@ sub render_global_class {
 
         push @lines_rb, "rtti_classname :$rtti_name\n" if $rtti_name;
 
+        push @lines_rb, "ienum $ienum\n" if $ienum;
+
         render_struct_fields($type);
 
         my $vms = $type->findnodes('child::virtual-methods')->[0];

plugins/ruby/ruby-autogen-defs.rb

@@ -205,6 +205,22 @@ module DFHack
             end
         end
 
+        class OtherVectors < Compound
+            class << self
+                attr_accessor :_enum
+                def ienum(enum)
+                    @_enum = enum
+                end
+            end
+
+            def [](i)
+                self.send(self.class._enum.sym(i))
+            end
+            def []=(i, v)
+                self.send((self.class._enum.sym(i).to_s + "=").to_sym, v)
+            end
+        end
+
         class Enum
             # number -> symbol
             def self.enum