Merge branch 'master' of https://github.com/angavrilov/dfhack

CMakeLists.txt

@@ -103,6 +103,7 @@ ENDIF()
 
 # use shared libraries for protobuf
 ADD_DEFINITIONS(-DPROTOBUF_USE_DLLS)
+ADD_DEFINITIONS(-DLUA_BUILD_AS_DLL)
 
 if(UNIX)
     add_definitions(-D_LINUX)

depends/lua/CMakeLists.txt

@@ -1,6 +1,9 @@
 PROJECT ( lua C )
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
+# TODO: make this RelWithDebInfo only
+ADD_DEFINITIONS(-DLUA_USE_APICHECK)
+
 IF(WIN32)
     ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE )
 ELSE()
@@ -76,8 +79,13 @@ src/lzio.c
 )
 LIST(APPEND SRC_LIBLUA ${HDR_LIBLUA})
 
-ADD_LIBRARY ( lua STATIC EXCLUDE_FROM_ALL ${SRC_LIBLUA} )
+ADD_LIBRARY ( lua SHARED EXCLUDE_FROM_ALL ${SRC_LIBLUA} )
 TARGET_LINK_LIBRARIES ( lua ${LIBS})
+
+install(TARGETS lua
+        LIBRARY DESTINATION ${DFHACK_LIBRARY_DESTINATION}
+        RUNTIME DESTINATION ${DFHACK_LIBRARY_DESTINATION})
+
 IDE_FOLDER(lua "Depends")
 
 #SET ( SRC_LUA src/lua.c )

depends/lua/include/luaconf.h

@@ -151,11 +151,20 @@
 ** the libraries, you may want to use the following definition (define
 ** LUA_BUILD_AS_DLL to get it).
 */
+#ifdef __cplusplus
+    #define LUA_API_EXTERN extern "C"
+#else
+    #define LUA_API_EXTERN extern
+#endif
 #if defined(LUA_BUILD_AS_DLL)
+    #if defined(_MSC_VER)
         #if defined(LUA_CORE) || defined(LUA_LIB)
-        #define LUA_API __declspec(dllexport)
+            #define LUA_API __declspec(dllexport) LUA_API_EXTERN
+        #else
+            #define LUA_API __declspec(dllimport) LUA_API_EXTERN
+        #endif
     #else
-        #define LUA_API __declspec(dllimport)
+        #define LUA_API  LUA_API_EXTERN __attribute__ ((visibility("default")))
     #endif
 #else
     #ifdef __cplusplus

library/CMakeLists.txt

@@ -55,8 +55,10 @@ SET(MAIN_SOURCES
 Core.cpp
 ColorText.cpp
 DataDefs.cpp
+LuaWrapper.cpp
 DataStatics.cpp
 DataStaticsCtor.cpp
+DataStaticsFields.cpp
 MiscUtils.cpp
 PluginManager.cpp
 TileTypes.cpp
@@ -195,6 +197,14 @@ ADD_CUSTOM_COMMAND(
 
 ADD_CUSTOM_TARGET(generate_headers DEPENDS ${dfapi_SOURCE_DIR}/include/df/static.inc)
 
+IF(UNIX)
+  # Don't produce debug info for generated stubs
+  SET_SOURCE_FILES_PROPERTIES(DataStatics.cpp DataStaticsCtor.cpp DataStaticsFields.cpp
+                              PROPERTIES COMPILE_FLAGS "-g0 -O1")
+ELSE(WIN32)
+ENDIF()
+
+
 # Compilation
 
 ADD_DEFINITIONS(-DBUILD_DFHACK_LIB)
@@ -242,7 +252,7 @@ ENDIF()
 #effectively disables debug builds...
 SET_TARGET_PROPERTIES(dfhack  PROPERTIES DEBUG_POSTFIX "-debug" )
 
-TARGET_LINK_LIBRARIES(dfhack protobuf-lite clsocket ${PROJECT_LIBS})
+TARGET_LINK_LIBRARIES(dfhack protobuf-lite clsocket lua ${PROJECT_LIBS})
 SET_TARGET_PROPERTIES(dfhack PROPERTIES LINK_INTERFACE_LIBRARIES "")
 
 TARGET_LINK_LIBRARIES(dfhack-client protobuf-lite clsocket)

library/DataDefs.cpp

@@ -34,28 +34,186 @@ distribution.
 #include "tinythread.h"
 // must be last due to MS stupidity
 #include "DataDefs.h"
+#include "DataIdentity.h"
 
 #include "MiscUtils.h"
 
 using namespace DFHack;
 
+
+void *type_identity::do_allocate_pod() {
+    void *p = malloc(size);
+    memset(p, 0, size);
+    return p;
+}
+
+void type_identity::do_copy_pod(void *tgt, const void *src) {
+    memmove(tgt, src, size);
+};
+
+void *type_identity::allocate() {
+    if (can_allocate())
+        return do_allocate();
+    else
+        return NULL;
+}
+
+bool type_identity::copy(void *tgt, const void *src) {
+    if (can_allocate() && tgt && src)
+        do_copy(tgt, src);
+    else
+        return false;
+}
+
+void *enum_identity::do_allocate() {
+    void *p = malloc(byte_size());
+    memcpy(p, &first_item_value, std::min(byte_size(), sizeof(int64_t)));
+    return p;
+}
+
 /* The order of global object constructor calls is
  * undefined between compilation units. Therefore,
  * this list has to be plain data, so that it gets
  * initialized by the loader in the initial mmap.
  */
-virtual_identity *virtual_identity::list = NULL;
+compound_identity *compound_identity::list = NULL;
+std::vector<compound_identity*> compound_identity::top_scope;
 
-virtual_identity::virtual_identity(const char *dfhack_name, const char *original_name, virtual_identity *parent)
+compound_identity::compound_identity(size_t size, TAllocateFn alloc,
-    : dfhack_name(dfhack_name), original_name(original_name), parent(parent),
+                                     compound_identity *scope_parent, const char *dfhack_name)
-      prev(NULL), vtable_ptr(NULL), has_children(true)
+    : constructed_identity(size, alloc), scope_parent(scope_parent), dfhack_name(dfhack_name)
 {
-    // Link into the static list. Nothing else can be safely done at this point.
     next = list; list = this;
 }
 
-/* Vtable to identity lookup. */
+void compound_identity::doInit(Core *)
+{
+    if (scope_parent)
+        scope_parent->scope_children.push_back(this);
+    else
+        top_scope.push_back(this);
+}
+
+std::string compound_identity::getFullName()
+{
+    if (scope_parent)
+        return scope_parent->getFullName() + "." + getName();
+    else
+        return getName();
+}
+
 static tthread::mutex *known_mutex = NULL;
+
+void compound_identity::Init(Core *core)
+{
+    if (!known_mutex)
+        known_mutex = new tthread::mutex();
+
+    // This cannot be done in the constructors, because
+    // they are called in an undefined order.
+    for (compound_identity *p = list; p; p = p->next)
+        p->doInit(core);
+
+    //FIXME: ... nuked. the group was empty...
+/*
+    // Read pre-filled vtable ptrs
+    OffsetGroup *ptr_table = core->vinfo->getGroup("vtable");
+    for (virtual_identity *p = list; p; p = p->next) {
+        void * tmp;
+        if (ptr_table->getSafeAddress(p->getName(),tmp))
+            p->vtable_ptr = tmp;
+    }
+    */
+}
+
+bitfield_identity::bitfield_identity(size_t size,
+                                     compound_identity *scope_parent, const char *dfhack_name,
+                                     int num_bits, const bitfield_item_info *bits)
+    : compound_identity(size, NULL, scope_parent, dfhack_name), bits(bits), num_bits(num_bits)
+{
+}
+
+enum_identity::enum_identity(size_t size,
+                             compound_identity *scope_parent, const char *dfhack_name,
+                             type_identity *base_type,
+                             int64_t first_item_value, int64_t last_item_value,
+                             const char *const *keys)
+    : compound_identity(size, NULL, scope_parent, dfhack_name),
+      first_item_value(first_item_value), last_item_value(last_item_value),
+      keys(keys), base_type(base_type)
+{
+}
+
+struct_identity::struct_identity(size_t size, TAllocateFn alloc,
+                                 compound_identity *scope_parent, const char *dfhack_name,
+                                 struct_identity *parent, const struct_field_info *fields)
+    : compound_identity(size, alloc, scope_parent, dfhack_name),
+      parent(parent), has_children(false), fields(fields)
+{
+}
+
+void struct_identity::doInit(Core *core)
+{
+    compound_identity::doInit(core);
+
+    if (parent) {
+        parent->children.push_back(this);
+        parent->has_children = true;
+    }
+}
+
+bool struct_identity::is_subclass(struct_identity *actual)
+{
+    for (; actual; actual = actual->getParent())
+        if (actual == this) return true;
+
+    return false;
+}
+
+std::string pointer_identity::getFullName()
+{
+    return (target ? target->getFullName() : std::string("void")) + "*";
+}
+
+std::string container_identity::getFullName(type_identity *item)
+{
+    return "<" + (item ? item->getFullName() : std::string("void")) + ">";
+}
+
+std::string ptr_container_identity::getFullName(type_identity *item)
+{
+    return "<" + (item ? item->getFullName() : std::string("void")) + "*>";
+}
+
+std::string bit_container_identity::getFullName(type_identity *)
+{
+    return "<bool>";
+}
+
+std::string df::buffer_container_identity::getFullName(type_identity *item)
+{
+    return (item ? item->getFullName() : std::string("void")) +
+           (size > 0 ? stl_sprintf("[%d]", size) : std::string("[]"));
+}
+
+virtual_identity::virtual_identity(size_t size, TAllocateFn alloc,
+                                   const char *dfhack_name, const char *original_name,
+                                   virtual_identity *parent, const struct_field_info *fields)
+    : struct_identity(size, alloc, NULL, dfhack_name, parent, fields), original_name(original_name),
+      vtable_ptr(NULL)
+{
+}
+
+static std::map<std::string, virtual_identity*> name_lookup;
+
+void virtual_identity::doInit(Core *core)
+{
+    struct_identity::doInit(core);
+
+    name_lookup[getOriginalName()] = this;
+}
+
+/* Vtable to identity lookup. */
 std::map<void*, virtual_identity*> virtual_identity::known;
 
 virtual_identity *virtual_identity::get(virtual_ptr instance_ptr)
@@ -78,8 +236,9 @@ virtual_identity *virtual_identity::get(virtual_ptr instance_ptr)
 
     virtual_identity *actual = NULL;
 
-    for (virtual_identity *p = list; p; p = p->next) {
+    auto name_it = name_lookup.find(name);
-        if (strcmp(name.c_str(), p->getOriginalName()) != 0) continue;
+    if (name_it != name_lookup.end()) {
+        virtual_identity *p = name_it->second;
 
         if (p->vtable_ptr && p->vtable_ptr != vtable) {
             std::cerr << "Conflicting vtable ptr for class '" << p->getName()
@@ -103,14 +262,6 @@ virtual_identity *virtual_identity::get(virtual_ptr instance_ptr)
     return NULL;
 }
 
-bool virtual_identity::is_subclass(virtual_identity *actual)
-{
-    for (; actual; actual = actual->parent)
-        if (actual == this) return true;
-
-    return false;
-}
-
 void virtual_identity::adjust_vtable(virtual_ptr obj, virtual_identity *main)
 {
     if (vtable_ptr) {
@@ -135,35 +286,6 @@ virtual_ptr virtual_identity::clone(virtual_ptr obj)
     return copy;
 }
 
-void virtual_identity::Init(Core *core)
-{
-    if (!known_mutex)
-        known_mutex = new tthread::mutex();
-
-    // This cannot be done in the constructors, because
-    // they are called in an undefined order.
-    for (virtual_identity *p = list; p; p = p->next) {
-        p->has_children = false;
-        p->children.clear();
-    }
-    for (virtual_identity *p = list; p; p = p->next) {
-        if (p->parent) {
-            p->parent->children.push_back(p);
-            p->parent->has_children = true;
-        }
-    }
-    //FIXME: ... nuked. the group was empty...
-/*
-    // Read pre-filled vtable ptrs
-    OffsetGroup *ptr_table = core->vinfo->getGroup("vtable");
-    for (virtual_identity *p = list; p; p = p->next) {
-        void * tmp;
-        if (ptr_table->getSafeAddress(p->getName(),tmp))
-            p->vtable_ptr = tmp;
-    }
-    */
-}
-
 bool DFHack::findBitfieldField(unsigned *idx, const std::string &name,
                                unsigned size, const bitfield_item_info *items)
 {

library/DataStatics.cpp

@@ -7,6 +7,8 @@
 #include "df/world_data.h"
 #include "df/ui.h"
 
+#include "DataIdentity.h"
+
 namespace {
     template<class T>
     inline T &_toref(T &r) { return r; }
@@ -21,6 +23,8 @@ namespace {
 #define INIT_GLOBAL_FUNCTION_ITEM(type,name) \
     if (global_table_->getAddress(#name,tmp_)) name = (type*)tmp_;
 
+#define TID(type) (&identity_traits< type >::identity)
+
 // Instantiate all the static objects
 #include "df/static.inc"
 #include "df/static.enums.inc"

library/DataStaticsFields.cpp

@@ -0,0 +1,49 @@
+#include "Internal.h"
+#include "DataDefs.h"
+#include "MiscUtils.h"
+#include "VersionInfo.h"
+
+#include "df/world.h"
+#include "df/world_data.h"
+#include "df/ui.h"
+
+#include "DataIdentity.h"
+
+#include <stddef.h>
+
+#pragma GCC diagnostic ignored "-Winvalid-offsetof"
+
+namespace df {
+#define NUMBER_IDENTITY_TRAITS(type) \
+    number_identity<type> identity_traits<type>::identity(#type);
+
+    NUMBER_IDENTITY_TRAITS(char);
+    NUMBER_IDENTITY_TRAITS(int8_t);
+    NUMBER_IDENTITY_TRAITS(uint8_t);
+    NUMBER_IDENTITY_TRAITS(int16_t);
+    NUMBER_IDENTITY_TRAITS(uint16_t);
+    NUMBER_IDENTITY_TRAITS(int32_t);
+    NUMBER_IDENTITY_TRAITS(uint32_t);
+    NUMBER_IDENTITY_TRAITS(int64_t);
+    NUMBER_IDENTITY_TRAITS(uint64_t);
+    NUMBER_IDENTITY_TRAITS(float);
+
+    bool_identity identity_traits<bool>::identity;
+    stl_string_identity identity_traits<std::string>::identity;
+    pointer_identity identity_traits<void*>::identity;
+    stl_ptr_vector_identity identity_traits<std::vector<void*> >::identity;
+    stl_bit_vector_identity identity_traits<std::vector<bool> >::identity;
+
+    buffer_container_identity buffer_container_identity::base_instance;
+
+#undef NUMBER_IDENTITY_TRAITS
+}
+
+#define TID(type) (&identity_traits< type >::identity)
+
+#define FLD(mode, name) struct_field_info::mode, #name, offsetof(CUR_STRUCT, name)
+#define GFLD(mode, name) struct_field_info::mode, #name, (size_t)&df::global::name
+#define FLD_END struct_field_info::END
+
+// Field definitions
+#include "df/static.fields.inc"

library/include/DataDefs.h

@@ -37,6 +37,8 @@ distribution.
 	#undef interface
 #endif
 
+typedef struct lua_State lua_State;
+
 /*
  * Definitions of DFHack namespace structs used by generated headers.
  */
@@ -45,47 +47,252 @@ namespace DFHack
 {
     class virtual_class {};
 
+    enum identity_type {
+        IDTYPE_GLOBAL,
+        IDTYPE_PRIMITIVE,
+        IDTYPE_POINTER,
+        IDTYPE_CONTAINER,
+        IDTYPE_PTR_CONTAINER,
+        IDTYPE_BIT_CONTAINER,
+        IDTYPE_BITFIELD,
+        IDTYPE_ENUM,
+        IDTYPE_STRUCT,
+        IDTYPE_CLASS,
+        IDTYPE_STL_PTR_VECTOR
+    };
+
+    typedef void *(*TAllocateFn)(void*,const void*);
+
+    class DFHACK_EXPORT type_identity {
+        size_t size;
+
+    protected:
+        type_identity(size_t size) : size(size) {};
+
+        void *do_allocate_pod();
+        void do_copy_pod(void *tgt, const void *src);
+
+        virtual bool can_allocate() { return true; }
+        virtual void *do_allocate() { return do_allocate_pod(); }
+        virtual void do_copy(void *tgt, const void *src) { do_copy_pod(tgt, src); }
+
+    public:
+        virtual ~type_identity() {}
+
+        size_t byte_size() { return size; }
+
+        virtual identity_type type() = 0;
+
+        virtual std::string getFullName() = 0;
+
+        // For internal use in the lua wrapper
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr) = 0;
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) = 0;
+        virtual void build_metatable(lua_State *state);
+
+        virtual bool isContainer() { return false; }
+
+        void *allocate();
+        bool copy(void *tgt, const void *src);
+    };
+
+    class DFHACK_EXPORT constructed_identity : public type_identity {
+        TAllocateFn allocator;
+
+    protected:
+        constructed_identity(size_t size, TAllocateFn alloc)
+            : type_identity(size), allocator(alloc) {};
+
+        virtual bool can_allocate() { return (allocator != NULL); }
+        virtual void *do_allocate() { return allocator(NULL,NULL); }
+        virtual void do_copy(void *tgt, const void *src) { allocator(tgt,src); }
+
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr);
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
+    };
+
+    class DFHACK_EXPORT compound_identity : public constructed_identity {
+        static compound_identity *list;
+        compound_identity *next;
+
+        const char *dfhack_name;
+        compound_identity *scope_parent;
+        std::vector<compound_identity*> scope_children;
+        static std::vector<compound_identity*> top_scope;
+
+    protected:
+        compound_identity(size_t size, TAllocateFn alloc,
+                          compound_identity *scope_parent, const char *dfhack_name);
+
+        virtual void doInit(Core *core);
+
+    public:
+        const char *getName() { return dfhack_name; }
+
+        virtual std::string getFullName();
+
+        compound_identity *getScopeParent() { return scope_parent; }
+        const std::vector<compound_identity*> &getScopeChildren() { return scope_children; }
+        static const std::vector<compound_identity*> &getTopScope() { return top_scope; }
+
+        static void Init(Core *core);
+    };
+
+    // Bitfields
+    struct bitfield_item_info {
+        const char *name;
+        int size;
+    };
+
+    class DFHACK_EXPORT bitfield_identity : public compound_identity {
+        const bitfield_item_info *bits;
+        int num_bits;
+
+    protected:
+        virtual bool can_allocate() { return true; }
+        virtual void *do_allocate() { return do_allocate_pod(); }
+        virtual void do_copy(void *tgt, const void *src) { do_copy_pod(tgt, src); }
+
+    public:
+        bitfield_identity(size_t size,
+                          compound_identity *scope_parent, const char *dfhack_name,
+                          int num_bits, const bitfield_item_info *bits);
+
+        virtual identity_type type() { return IDTYPE_BITFIELD; }
+
+        int getNumBits() { return num_bits; }
+        const bitfield_item_info *getBits() { return bits; }
+
+        virtual void build_metatable(lua_State *state);
+    };
+
+    class DFHACK_EXPORT enum_identity : public compound_identity {
+        const char *const *keys;
+        int64_t first_item_value;
+        int64_t last_item_value;
+
+        type_identity *base_type;
+
+    protected:
+        virtual bool can_allocate() { return true; }
+        virtual void *do_allocate();
+        virtual void do_copy(void *tgt, const void *src) { do_copy_pod(tgt, src); }
+
+    public:
+        enum_identity(size_t size,
+                      compound_identity *scope_parent, const char *dfhack_name,
+                      type_identity *base_type,
+                      int64_t first_item_value, int64_t last_item_value,
+                      const char *const *keys);
+
+        virtual identity_type type() { return IDTYPE_ENUM; }
+
+        int64_t getFirstItem() { return first_item_value; }
+        int64_t getLastItem() { return last_item_value; }
+        int getCount() { return int(last_item_value-first_item_value+1); }
+        const char *const *getKeys() { return keys; }
+
+        type_identity *getBaseType() { return base_type; }
+
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr);
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
+    };
+
+    struct struct_field_info {
+        enum Mode {
+            END,
+            PRIMITIVE,
+            STATIC_STRING,
+            POINTER,
+            STATIC_ARRAY,
+            SUBSTRUCT,
+            CONTAINER,
+            STL_VECTOR_PTR
+        };
+        Mode mode;
+        const char *name;
+        size_t offset;
+        type_identity *type;
+        size_t count;
+        enum_identity *eid;
+    };
+
+    class DFHACK_EXPORT struct_identity : public compound_identity {
+        struct_identity *parent;
+        std::vector<struct_identity*> children;
+        bool has_children;
+
+        const struct_field_info *fields;
+
+    protected:
+        virtual void doInit(Core *core);
+
+    public:
+        struct_identity(size_t size, TAllocateFn alloc,
+                        compound_identity *scope_parent, const char *dfhack_name,
+                        struct_identity *parent, const struct_field_info *fields);
+
+        virtual identity_type type() { return IDTYPE_STRUCT; }
+
+        struct_identity *getParent() { return parent; }
+        const std::vector<struct_identity*> &getChildren() { return children; }
+        bool hasChildren() { return has_children; }
+
+        const struct_field_info *getFields() { return fields; }
+
+        bool is_subclass(struct_identity *subtype);
+
+        virtual void build_metatable(lua_State *state);
+    };
+
+    class DFHACK_EXPORT global_identity : public struct_identity {
+    public:
+        global_identity(const struct_field_info *fields)
+            : struct_identity(0,NULL,NULL,"global",NULL,fields) {}
+
+        virtual identity_type type() { return IDTYPE_GLOBAL; }
+
+        virtual void build_metatable(lua_State *state);
+    };
+
 #ifdef _MSC_VER
     typedef void *virtual_ptr;
 #else
     typedef virtual_class *virtual_ptr;
 #endif
 
-    class DFHACK_EXPORT virtual_identity {
+    class DFHACK_EXPORT virtual_identity : public struct_identity {
-        static virtual_identity *list;
         static std::map<void*, virtual_identity*> known;
 
-        virtual_identity *prev, *next;
-        const char *dfhack_name;
         const char *original_name;
-        virtual_identity *parent;
-        std::vector<virtual_identity*> children;
 
         void *vtable_ptr;
-        bool has_children;
 
     protected:
-        virtual_identity(const char *dfhack_name, const char *original_name, virtual_identity *parent);
+        virtual void doInit(Core *core);
 
         static void *get_vtable(virtual_ptr instance_ptr) { return *(void**)instance_ptr; }
 
+        bool can_allocate() { return struct_identity::can_allocate() && (vtable_ptr != NULL); }
+
     public:
-        const char *getName() { return dfhack_name; }
+        virtual_identity(size_t size, TAllocateFn alloc,
-        const char *getOriginalName() { return original_name ? original_name : dfhack_name; }
+                         const char *dfhack_name, const char *original_name,
+                         virtual_identity *parent, const struct_field_info *fields);
+
+        virtual identity_type type() { return IDTYPE_CLASS; }
 
-        virtual_identity *getParent() { return parent; }
+        const char *getOriginalName() { return original_name ? original_name : getName(); }
-        const std::vector<virtual_identity*> &getChildren() { return children; }
 
     public:
         static virtual_identity *get(virtual_ptr instance_ptr);
 
-        bool is_subclass(virtual_identity *subtype);
         bool is_instance(virtual_ptr instance_ptr) {
             if (!instance_ptr) return false;
             if (vtable_ptr) {
                 void *vtable = get_vtable(instance_ptr);
                 if (vtable == vtable_ptr) return true;
-                if (!has_children) return false;
+                if (!hasChildren()) return false;
             }
             return is_subclass(get(instance_ptr));
         }
@@ -97,16 +304,11 @@ namespace DFHack
         }
 
     public:
-        bool can_instantiate() { return (vtable_ptr != NULL); }
+        bool can_instantiate() { return can_allocate(); }
-        virtual_ptr instantiate() { return can_instantiate() ? do_instantiate() : NULL; }
+        virtual_ptr instantiate() { return can_instantiate() ? (virtual_ptr)do_allocate() : NULL; }
         static virtual_ptr clone(virtual_ptr obj);
 
-    protected:
-        virtual virtual_ptr do_instantiate() = 0;
-        virtual void do_copy(virtual_ptr tgt, virtual_ptr src) = 0;
     public:
-        static void Init(Core *core);
-
         // Strictly for use in virtual class constructors
         void adjust_vtable(virtual_ptr obj, virtual_identity *main);
     };
@@ -127,6 +329,23 @@ namespace DFHack
 
     void InitDataDefGlobals(Core *core);
 
+    // LUA wrapper
+
+    /**
+     * Make DF objects available to the given interpreter.
+     */
+    DFHACK_EXPORT void AttachDFGlobals(lua_State *state);
+
+    /**
+     * Push the pointer onto the stack as a wrapped DF object of the given type.
+     */
+    DFHACK_EXPORT void PushDFObject(lua_State *state, type_identity *type, void *ptr);
+
+    /**
+     * Check that the value is a wrapped DF object of the given type, and if so return the pointer.
+     */
+    DFHACK_EXPORT void *GetDFObject(lua_State *state, type_identity *type, int val_index);
+
     template<class T>
     T *ifnull(T *a, T *b) { return a ? a : b; }
 
@@ -135,12 +354,6 @@ namespace DFHack
         size_t size;
         const T *items;
     };
-
-    // Bitfields
-    struct bitfield_item_info {
-        const char *name;
-        int size;
-    };
 }
 
 template<class T>
@@ -164,33 +377,37 @@ inline int linear_index(const DFHack::enum_list_attr<const char*> &lst, const st
 
 namespace df
 {
+    using DFHack::type_identity;
+    using DFHack::compound_identity;
     using DFHack::virtual_ptr;
     using DFHack::virtual_identity;
     using DFHack::virtual_class;
+    using DFHack::global_identity;
+    using DFHack::struct_identity;
+    using DFHack::struct_field_info;
     using DFHack::bitfield_item_info;
+    using DFHack::bitfield_identity;
+    using DFHack::enum_identity;
     using DFHack::enum_list_attr;
     using DFHack::BitArray;
     using DFHack::DfArray;
 
     template<class T>
-    struct enum_traits {};
+    void *allocator_fn(void *out, const void *in) {
+        if (out) { *(T*)out = *(const T*)in; return out; }
+        else return new T();
+    }
 
     template<class T>
-    struct bitfield_traits {};
+    struct identity_traits {
+        static compound_identity *get() { return &T::_identity; }
+    };
 
     template<class T>
-    class class_virtual_identity : public virtual_identity {
+    struct enum_traits {};
-    public:
-        class_virtual_identity(const char *dfhack_name, const char *original_name, virtual_identity *parent)
-            : virtual_identity(dfhack_name, original_name, parent) {};
-
-        T *instantiate() { return static_cast<T*>(virtual_identity::instantiate()); }
-        T *clone(T* obj) { return static_cast<T*>(virtual_identity::clone(obj)); }
 
-    protected:
+    template<class T>
-        virtual virtual_ptr do_instantiate() { return new T(); }
+    struct bitfield_traits {};
-        virtual void do_copy(virtual_ptr tgt, virtual_ptr src) { *static_cast<T*>(tgt) = *static_cast<T*>(src); }
-    };
 
     template<class EnumType, class IntType = int32_t>
     struct enum_field {
@@ -427,6 +644,24 @@ namespace DFHack {
         flagarray_to_string<T>(&tmp, val);
         return join_strings(sep, tmp);
     }
+
+    // LUA wrapper
+
+    /**
+     * Push the pointer onto the stack as a wrapped DF object of a specific type.
+     */
+    template<class T>
+    void PushDFObject(lua_State *state, T *ptr) {
+        PushDFObject(state, df::identity_traits<T>::get(), ptr);
+    }
+
+    /**
+     * Check that the value is a wrapped DF object of the correct type, and if so return the pointer.
+     */
+    template<class T>
+    T *GetDFObject(lua_State *state, int val_index) {
+        return GetDFObject(state, df::identity_traits<T>::get(), val_index);
+    }
 }
 
 #define ENUM_ATTR(enum,attr,val) (df::enum_traits<df::enum>::attrs(val).attr)

library/include/DataIdentity.h

@@ -0,0 +1,438 @@
+/*
+https://github.com/peterix/dfhack
+Copyright (c) 2009-2011 Petr Mrázek (peterix@gmail.com)
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any
+damages arising from the use of this software.
+
+Permission is granted to anyone to use this software for any
+purpose, including commercial applications, and to alter it and
+redistribute it freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must
+not claim that you wrote the original software. If you use this
+software in a product, an acknowledgment in the product documentation
+would be appreciated but is not required.
+
+2. Altered source versions must be plainly marked as such, and
+must not be misrepresented as being the original software.
+
+3. This notice may not be removed or altered from any source
+distribution.
+*/
+
+#pragma once
+
+#include <string>
+#include <sstream>
+#include <vector>
+#include <map>
+
+#include "DataDefs.h"
+
+/*
+ * Definitions of DFHack namespace structs used by generated headers.
+ */
+
+namespace DFHack
+{
+    class DFHACK_EXPORT primitive_identity : public type_identity {
+    public:
+        primitive_identity(size_t size) : type_identity(size) {};
+
+        virtual identity_type type() { return IDTYPE_PRIMITIVE; }
+    };
+
+    class DFHACK_EXPORT pointer_identity : public primitive_identity {
+        type_identity *target;
+
+    public:
+        pointer_identity(type_identity *target = NULL)
+            : primitive_identity(sizeof(void*)), target(target) {};
+
+        virtual identity_type type() { return IDTYPE_POINTER; }
+
+        type_identity *getTarget() { return target; }
+
+        std::string getFullName();
+
+        static void lua_read(lua_State *state, int fname_idx, void *ptr, type_identity *target);
+        static void lua_write(lua_State *state, int fname_idx, void *ptr, type_identity *target, int val_index);
+
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr);
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
+    };
+
+    class DFHACK_EXPORT container_identity : public constructed_identity {
+        type_identity *item;
+        enum_identity *ienum;
+
+    public:
+        container_identity(size_t size, TAllocateFn alloc, type_identity *item, enum_identity *ienum = NULL)
+            : constructed_identity(size, alloc), item(item), ienum(ienum) {};
+
+        virtual identity_type type() { return IDTYPE_CONTAINER; }
+
+        std::string getFullName() { return getFullName(item); }
+
+        virtual void build_metatable(lua_State *state);
+        virtual bool isContainer() { return true; }
+
+        type_identity *getItemType() { return item; }
+        type_identity *getIndexEnumType() { return ienum; }
+
+        virtual std::string getFullName(type_identity *item);
+
+        int lua_item_count(lua_State *state, void *ptr);
+
+        virtual void lua_item_reference(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+
+    protected:
+        virtual int item_count(void *ptr) = 0;
+        virtual void *item_pointer(type_identity *item, void *ptr, int idx) = 0;
+    };
+
+    class DFHACK_EXPORT ptr_container_identity : public container_identity {
+    public:
+        ptr_container_identity(size_t size, TAllocateFn alloc,
+                               type_identity *item, enum_identity *ienum = NULL)
+            : container_identity(size, alloc, item, ienum) {};
+
+        virtual identity_type type() { return IDTYPE_PTR_CONTAINER; }
+
+        std::string getFullName(type_identity *item);
+
+        virtual void lua_item_reference(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+    };
+
+    class DFHACK_EXPORT bit_container_identity : public container_identity {
+    public:
+        bit_container_identity(size_t size, TAllocateFn alloc, enum_identity *ienum = NULL)
+            : container_identity(size, alloc, NULL, ienum) {};
+
+        virtual identity_type type() { return IDTYPE_BIT_CONTAINER; }
+
+        std::string getFullName(type_identity *item);
+
+        virtual void lua_item_reference(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+
+    protected:
+        virtual void *item_pointer(type_identity *, void *, int) { return NULL; }
+
+        virtual bool get_item(void *ptr, int idx) = 0;
+        virtual void set_item(void *ptr, int idx, bool val) = 0;
+    };
+}
+
+namespace df
+{
+    using DFHack::primitive_identity;
+    using DFHack::pointer_identity;
+    using DFHack::container_identity;
+    using DFHack::ptr_container_identity;
+    using DFHack::bit_container_identity;
+
+    class number_identity_base : public primitive_identity {
+        const char *name;
+
+    public:
+        number_identity_base(size_t size, const char *name)
+            : primitive_identity(size), name(name) {};
+
+        std::string getFullName() { return name; }
+
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr);
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
+
+    protected:
+        virtual double read(void *ptr) = 0;
+        virtual void write(void *ptr, double val) = 0;
+    };
+
+    template<class T>
+    class number_identity : public number_identity_base {
+    public:
+        number_identity(const char *name) : number_identity_base(sizeof(T), name) {}
+    protected:
+        virtual double read(void *ptr) { return double(*(T*)ptr); }
+        virtual void write(void *ptr, double val) { *(T*)ptr = T(val); }
+    };
+
+    class bool_identity : public primitive_identity {
+    public:
+        bool_identity() : primitive_identity(sizeof(bool)) {};
+
+        std::string getFullName() { return "bool"; }
+
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr);
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
+    };
+
+    class stl_string_identity : public primitive_identity {
+    public:
+        stl_string_identity() : primitive_identity(sizeof(std::string)) {};
+
+        std::string getFullName() { return "string"; }
+
+        virtual void lua_read(lua_State *state, int fname_idx, void *ptr);
+        virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
+    };
+
+    class stl_ptr_vector_identity : public ptr_container_identity {
+    public:
+        /*
+         * This class assumes that std::vector<T*> is equivalent
+         * in layout and behavior to std::vector<void*> for any T.
+         */
+
+        stl_ptr_vector_identity(type_identity *item = NULL, enum_identity *ienum = NULL)
+            : ptr_container_identity(sizeof(std::vector<void*>),allocator_fn<std::vector<void*> >,item, ienum)
+        {};
+
+        std::string getFullName(type_identity *item) {
+            return "vector" + ptr_container_identity::getFullName(item);
+        }
+
+        virtual DFHack::identity_type type() { return DFHack::IDTYPE_STL_PTR_VECTOR; }
+
+    protected:
+        virtual int item_count(void *ptr) {
+            return ((std::vector<void*>*)ptr)->size();
+        };
+        virtual void *item_pointer(type_identity *, void *ptr, int idx) {
+            return &(*(std::vector<void*>*)ptr)[idx];
+        }
+    };
+
+    class buffer_container_identity : public container_identity {
+        int size;
+
+    public:
+        buffer_container_identity()
+            : container_identity(0, NULL, NULL, NULL), size(0)
+        {}
+
+        buffer_container_identity(int size, type_identity *item, enum_identity *ienum = NULL)
+            : container_identity(item->byte_size()*size, NULL, item, ienum), size(size)
+        {}
+
+        std::string getFullName(type_identity *item);
+
+        static buffer_container_identity base_instance;
+
+    protected:
+        virtual int item_count(void *ptr) { return size; }
+        virtual void *item_pointer(type_identity *item, void *ptr, int idx) {
+            return ((uint8_t*)ptr) + idx * item->byte_size();
+        }
+    };
+
+    template<class T>
+    class stl_container_identity : public container_identity {
+        const char *name;
+
+    public:
+        stl_container_identity(const char *name, type_identity *item, enum_identity *ienum = NULL)
+            : container_identity(sizeof(T), &allocator_fn<T>, item, ienum), name(name)
+        {}
+
+        std::string getFullName(type_identity *item) {
+            return name + container_identity::getFullName(item);
+        }
+
+    protected:
+        virtual int item_count(void *ptr) { return ((T*)ptr)->size(); }
+        virtual void *item_pointer(type_identity *item, void *ptr, int idx) {
+            return &(*(T*)ptr)[idx];
+        }
+    };
+
+    class bit_array_identity : public bit_container_identity {
+    public:
+        /*
+         * This class assumes that BitArray<T> is equivalent
+         * in layout and behavior to BitArray<int> for any T.
+         */
+
+        typedef BitArray<int> container;
+
+        bit_array_identity(enum_identity *ienum = NULL)
+            : bit_container_identity(sizeof(container), &allocator_fn<container>, ienum)
+        {}
+
+        std::string getFullName(type_identity *item) {
+            return "BitArray<>";
+        }
+
+    protected:
+        virtual int item_count(void *ptr) { return ((container*)ptr)->size * 8; }
+        virtual bool get_item(void *ptr, int idx) {
+            return ((container*)ptr)->is_set(idx);
+        }
+        virtual void set_item(void *ptr, int idx, bool val) {
+            ((container*)ptr)->set(idx, val);
+        }
+    };
+
+    class stl_bit_vector_identity : public bit_container_identity {
+    public:
+        typedef std::vector<bool> container;
+
+        stl_bit_vector_identity(enum_identity *ienum = NULL)
+            : bit_container_identity(sizeof(container), &allocator_fn<container>, ienum)
+        {}
+
+        std::string getFullName(type_identity *item) {
+            return "vector" + bit_container_identity::getFullName(item);
+        }
+
+    protected:
+        virtual int item_count(void *ptr) { return ((container*)ptr)->size(); }
+        virtual bool get_item(void *ptr, int idx) {
+            return (*(container*)ptr)[idx];
+        }
+        virtual void set_item(void *ptr, int idx, bool val) {
+            (*(container*)ptr)[idx] = val;
+        }
+    };
+
+#define NUMBER_IDENTITY_TRAITS(type) \
+    template<> struct identity_traits<type> { \
+        static number_identity<type> identity; \
+        static number_identity_base *get() { return &identity; } \
+    };
+
+    NUMBER_IDENTITY_TRAITS(char);
+    NUMBER_IDENTITY_TRAITS(int8_t);
+    NUMBER_IDENTITY_TRAITS(uint8_t);
+    NUMBER_IDENTITY_TRAITS(int16_t);
+    NUMBER_IDENTITY_TRAITS(uint16_t);
+    NUMBER_IDENTITY_TRAITS(int32_t);
+    NUMBER_IDENTITY_TRAITS(uint32_t);
+    NUMBER_IDENTITY_TRAITS(int64_t);
+    NUMBER_IDENTITY_TRAITS(uint64_t);
+    NUMBER_IDENTITY_TRAITS(float);
+
+    template<> struct identity_traits<bool> {
+        static bool_identity identity;
+        static bool_identity *get() { return &identity; }
+    };
+
+    template<> struct identity_traits<std::string> {
+        static stl_string_identity identity;
+        static stl_string_identity *get() { return &identity; }
+    };
+
+    template<> struct identity_traits<void*> {
+        static pointer_identity identity;
+        static pointer_identity *get() { return &identity; }
+    };
+
+    template<> struct identity_traits<std::vector<void*> > {
+        static stl_ptr_vector_identity identity;
+        static stl_ptr_vector_identity *get() { return &identity; }
+    };
+
+    template<> struct identity_traits<std::vector<bool> > {
+        static stl_bit_vector_identity identity;
+        static stl_bit_vector_identity *get() { return &identity; }
+    };
+
+#undef NUMBER_IDENTITY_TRAITS
+
+    // Container declarations
+
+    template<class Enum, class FT> struct identity_traits<enum_field<Enum,FT> > {
+        static primitive_identity *get();
+    };
+
+    template<class T> struct identity_traits<T *> {
+        static pointer_identity *get();
+    };
+
+    template<class T, int sz> struct identity_traits<T [sz]> {
+        static container_identity *get();
+    };
+
+    template<class T> struct identity_traits<std::vector<T> > {
+        static container_identity *get();
+    };
+
+    template<class T> struct identity_traits<std::vector<T*> > {
+        static stl_ptr_vector_identity *get();
+    };
+
+    template<class T> struct identity_traits<std::deque<T> > {
+        static container_identity *get();
+    };
+
+    template<class T> struct identity_traits<BitArray<T> > {
+        static bit_container_identity *get();
+    };
+
+    template<class T> struct identity_traits<DfArray<T> > {
+        static container_identity *get();
+    };
+
+    // Container definitions
+
+    template<class Enum, class FT>
+    primitive_identity *identity_traits<enum_field<Enum,FT> >::get() {
+        return identity_traits<FT>::get();
+    }
+
+    template<class T>
+    pointer_identity *identity_traits<T *>::get() {
+        static pointer_identity identity(identity_traits<T>::get());
+        return &identity;
+    }
+
+    template<class T, int sz>
+    container_identity *identity_traits<T [sz]>::get() {
+        static buffer_container_identity identity(sz, identity_traits<T>::get());
+        return &identity;
+    }
+
+    template<class T>
+    container_identity *identity_traits<std::vector<T> >::get() {
+        typedef std::vector<T> container;
+        static stl_container_identity<container> identity("vector", identity_traits<T>::get());
+        return &identity;
+    }
+
+    template<class T>
+    stl_ptr_vector_identity *identity_traits<std::vector<T*> >::get() {
+        static stl_ptr_vector_identity identity(identity_traits<T>::get());
+        return &identity;
+    }
+
+    template<class T>
+    container_identity *identity_traits<std::deque<T> >::get() {
+        typedef std::deque<T> container;
+        static stl_container_identity<container> identity("deque", identity_traits<T>::get());
+        return &identity;
+    }
+
+    template<class T>
+    bit_container_identity *identity_traits<BitArray<T> >::get() {
+        static type_identity *eid = identity_traits<T>::get();
+        static enum_identity *reid = eid->type() == DFHack::IDTYPE_ENUM ? (enum_identity*)eid : NULL;
+        static bit_array_identity identity(reid);
+        return &identity;
+    }
+
+    template<class T>
+    container_identity *identity_traits<DfArray<T> >::get() {
+        typedef DfArray<T> container;
+        static stl_container_identity<container> identity("DfArray", identity_traits<T>::get());
+        return &identity;
+    }
+}
+

library/proto/BasicApi.proto

@@ -4,6 +4,10 @@ option optimize_for = LITE_RUNTIME;
 
 import "Basic.proto";
 
+// RPC GetVersion : EmptyMessage -> StringMessage
+// RPC GetDFVersion : EmptyMessage -> StringMessage
+
+// RPC GetWorldInfo : EmptyMessage -> GetWorldInfoOut
 message GetWorldInfoOut {
     enum Mode {
         MODE_DWARF = 1;
@@ -27,6 +31,7 @@ message GetWorldInfoOut {
     repeated int32 companion_histfig_ids = 10;
 };
 
+// RPC ListEnums : EmptyMessage -> ListEnumsOut
 message ListEnumsOut {
     repeated EnumItemName material_flags = 1;
     repeated EnumItemName inorganic_flags = 2;
@@ -46,6 +51,7 @@ message ListEnumsOut {
     repeated EnumItemName profession = 11;
 };
 
+// RPC ListMaterials : ListMaterialsIn -> ListMaterialsOut
 message ListMaterialsIn {
     optional BasicMaterialInfoMask mask = 1;
 
@@ -62,6 +68,7 @@ message ListMaterialsOut {
     repeated BasicMaterialInfo value = 1;
 };
 
+// RPC ListUnits : ListUnitsIn -> ListUnitsOut
 message ListUnitsIn {
     optional BasicUnitInfoMask mask = 1;
 
@@ -81,6 +88,7 @@ message ListUnitsOut {
     repeated BasicUnitInfo value = 1;
 };
 
+// RPC ListSquads : ListSquadsIn -> ListSquadsOut
 message ListSquadsIn {}
 message ListSquadsOut {
     repeated BasicSquadInfo value = 1;

library/proto/CoreProtocol.proto

@@ -62,18 +62,22 @@ message StringListMessage {
     repeated string value = 1;
 }
 
+// RPC BindMethod : CoreBindRequest -> CoreBindReply
 message CoreBindRequest {
     required string method = 1;
     required string input_msg = 2;
     required string output_msg = 3;
     optional string plugin = 4;
 }
-
 message CoreBindReply {
     required int32 assigned_id = 1;
 }
 
+// RPC RunCommand : CoreRunCommandRequest -> EmptyMessage
 message CoreRunCommandRequest {
     required string command = 1;
     repeated string arguments = 2;
 }
+
+// RPC CoreSuspend : EmptyMessage -> IntMessage
+// RPC CoreResume : EmptyMessage -> IntMessage

library/xml

@@ -1 +1 @@
-Subproject commit 08e1f71e89c1af6b3bef940914ed7f3d8fed89b0
+Subproject commit b28296a8c0c2e5aaa522840598738109018f0146

plugins/proto/rename.proto

@@ -2,6 +2,7 @@ package dfproto;
 
 option optimize_for = LITE_RUNTIME;
 
+// RPC RenameSquad : RenameSquadIn -> EmptyMessage
 message RenameSquadIn {
     required int32 squad_id = 1;
 
@@ -9,6 +10,7 @@ message RenameSquadIn {
     optional string alias = 3;
 }
 
+// RPC RenameUnit : RenameUnitIn -> EmptyMessage
 message RenameUnitIn {
     required int32 unit_id = 1;