Support containers in the lua wrapper.

library/DataDefs.cpp

@@ -34,6 +34,7 @@ distribution.
 #include "tinythread.h"
 // must be last due to MS stupidity
 #include "DataDefs.h"
+#include "DataIdentity.h"
 
 #include "MiscUtils.h"
 
@@ -62,6 +63,14 @@ void compound_identity::doInit(Core *)
         top_scope.push_back(this);
 }
 
+std::string compound_identity::getFullName()
+{
+    if (scope_parent)
+        return scope_parent->getFullName() + "." + getName();
+    else
+        return getName();
+}
+
 static tthread::mutex *known_mutex = NULL;
 
 void compound_identity::Init(Core *core)
@@ -130,6 +139,32 @@ bool struct_identity::is_subclass(struct_identity *actual)
     return false;
 }
 
+std::string pointer_identity::getFullName()
+{
+    return (target ? target->getFullName() : std::string("void")) + "*";
+}
+
+std::string container_identity::getFullName(type_identity *item)
+{
+    return "<" + (item ? item->getFullName() : std::string("void")) + ">";
+}
+
+std::string ptr_container_identity::getFullName(type_identity *item)
+{
+    return "<" + (item ? item->getFullName() : std::string("void")) + "*>";
+}
+
+std::string bit_container_identity::getFullName(type_identity *)
+{
+    return "<bool>";
+}
+
+std::string df::buffer_container_identity::getFullName(type_identity *item)
+{
+    return (item ? item->getFullName() : std::string("void")) +
+           (size > 0 ? stl_sprintf("[%d]", size) : std::string("[]"));
+}
+
 virtual_identity::virtual_identity(size_t size, TAllocateFn alloc,
                                    const char *dfhack_name, const char *original_name,
                                    virtual_identity *parent, const struct_field_info *fields)

library/DataStaticsFields.cpp

@@ -15,7 +15,7 @@
 
 namespace df {
 #define NUMBER_IDENTITY_TRAITS(type) \
-    number_identity<type> identity_traits<type>::identity;
+    number_identity<type> identity_traits<type>::identity(#type);
 
     NUMBER_IDENTITY_TRAITS(char);
     NUMBER_IDENTITY_TRAITS(int8_t);
@@ -32,6 +32,9 @@ namespace df {
     stl_string_identity identity_traits<std::string>::identity;
     pointer_identity identity_traits<void*>::identity;
     stl_ptr_vector_identity identity_traits<std::vector<void*> >::identity;
+    stl_bit_vector_identity identity_traits<std::vector<bool> >::identity;
+
+    buffer_container_identity buffer_container_identity::base_instance;
 
 #undef NUMBER_IDENTITY_TRAITS
 }

library/LuaWrapper.cpp

@@ -50,6 +50,7 @@ inline void lua_swap(lua_State *state) { lua_insert(state, -2); }
 
 #define DFHACK_TYPETABLE_NAME "DFHack::DFTypes"
 #define DFHACK_TYPEID_TABLE_NAME "DFHack::DFTypeIds"
+#define DFHACK_ENUM_TABLE_NAME "DFHack::DFEnums"
 #define DFHACK_CHANGEERROR_NAME "DFHack::ChangeError"
 #define DFHACK_COMPARE_NAME "DFHack::ComparePtrs"
 #define DFHACK_TYPE_TOSTRING_NAME "DFHack::TypeToString"
@@ -58,6 +59,10 @@ inline void lua_swap(lua_State *state) { lua_insert(state, -2); }
 #define UPVAL_METATABLE lua_upvalueindex(2)
 #define UPVAL_FIELDTABLE lua_upvalueindex(3)
 
+#define UPVAL_CONTAINER_ID lua_upvalueindex(4)
+#define UPVAL_ITEM_ID lua_upvalueindex(5)
+#define UPVAL_ITEM_COUNT lua_upvalueindex(6)
+
 namespace {
     struct DFRefHeader {
         void *ptr;
@@ -138,10 +143,10 @@ void df::bool_identity::lua_write(lua_State *state, int fname_idx, void *ptr, in
 {
     char *pb = (char*)ptr;
 
-    if (lua_isboolean(state, val_index))
+    if (lua_isboolean(state, val_index) || lua_isnil(state, val_index))
         *pb = lua_toboolean(state, val_index);
     else if (lua_isnumber(state, val_index))
-        *pb = lua_tonumber(state, val_index);
+        *pb = lua_tointeger(state, val_index);
     else
         field_error(state, fname_idx, "boolean or number expected", "write");
 }
@@ -196,6 +201,72 @@ void df::pointer_identity::lua_write(lua_State *state, int fname_idx, void *ptr,
     lua_write(state, fname_idx, ptr, target, val_index);
 }
 
+int container_identity::lua_item_count(lua_State *state, void *ptr)
+{
+    if (lua_isnumber(state, UPVAL_ITEM_COUNT))
+        return lua_tointeger(state, UPVAL_ITEM_COUNT);
+    else
+        return item_count(ptr);
+}
+
+int container_identity::lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx)
+{
+    void *pitem = item_pointer(ptr, idx);
+    auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID);
+    return id->lua_read(state, fname_idx, pitem);
+}
+
+void container_identity::lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index)
+{
+    void *pitem = item_pointer(ptr, idx);
+    auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID);
+    id->lua_write(state, fname_idx, pitem, val_index);
+}
+
+int ptr_container_identity::lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx)
+{
+    void *pitem = item_pointer(ptr, idx);
+    auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID);
+    return df::pointer_identity::lua_read(state, fname_idx, pitem, id);
+}
+
+void ptr_container_identity::lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index)
+{
+    void *pitem = item_pointer(ptr, idx);
+    auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID);
+    df::pointer_identity::lua_write(state, fname_idx, pitem, id, val_index);
+}
+
+int bit_container_identity::lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx)
+{
+    lua_pushboolean(state, get_item(ptr, idx));
+    return 1;
+}
+
+void bit_container_identity::lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index)
+{
+    if (lua_isboolean(state, val_index) || lua_isnil(state, val_index))
+        set_item(ptr, idx, lua_toboolean(state, val_index));
+    else if (lua_isnumber(state, val_index))
+        set_item(ptr, idx, lua_tointeger(state, val_index) != 0);
+    else
+        field_error(state, fname_idx, "boolean or number expected", "write");
+}
+
+int df::buffer_container_identity::lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx)
+{
+    auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID);
+    void *pitem = ((uint8_t*)ptr) + idx * id->byte_size();
+    return id->lua_read(state, fname_idx, pitem);
+}
+
+void df::buffer_container_identity::lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index)
+{
+    auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID);
+    void *pitem = ((uint8_t*)ptr) + idx * id->byte_size();
+    id->lua_write(state, fname_idx, pitem, val_index);
+}
+
 /* */
 
 static int change_error(lua_State *state)
@@ -253,6 +324,14 @@ static bool LookupTypeInfo(lua_State *state, bool in_method)
         return true;
 }
 
+static void LookupInTable(lua_State *state, void *id, const char *tname)
+{
+    lua_getfield(state, LUA_REGISTRYINDEX, tname);
+    lua_pushlightuserdata(state, id);
+    lua_rawget(state, -2);
+    lua_remove(state, -2);
+}
+
 static void SaveInTable(lua_State *state, void *node, const char *tname)
 {
     // stack: [info]
@@ -311,10 +390,7 @@ static void push_object_internal(lua_State *state, type_identity *type, void *pt
     lua_pushlightuserdata(state, type); // () -> type
 
     if (!LookupTypeInfo(state, in_method)) // type -> metatable?
-    {
         BuildTypeMetatable(state, type); // () -> metatable
-        SaveTypeInfo(state, type);
-    }
 
     push_object_ref(state, ptr); // metatable -> userdata
 }
@@ -371,17 +447,22 @@ static int meta_ptr_compare(lua_State *state)
     return 1;
 }
 
-static const struct_field_info *find_field(lua_State *state, int index, const char *mode)
+static void lookup_field(lua_State *state, int index, const char *mode)
 {
     lua_pushvalue(state, index);
-    lua_rawget(state, UPVAL_FIELDTABLE);
+    lua_gettable(state, UPVAL_FIELDTABLE); // uses metatable with enum keys
 
-    if (!lua_islightuserdata(state, -1))
+    if (lua_isnil(state, -1))
         field_error(state, index, "not found", mode);
+}
+
+static void *find_field(lua_State *state, int index, const char *mode)
+{
+    lookup_field(state, index, mode);
 
     void *p = lua_touserdata(state, -1);
     lua_pop(state, 1);
-    return (struct_field_info*)p;
+    return p;
 }
 
 static uint8_t *get_object_addr(lua_State *state, int obj, int field, const char *mode)
@@ -399,6 +480,8 @@ static uint8_t *get_object_addr(lua_State *state, int obj, int field, const char
     return (uint8_t*)ref->ptr;
 }
 
+static void GetAdHocMetatable(lua_State *state, const struct_field_info *field);
+
 static int read_field(lua_State *state, const struct_field_info *field, void *ptr)
 {
     switch (field->mode)
@@ -412,14 +495,21 @@ static int read_field(lua_State *state, const struct_field_info *field, void *pt
 
         case struct_field_info::PRIMITIVE:
         case struct_field_info::SUBSTRUCT:
-        case struct_field_info::CONTAINER:
             return field->type->lua_read(state, 2, ptr);
 
         case struct_field_info::POINTER:
             return df::pointer_identity::lua_read(state, 2, ptr, field->type);
 
+        case struct_field_info::CONTAINER:
+            if (!field->eid || !field->type->isContainer() ||
+                field->eid == ((container_identity*)field->type)->getIndexEnumType())
+                return field->type->lua_read(state, 2, ptr);
+
         case struct_field_info::STATIC_ARRAY:
         case struct_field_info::STL_VECTOR_PTR:
+            GetAdHocMetatable(state, field);
+            push_object_ref(state, ptr);
+            return 1;
 
         case struct_field_info::END:
             return 0;
@@ -490,7 +580,7 @@ static int get_metafield(lua_State *state)
 static int meta_struct_index(lua_State *state)
 {
     uint8_t *ptr = get_object_addr(state, 1, 2, "read");
-    const struct_field_info *field = find_field(state, 2, "read");
+    auto field = (struct_field_info*)find_field(state, 2, "read");
     if (!field)
         return get_metafield(state);
     return read_field(state, field, ptr + field->offset);
@@ -499,14 +589,88 @@ static int meta_struct_index(lua_State *state)
 static int meta_struct_newindex(lua_State *state)
 {
     uint8_t *ptr = get_object_addr(state, 1, 2, "write");
-    const struct_field_info *field = find_field(state, 2, "write");
+    auto field = (struct_field_info*)find_field(state, 2, "write");
     write_field(state, field, ptr + field->offset, 3);
     return 0;
 }
 
+static int meta_primitive_index(lua_State *state)
+{
+    uint8_t *ptr = get_object_addr(state, 1, 2, "read");
+    auto type = (type_identity*)find_field(state, 2, "read");
+    if (!type)
+        return get_metafield(state);
+    return type->lua_read(state, 2, ptr);
+}
+
+static int meta_primitive_newindex(lua_State *state)
+{
+    uint8_t *ptr = get_object_addr(state, 1, 2, "write");
+    auto type = (type_identity*)find_field(state, 2, "write");
+    type->lua_write(state, 2, ptr, 3);
+    return 0;
+}
+
+static int meta_container_len(lua_State *state)
+{
+    uint8_t *ptr = get_object_addr(state, 1, 0, "get length");
+    auto id = (container_identity*)lua_touserdata(state, UPVAL_CONTAINER_ID);
+    int len = id->lua_item_count(state, ptr);
+    lua_pushinteger(state, len);
+    return 1;
+}
+
+static int lookup_container_field(lua_State *state, int field, const char *mode)
+{
+    if (lua_type(state, field) == LUA_TNUMBER)
+        return field;
+
+    lookup_field(state, field, mode);
+    return -1;
+}
+
+static int check_container_index(lua_State *state, container_identity *container, void *ptr,
+                                 int fidx, int iidx, const char *mode)
+{
+    if (!lua_isnumber(state, iidx))
+        field_error(state, fidx, "invalid index", mode);
+
+    int idx = lua_tointeger(state, iidx);
+    int len = container->lua_item_count(state, ptr);
+    if (idx < 0 || idx >= len)
+        field_error(state, fidx, "index out of bounds", mode);
+
+    return idx;
+}
+
+static int meta_container_index(lua_State *state)
+{
+    uint8_t *ptr = get_object_addr(state, 1, 2, "read");
+    int iidx = lookup_container_field(state, 2, "read");
+    if (lua_isuserdata(state, iidx))
+    {
+        lua_pop(state, 1);
+        return get_metafield(state);
+    }
+
+    auto id = (container_identity*)lua_touserdata(state, UPVAL_CONTAINER_ID);
+    int idx = check_container_index(state, id, ptr, 2, iidx, "read");
+    return id->lua_item_read(state, 2, ptr, idx);
+}
+
+static int meta_container_newindex(lua_State *state)
+{
+    uint8_t *ptr = get_object_addr(state, 1, 2, "write");
+    int iidx = lookup_container_field(state, 2, "write");
+    auto id = (container_identity*)lua_touserdata(state, UPVAL_CONTAINER_ID);
+    int idx = check_container_index(state, id, ptr, 2, iidx, "write");
+    id->lua_item_write(state, 2, ptr, idx, 3);
+    return 0;
+}
+
 static int meta_global_index(lua_State *state)
 {
-    const struct_field_info *field = find_field(state, 2, "read");
+    auto field = (struct_field_info*)find_field(state, 2, "read");
     if (!field)
         return get_metafield(state);
     void *ptr = *(void**)field->offset;
@@ -517,7 +681,7 @@ static int meta_global_index(lua_State *state)
 
 static int meta_global_newindex(lua_State *state)
 {
-    const struct_field_info *field = find_field(state, 2, "write");
+    auto field = (struct_field_info*)find_field(state, 2, "write");
     void *ptr = *(void**)field->offset;
     if (!ptr)
         field_error(state, 2, "global address not known", "write");
@@ -527,16 +691,16 @@ static int meta_global_newindex(lua_State *state)
 
 static void IndexFields(lua_State *state, const struct_field_info *fields)
 {
-    int base = lua_gettop(state);
-    lua_newtable(state); // read
-    lua_newtable(state); // write
+    // stack: read write
 
-    for (; fields->mode != struct_field_info::END; ++fields)
+    int base = lua_gettop(state) - 2;
+
+    for (; fields; ++fields)
     {
         switch (fields->mode)
         {
             case struct_field_info::END:
-                break;
+                return;
 
             case struct_field_info::PRIMITIVE:
             case struct_field_info::STATIC_STRING:
@@ -558,7 +722,13 @@ static void IndexFields(lua_State *state, const struct_field_info *fields)
     }
 }
 
-static void SetPtrMethods(lua_State *state, int meta_idx, void *node)
+static void EnableMetaField(lua_State *state, int ftable_idx, const char *name, void *id = NULL)
+{
+    lua_pushlightuserdata(state, id);
+    lua_setfield(state, ftable_idx, name);
+}
+
+static void SetPtrMethods(lua_State *state, int meta_idx, int read_idx)
 {
     lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_COMPARE_NAME);
     lua_setfield(state, meta_idx, "__eq");
@@ -568,12 +738,7 @@ static void SetPtrMethods(lua_State *state, int meta_idx, void *node)
     lua_pushcclosure(state, meta_ptr_tostring, 2);
     lua_setfield(state, meta_idx, "__tostring");
 
-    // type field
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPEID_TABLE_NAME);
-    lua_pushlightuserdata(state, node);
-    lua_rawget(state, -2);
-    lua_setfield(state, meta_idx, "_type");
-    lua_pop(state, 1);
+    EnableMetaField(state, read_idx, "_type");
 }
 
 static void SetStructMethod(lua_State *state, int meta_idx, int ftable_idx,
@@ -586,62 +751,199 @@ static void SetStructMethod(lua_State *state, int meta_idx, int ftable_idx,
     lua_setfield(state, meta_idx, name);
 }
 
+static void MakeMetatable(lua_State *state, type_identity *type)
+{
+    int base = lua_gettop(state);
+    lua_newtable(state); // metatable
+
+    lua_pushstring(state, type->getFullName().c_str());
+    lua_setfield(state, base+1, "__metatable");
+
+    lua_pushlightuserdata(state, type);
+    lua_setfield(state, base+1, "_identity");
+
+    LookupInTable(state, type, DFHACK_TYPEID_TABLE_NAME);
+    if (lua_isnil(state, -1))
+    {
+        lua_pop(state, 1);
+        lua_getfield(state, base+1, "__metatable");
+    }
+    lua_setfield(state, base+1, "_type");
+
+    lua_newtable(state); // read
+    lua_newtable(state); // write
+}
+
 static void MakeFieldMetatable(lua_State *state, struct_identity *pstruct,
                                lua_CFunction reader, lua_CFunction writer)
 {
     int base = lua_gettop(state);
 
-    lua_newtable(state); // metatable
-    IndexFields(state, pstruct->getFields()); // read, write
+    MakeMetatable(state, pstruct); // meta, read, write
 
-    lua_pushstring(state, pstruct->getName());
-    lua_setfield(state, base+1, "__metatable");
+    for (struct_identity *p = pstruct; p; p = p->getParent())
+    {
+        IndexFields(state, p->getFields());
+    }
 
     SetStructMethod(state, base+1, base+2, reader, "__index");
     SetStructMethod(state, base+1, base+3, writer, "__newindex");
 
-    // Custom fields
+    // returns: [metatable readfields writefields];
+}
 
-    lua_pushlightuserdata(state, pstruct);
-    lua_setfield(state, base+1, "_identity");
+static void MakePrimitiveMetatable(lua_State *state, type_identity *type)
+{
+    int base = lua_gettop(state);
 
-    // returns: [metatable readfields writefields];
+    MakeMetatable(state, type);
+    SetPtrMethods(state, base+1, base+2);
+
+    EnableMetaField(state, base+2, "value", type);
+    EnableMetaField(state, base+3, "value", type);
+
+    SetStructMethod(state, base+1, base+2, meta_primitive_index, "__index");
+    SetStructMethod(state, base+1, base+3, meta_primitive_newindex, "__newindex");
 }
 
-static void EnableMetaField(lua_State *state, int ftable_idx, const char *name)
+static void SetContainerMethod(lua_State *state, int meta_idx, int ftable_idx,
+                               lua_CFunction function, const char *name,
+                               type_identity *container, type_identity *item, int count)
 {
-    lua_pushlightuserdata(state, NULL);
-    lua_setfield(state, ftable_idx, name);
+    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_pushvalue(state, meta_idx);
+    lua_pushvalue(state, ftable_idx);
+
+    lua_pushlightuserdata(state, container);
+    lua_pushlightuserdata(state, item);
+    if (count < 0)
+        lua_pushnil(state);
+    else
+        lua_pushinteger(state, count);
+
+    lua_pushcclosure(state, function, 6);
+    lua_setfield(state, meta_idx, name);
 }
 
-static void BuildTypeMetatable(lua_State *state, type_identity *type)
+static void AttachEnumKeys(lua_State *state, int base, type_identity *ienum)
+{
+    LookupInTable(state, ienum, DFHACK_ENUM_TABLE_NAME);
+
+    if (!lua_isnil(state, -1))
+    {
+        lua_newtable(state);
+        lua_swap(state);
+        lua_setfield(state, -2, "__index");
+        lua_dup(state);
+        lua_setmetatable(state, base+2);
+        lua_setmetatable(state, base+3);
+    }
+    else
+        lua_pop(state, 1);
+}
+
+static void MakeContainerMetatable(lua_State *state, container_identity *type,
+                                   type_identity *item, int count, type_identity *ienum)
 {
     int base = lua_gettop(state);
 
-    switch (type->type())
+    MakeMetatable(state, type);
+    SetPtrMethods(state, base+1, base+2);
+
+    lua_pushstring(state, type->getFullName(item).c_str());
+    lua_dup(state);
+    lua_setfield(state, base+1, "__metatable");
+    lua_setfield(state, base+1, "_type");
+
+    lua_pushlightuserdata(state, item);
+    lua_setfield(state, base+1, "_field_identity");
+
+    if (count >= 0)
     {
-    case IDTYPE_GLOBAL:
-        assert(false);
-
-    case IDTYPE_STRUCT:
-    case IDTYPE_CLASS:
-        MakeFieldMetatable(state, (struct_identity*)type, meta_struct_index, meta_struct_newindex);
-        SetPtrMethods(state, base+1, type);
-        EnableMetaField(state, base+2, "_type");
-        lua_pop(state, 2);
-        return;
+        lua_pushinteger(state, count);
+        lua_setfield(state, base+1, "_count");
+    }
 
-    case IDTYPE_PRIMITIVE:
-    case IDTYPE_ENUM:
-    case IDTYPE_POINTER:
-        luaL_error(state, "primitive not implemented");
+    SetContainerMethod(state, base+1, base+2, meta_container_len, "__len", type, item, count);
+    SetContainerMethod(state, base+1, base+2, meta_container_index, "__index", type, item, count);
+    SetContainerMethod(state, base+1, base+3, meta_container_newindex, "__newindex", type, item, count);
+
+    AttachEnumKeys(state, base, ienum);
+}
+
+void type_identity::build_metatable(lua_State *state)
+{
+    MakePrimitiveMetatable(state, this);
+}
+
+void container_identity::build_metatable(lua_State *state)
+{
+    MakeContainerMetatable(state, this, getItemType(), -1, getIndexEnumType());
+}
+
+void pointer_identity::build_metatable(lua_State *state)
+{
+    int base = lua_gettop(state);
+
+    primitive_identity::build_metatable(state);
+
+    EnableMetaField(state, base+2, "target", this);
+    EnableMetaField(state, base+3, "target", this);
+}
+
+void struct_identity::build_metatable(lua_State *state)
+{
+    int base = lua_gettop(state);
+    MakeFieldMetatable(state, this, meta_struct_index, meta_struct_newindex);
+    SetPtrMethods(state, base+1, base+2);
+}
+
+void global_identity::build_metatable(lua_State *state)
+{
+    MakeFieldMetatable(state, this, meta_global_index, meta_global_newindex);
+}
+
+static void BuildTypeMetatable(lua_State *state, type_identity *type)
+{
+    type->build_metatable(state);
+
+    lua_pop(state, 2);
+
+    SaveTypeInfo(state, type);
+}
 
-    case IDTYPE_BITFIELD:
-        luaL_error(state, "bitfield not implemented");
+static void GetAdHocMetatable(lua_State *state, const struct_field_info *field)
+{
+    lua_pushlightuserdata(state, (void*)field);
+
+    if (!LookupTypeInfo(state, true))
+    {
+        switch (field->mode)
+        {
+        case struct_field_info::CONTAINER:
+        {
+            auto ctype = (container_identity*)field->type;
+            MakeContainerMetatable(state, ctype, ctype->getItemType(), -1, field->eid);
+            break;
+        }
+
+        case struct_field_info::STATIC_ARRAY:
+            MakeContainerMetatable(state, &df::buffer_container_identity::base_instance,
+                                   field->type, field->count, field->eid);
+            break;
+
+        case struct_field_info::STL_VECTOR_PTR:
+            MakeContainerMetatable(state, &df::identity_traits<std::vector<void*> >::identity,
+                                   field->type, -1, field->eid);
+            break;
+
+        default:
+            luaL_error(state, "Invalid ad-hoc field: %d", field->mode);
+        }
 
-    case IDTYPE_CONTAINER:
-    case IDTYPE_STL_PTR_VECTOR:
-        luaL_error(state, "container not implemented");
+        lua_pop(state, 2);
+
+        SaveTypeInfo(state, (void*)field);
     }
 }
 
@@ -650,6 +952,7 @@ static void RenderTypeChildren(lua_State *state, const std::vector<compound_iden
 static void RenderType(lua_State *state, compound_identity *node)
 {
     assert(node->getName());
+    std::string name = node->getFullName();
 
     lua_newtable(state);
     if (!lua_checkstack(state, 20))
@@ -691,7 +994,7 @@ static void RenderType(lua_State *state, compound_identity *node)
                 lua_setfield(state, base, "_last_item");
             }
 
-            SaveTypeInfo(state, node);
+            SaveInTable(state, node, DFHACK_ENUM_TABLE_NAME);
         }
         break;
 
@@ -705,10 +1008,7 @@ static void RenderType(lua_State *state, compound_identity *node)
 
     if (node->type() == IDTYPE_GLOBAL)
     {
-        auto gid = (global_identity*)node;
-
-        MakeFieldMetatable(state, gid, meta_global_index, meta_global_newindex);
-        lua_pop(state, 2);
+        BuildTypeMetatable(state, node);
 
         lua_dup(state);
         lua_setmetatable(state, base);
@@ -719,22 +1019,17 @@ static void RenderType(lua_State *state, compound_identity *node)
         lua_getfield(state, base, "__newindex");
         lua_setfield(state, -2, "__newindex");
 
-        lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPE_TOSTRING_NAME);
-        lua_setfield(state, -2, "__tostring");
-
-        lua_pop(state, 1);
-
         lua_remove(state, base);
     }
     else
     {
-        freeze_table(state, true, node->getName());
+        freeze_table(state, true, name.c_str());
+    }
 
-        lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPE_TOSTRING_NAME);
-        lua_setfield(state, -2, "__tostring");
+    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPE_TOSTRING_NAME);
+    lua_setfield(state, -2, "__tostring");
 
-        lua_pop(state, 1);
-    }
+    lua_pop(state, 1);
 
     SaveInTable(state, node, DFHACK_TYPEID_TABLE_NAME);
 }
@@ -757,6 +1052,9 @@ static void DoAttach(lua_State *state)
     lua_newtable(state);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_TYPEID_TABLE_NAME);
 
+    lua_newtable(state);
+    lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_ENUM_TABLE_NAME);
+
     lua_pushcfunction(state, change_error);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_CHANGEERROR_NAME);
 

library/include/DataDefs.h

@@ -52,6 +52,8 @@ namespace DFHack
         IDTYPE_PRIMITIVE,
         IDTYPE_POINTER,
         IDTYPE_CONTAINER,
+        IDTYPE_PTR_CONTAINER,
+        IDTYPE_BIT_CONTAINER,
         IDTYPE_BITFIELD,
         IDTYPE_ENUM,
         IDTYPE_STRUCT,
@@ -85,6 +87,11 @@ namespace DFHack
 
         virtual int lua_read(lua_State *state, int fname_idx, void *ptr) = 0;
         virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) = 0;
+
+        virtual std::string getFullName() = 0;
+        virtual void build_metatable(lua_State *state);
+
+        virtual bool isContainer() { return false; }
     };
 
     class DFHACK_EXPORT constructed_identity : public type_identity {
@@ -124,6 +131,8 @@ namespace DFHack
     public:
         const char *getName() { return dfhack_name; }
 
+        virtual std::string getFullName();
+
         compound_identity *getScopeParent() { return scope_parent; }
         const std::vector<compound_identity*> &getScopeChildren() { return scope_children; }
         static const std::vector<compound_identity*> &getTopScope() { return top_scope; }
@@ -222,6 +231,8 @@ namespace DFHack
         const struct_field_info *getFields() { return fields; }
 
         bool is_subclass(struct_identity *subtype);
+
+        virtual void build_metatable(lua_State *state);
     };
 
     class DFHACK_EXPORT global_identity : public struct_identity {
@@ -230,6 +241,8 @@ namespace DFHack
             : struct_identity(0,NULL,NULL,"global",NULL,fields) {}
 
         virtual identity_type type() { return IDTYPE_GLOBAL; }
+
+        virtual void build_metatable(lua_State *state);
     };
 
 #ifdef _MSC_VER

library/include/DataIdentity.h

@@ -55,6 +55,9 @@ namespace DFHack
 
         type_identity *getTarget() { return target; }
 
+        std::string getFullName();
+        virtual void build_metatable(lua_State *state);
+
         static int lua_read(lua_State *state, int fname_idx, void *ptr, type_identity *target);
         static void lua_write(lua_State *state, int fname_idx, void *ptr, type_identity *target, int val_index);
 
@@ -72,7 +75,57 @@ namespace DFHack
 
         virtual identity_type type() { return IDTYPE_CONTAINER; }
 
+        std::string getFullName() { return getFullName(item); }
+
+        virtual void build_metatable(lua_State *state);
+        virtual bool isContainer() { return true; }
+
         type_identity *getItemType() { return item; }
+        type_identity *getIndexEnumType() { return ienum; }
+
+        virtual std::string getFullName(type_identity *item);
+
+        int lua_item_count(lua_State *state, void *ptr);
+
+        virtual int lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+
+    protected:
+        virtual int item_count(void *ptr) = 0;
+        virtual void *item_pointer(void *ptr, int idx) = 0;
+    };
+
+    class DFHACK_EXPORT ptr_container_identity : public container_identity {
+    public:
+        ptr_container_identity(size_t size, TAllocateFn alloc,
+                               type_identity *item, enum_identity *ienum = NULL)
+            : container_identity(size, alloc, item, ienum) {};
+
+        virtual identity_type type() { return IDTYPE_PTR_CONTAINER; }
+
+        std::string getFullName(type_identity *item);
+
+        virtual int lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+    };
+
+    class DFHACK_EXPORT bit_container_identity : public container_identity {
+    public:
+        bit_container_identity(size_t size, TAllocateFn alloc, enum_identity *ienum = NULL)
+            : container_identity(size, alloc, NULL, ienum) {};
+
+        virtual identity_type type() { return IDTYPE_BIT_CONTAINER; }
+
+        std::string getFullName(type_identity *item);
+
+        virtual int lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+
+    protected:
+        virtual void *item_pointer(void *, int) { return NULL; }
+
+        virtual bool get_item(void *ptr, int idx) = 0;
+        virtual void set_item(void *ptr, int idx, bool val) = 0;
     };
 }
 
@@ -81,10 +134,17 @@ namespace df
     using DFHack::primitive_identity;
     using DFHack::pointer_identity;
     using DFHack::container_identity;
+    using DFHack::ptr_container_identity;
+    using DFHack::bit_container_identity;
 
     class number_identity_base : public primitive_identity {
+        const char *name;
+
     public:
-        number_identity_base(size_t size) : primitive_identity(size) {};
+        number_identity_base(size_t size, const char *name)
+            : primitive_identity(size), name(name) {};
+
+        std::string getFullName() { return name; }
 
         virtual int lua_read(lua_State *state, int fname_idx, void *ptr);
         virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
@@ -97,7 +157,7 @@ namespace df
     template<class T>
     class number_identity : public number_identity_base {
     public:
-        number_identity() : number_identity_base(sizeof(T)) {}
+        number_identity(const char *name) : number_identity_base(sizeof(T), name) {}
     protected:
         virtual double read(void *ptr) { return double(*(T*)ptr); }
         virtual void write(void *ptr, double val) { *(T*)ptr = T(val); }
@@ -107,6 +167,8 @@ namespace df
     public:
         bool_identity() : primitive_identity(sizeof(bool)) {};
 
+        std::string getFullName() { return "bool"; }
+
         virtual int lua_read(lua_State *state, int fname_idx, void *ptr);
         virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
     };
@@ -115,17 +177,118 @@ namespace df
     public:
         stl_string_identity() : primitive_identity(sizeof(std::string)) {};
 
+        std::string getFullName() { return "string"; }
+
         virtual int lua_read(lua_State *state, int fname_idx, void *ptr);
         virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index);
     };
 
-    class stl_ptr_vector_identity : public container_identity {
+    class stl_ptr_vector_identity : public ptr_container_identity {
     public:
         stl_ptr_vector_identity(type_identity *item = NULL, enum_identity *ienum = NULL)
-            : container_identity(sizeof(std::vector<void*>),allocator_fn<std::vector<void*> >,item, ienum)
+            : ptr_container_identity(sizeof(std::vector<void*>),allocator_fn<std::vector<void*> >,item, ienum)
         {};
 
+        std::string getFullName(type_identity *item) {
+            return "vector" + ptr_container_identity::getFullName(item);
+        }
+
         virtual DFHack::identity_type type() { return DFHack::IDTYPE_STL_PTR_VECTOR; }
+
+    protected:
+        virtual int item_count(void *ptr) {
+            return ((std::vector<void*>*)ptr)->size();
+        };
+        virtual void *item_pointer(void *ptr, int idx) {
+            return &(*(std::vector<void*>*)ptr)[idx];
+        }
+    };
+
+    class buffer_container_identity : public container_identity {
+        int size;
+
+    public:
+        buffer_container_identity()
+            : container_identity(0, NULL, NULL, NULL), size(0)
+        {}
+
+        buffer_container_identity(int size, type_identity *item, enum_identity *ienum = NULL)
+            : container_identity(item->byte_size()*size, NULL, item, ienum), size(size)
+        {}
+
+        std::string getFullName(type_identity *item);
+
+        static buffer_container_identity base_instance;
+
+        virtual int lua_item_read(lua_State *state, int fname_idx, void *ptr, int idx);
+        virtual void lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index);
+
+    protected:
+        virtual int item_count(void *ptr) { return size; }
+        virtual void *item_pointer(void *ptr, int idx) { return NULL; }
+    };
+
+    template<class T>
+    class stl_container_identity : public container_identity {
+        const char *name;
+
+    public:
+        stl_container_identity(const char *name, type_identity *item, enum_identity *ienum = NULL)
+            : container_identity(sizeof(T), &allocator_fn<T>, item, ienum), name(name)
+        {}
+
+        std::string getFullName(type_identity *item) {
+            return name + container_identity::getFullName(item);
+        }
+
+    protected:
+        virtual int item_count(void *ptr) { return ((T*)ptr)->size(); }
+        virtual void *item_pointer(void *ptr, int idx) { return &(*(T*)ptr)[idx]; }
+    };
+
+    template<class T>
+    class bit_array_identity : public bit_container_identity {
+    public:
+        typedef BitArray<T> container;
+
+        bit_array_identity(enum_identity *ienum = NULL)
+            : bit_container_identity(sizeof(container), &allocator_fn<container>, ienum)
+        {}
+
+        std::string getFullName(type_identity *item) {
+            return "BitArray" + bit_container_identity::getFullName(item);
+        }
+
+    protected:
+        virtual int item_count(void *ptr) { return ((container*)ptr)->size * 8; }
+        virtual bool get_item(void *ptr, int idx) {
+            return ((container*)ptr)->is_set(T(idx));
+        }
+        virtual void set_item(void *ptr, int idx, bool val) {
+            ((container*)ptr)->set(T(idx), val);
+        }
+    };
+
+    class stl_bit_vector_identity : public bit_container_identity {
+    public:
+        typedef std::vector<bool> container;
+
+        stl_bit_vector_identity(enum_identity *ienum = NULL)
+            : bit_container_identity(sizeof(container), &allocator_fn<container>, ienum)
+        {}
+
+        std::string getFullName(type_identity *item) {
+            return "vector" + bit_container_identity::getFullName(item);
+        }
+
+    protected:
+        virtual int item_count(void *ptr) { return ((container*)ptr)->size(); }
+        virtual bool get_item(void *ptr, int idx) {
+            return (*(container*)ptr)[idx];
+        }
+        virtual void set_item(void *ptr, int idx, bool val) {
+            (*(container*)ptr)[idx] = val;
+        }
     };
 
 #define NUMBER_IDENTITY_TRAITS(type) \
@@ -165,6 +328,11 @@ namespace df
         static stl_ptr_vector_identity *get() { return &identity; }
     };
 
+    template<> struct identity_traits<std::vector<bool> > {
+        static stl_bit_vector_identity identity;
+        static stl_bit_vector_identity *get() { return &identity; }
+    };
+
 #undef NUMBER_IDENTITY_TRAITS
 
     // Container declarations
@@ -194,7 +362,7 @@ namespace df
     };
 
     template<class T> struct identity_traits<BitArray<T> > {
-        static container_identity *get();
+        static bit_container_identity *get();
     };
 
     template<class T> struct identity_traits<DfArray<T> > {
@@ -216,17 +384,14 @@ namespace df
 
     template<class T, int sz>
     container_identity *identity_traits<T [sz]>::get() {
-        typedef T container[sz];
-        static container_identity identity(sizeof(container), NULL,
-                                            identity_traits<T>::get());
+        static buffer_container_identity identity(sz, identity_traits<T>::get());
         return &identity;
     }
 
     template<class T>
     container_identity *identity_traits<std::vector<T> >::get() {
         typedef std::vector<T> container;
-        static container_identity identity(sizeof(container), &allocator_fn<container>,
-                                            identity_traits<T>::get());
+        static stl_container_identity<container> identity("vector", identity_traits<T>::get());
         return &identity;
     }
 
@@ -239,26 +404,22 @@ namespace df
     template<class T>
     container_identity *identity_traits<std::deque<T> >::get() {
         typedef std::deque<T> container;
-        static container_identity identity(sizeof(container), &allocator_fn<container>,
-                                            identity_traits<T>::get());
+        static stl_container_identity<container> identity("deque", identity_traits<T>::get());
         return &identity;
     }
 
     template<class T>
-    container_identity *identity_traits<BitArray<T> >::get() {
-        typedef BitArray<T> container;
+    bit_container_identity *identity_traits<BitArray<T> >::get() {
         static type_identity *eid = identity_traits<T>::get();
         static enum_identity *reid = eid->type() == DFHack::IDTYPE_ENUM ? (enum_identity*)eid : NULL;
-        static container_identity identity(sizeof(container), &allocator_fn<container>,
-                                            &identity_traits<bool>::identity, reid);
+        static bit_array_identity<T> identity(reid);
         return &identity;
     }
 
     template<class T>
     container_identity *identity_traits<DfArray<T> >::get() {
         typedef DfArray<T> container;
-        static container_identity identity(sizeof(container), &allocator_fn<container>,
-                                            identity_traits<T>::get());
+        static stl_container_identity<container> identity("DfArray", identity_traits<T>::get());
         return &identity;
     }
 }