Merge remote-tracking branch 'lethosor/struct-fields-lua' into develop

docs/dev/Lua API.rst

@@ -306,7 +306,32 @@ All types and the global object have the following features:
 * ``type._identity``
 
   Contains a lightuserdata pointing to the underlying
-  ``DFHack::type_instance`` object.
+  ``DFHack::type_identity`` object.
+
+All compound types (structs, classes, unions, and the global object) support:
+
+* ``type._fields``
+
+  Contains a table mapping field names to descriptions of the type's fields,
+  including data members and functions. Iterating with ``pairs()`` returns data
+  fields in the order they are defined in the type. Functions and globals may
+  appear in an arbitrary order.
+
+  Each entry contains the following fields:
+
+  * ``name``: the name of the field (matches the ``_fields`` table key)
+  * ``offset``: for data members, the position of the field relative to the start of the type, in bytes
+  * ``count``: for arrays, the number of elements
+  * ``mode``: implementation detail. See ``struct_field_info::Mode`` in ``DataDefs.h``.
+
+  Each entry may also contain the following fields, depending on its type:
+
+  * ``type_name``: present for most fields; a string representation of the field's type
+  * ``type``: the type object matching the field's type; present if such an object exists
+    (e.g. present for DF types, absent for primitive types)
+  * ``type_identity``: present for most fields; a lightuserdata pointing to the field's underlying ``DFHack::type_identity`` object
+  * ``index_enum``, ``ref_target``: the type object corresponding to the field's similarly-named XML attribute, if present
+  * ``union_tag_field``, ``union_tag_attr``, ``original_name``: the string value of the field's similarly-named XML attribute, if present
 
 Types excluding the global object also support:
 

library/LuaTypes.cpp

@@ -1291,11 +1291,27 @@ void LuaWrapper::SetFunctionWrappers(lua_State *state, const FunctionReg *reg)
 
 /**
  * Add fields in the array to the UPVAL_FIELDTABLE candidates on the stack.
+ *
+ * flags:
+ *  GLOBALS: if true, pstruct is a global_identity and fields with addresses of 0 are skipped
+ *  RAW: if true, no fields are skipped (supersedes `GLOBALS` flag) and
+ *       special-case fields like OBJ_METHODs are not added to the metatable
+ *
+ * Stack in & out:
+ *  base+1: metatable
+ *  base+2: fields table (to be populated, map of name -> struct_field_info*)
+ *  base+3: field iter table (to be populated, bimap of name <-> integer index)
  */
-static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bool globals)
+namespace IndexFieldsFlags {
+    enum IndexFieldsFlags {
+        GLOBALS = 1 << 0,
+        RAW     = 1 << 1,
+    };
+}
+static void IndexFields(lua_State *state, int base, struct_identity *pstruct, int flags)
 {
     if (pstruct->getParent())
-        IndexFields(state, base, pstruct->getParent(), globals);
+        IndexFields(state, base, pstruct->getParent(), flags);
 
     auto fields = pstruct->getFields();
     if (!fields)
@@ -1315,6 +1331,7 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo
 
         bool add_to_enum = true;
 
+        if (!(flags & IndexFieldsFlags::RAW))
         // Handle the field
         switch (fields[i].mode)
         {
@@ -1337,10 +1354,10 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo
         }
 
         // Do not add invalid globals to the enumeration order
-        if (globals && !*(void**)fields[i].offset)
+        if ((flags & IndexFieldsFlags::GLOBALS) && !*(void**)fields[i].offset)
             add_to_enum = false;
 
-        if (add_to_enum)
+        if (add_to_enum || (flags & IndexFieldsFlags::RAW))
             AssociateId(state, base+3, ++cnt, name.c_str());
 
         lua_pushlightuserdata(state, (void*)&fields[i]);
@@ -1348,10 +1365,168 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo
     }
 }
 
+static void PushTypeIdentity(lua_State *state, const type_identity *id)
+{
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPEID_TABLE_TOKEN);
+    lua_rawgetp(state, -1, id);
+    lua_remove(state, -2);  // TYPEID_TABLE
+}
+
+static void PushFieldInfoSubTable(lua_State *state, const struct_field_info *field)
+{
+    if (!field) {
+        lua_pushnil(state);
+        return;
+    }
+
+    lua_newtable(state);    // new field info
+    Lua::TableInsert(state, "mode", field->mode);
+    Lua::TableInsert(state, "name", field->name);
+    Lua::TableInsert(state, "offset", field->offset);
+    Lua::TableInsert(state, "count", field->count);
+
+    if (field->type) {
+        Lua::TableInsert(state, "type_name", field->type->getFullName());
+
+        lua_pushlightuserdata(state, field->type);
+        lua_setfield(state, -2, "type_identity");
+
+        PushTypeIdentity(state, field->type);
+        lua_setfield(state, -2, "type");
+    }
+
+    if (field->extra) {
+        if (field->extra->index_enum) {
+            PushTypeIdentity(state, field->extra->index_enum);
+            lua_setfield(state, -2, "index_enum");
+        }
+        if (field->extra->ref_target) {
+            PushTypeIdentity(state, field->extra->ref_target);
+            lua_setfield(state, -2, "ref_target");
+        }
+        if (field->extra->union_tag_field) {
+            Lua::TableInsert(state, "union_tag_field", field->extra->union_tag_field);
+        }
+        if (field->extra->union_tag_attr) {
+            Lua::TableInsert(state, "union_tag_attr", field->extra->union_tag_attr);
+        }
+        if (field->extra->original_name) {
+            Lua::TableInsert(state, "original_name", field->extra->original_name);
+        }
+    }
+}
+
+/**
+ * Metamethod: __index for struct._fields
+ *
+ * upvalue 1: name -> struct_field_info* table
+ */
+static int meta_fieldinfo_index(lua_State *state)
+{
+    luaL_checktype(state, -1, LUA_TSTRING);
+
+    lua_gettable(state, lua_upvalueindex(1));
+    auto field = static_cast<const struct_field_info*>(lua_touserdata(state, -1));
+    lua_pop(state, 1);
+    PushFieldInfoSubTable(state, field);
+
+    return 1;
+}
+
+/**
+ * Metamethod: iterator for struct._fields
+ *
+ * upvalue 1: name -> struct_field_info* table
+ * upvalue 3: field table (int <-> name)
+ */
+static int meta_fieldinfo_next(lua_State *state)
+{
+    if (lua_gettop(state) < 2) lua_pushnil(state);
+
+    int len = lua_rawlen(state, UPVAL_FIELDTABLE);
+    int idx = cur_iter_index(state, len+1, 2, 0);
+    if (idx == len)
+        return 0;
+
+    lua_rawgeti(state, UPVAL_FIELDTABLE, idx+1);
+
+    // modified from meta_struct_next:
+    // retrieve the struct_field_info* from the table and convert it
+    lua_dup(state);
+    lua_gettable(state, lua_upvalueindex(1));
+    auto field = static_cast<const struct_field_info*>(lua_touserdata(state, -1));
+    lua_pop(state, 1);
+    PushFieldInfoSubTable(state, field);
+
+    return 2;
+}
+
+static void AddFieldInfoTable(lua_State *state, int ftable_idx, struct_identity *pstruct)
+{
+    Lua::StackUnwinder base{state};
+
+    // metatable
+    lua_newtable(state);
+    int ix_meta = lua_gettop(state);
+
+    // field info table (name -> struct_field_info*)
+    lua_newtable(state);
+    int ix_fieldinfo = lua_gettop(state);
+
+    // field iter table (int <-> name)
+    lua_newtable(state);
+    int ix_fielditer = lua_gettop(state);
+    IndexFields(state, base, pstruct, IndexFieldsFlags::RAW);
+
+    PushStructMethod(state, ix_meta, ix_fielditer, meta_fieldinfo_next);
+    // change upvalue 1 to the field info table since we don't need the original
+    lua_pushvalue(state, ix_fieldinfo);
+    lua_setupvalue(state, -2, 1);
+    SetPairsMethod(state, ix_meta, "__pairs");
+
+    // field table (name -> table representation of struct_field_info)
+    lua_newtable(state);
+    int ix_fields = lua_gettop(state);
+
+    // wrapper table (empty, indexes into field table with metamethods)
+    lua_newtable(state);
+    int ix_wrapper = lua_gettop(state);
+
+    // set up metatable for the wrapper
+    // use field table for __index
+    lua_pushstring(state, "__index");
+    lua_pushvalue(state, ix_fieldinfo);
+    lua_pushcclosure(state, meta_fieldinfo_index, 1);
+    lua_settable(state, ix_meta);
+
+    // use change_error() for __newindex
+    lua_pushstring(state, "__newindex");
+    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_CHANGEERROR_NAME);
+    lua_settable(state, ix_meta);
+
+    lua_pushvalue(state, ix_meta);
+    lua_setmetatable(state, ix_wrapper);
+
+    // convert field info table (struct_field_info) to field table (lua tables)
+    lua_pushnil(state);  // initial key for next()
+    while (lua_next(state, ix_fieldinfo)) {
+        auto field = static_cast<const struct_field_info*>(lua_touserdata(state, -1));
+        lua_pushvalue(state, -2);  // field name
+        PushFieldInfoSubTable(state, field);
+        lua_settable(state, ix_fields);
+        lua_pop(state, 1); // struct_field_info
+    }
+
+    // lua_pushvalue(state, ix_fields);
+    // freeze_table(state); // TODO: figure out why this creates an __index cycle for nonexistent fields
+    lua_pushvalue(state, ix_wrapper);
+    lua_setfield(state, ftable_idx, "_fields");
+}
+
 void LuaWrapper::IndexStatics(lua_State *state, int meta_idx, int ftable_idx, struct_identity *pstruct)
 {
     // stack: metatable fieldtable
-
+    AddFieldInfoTable(state, ftable_idx, pstruct);
     for (struct_identity *p = pstruct; p; p = p->getParent())
     {
         auto fields = p->getFields();
@@ -1387,8 +1562,7 @@ static void MakeFieldMetatable(lua_State *state, struct_identity *pstruct,
 
     // Index the fields
     lua_newtable(state);
-
+    IndexFields(state, base, pstruct, globals ? IndexFieldsFlags::GLOBALS : 0);
-    IndexFields(state, base, pstruct, globals);
 
     // Add the iteration metamethods
     PushStructMethod(state, base+1, base+3, iterator);

library/LuaWrapper.cpp

@@ -1670,6 +1670,7 @@ static void RenderType(lua_State *state, compound_identity *node)
 
         {
             RenderTypeChildren(state, node->getScopeChildren());
+            IndexStatics(state, ix_meta, ftable, (struct_identity*)node);
 
             lua_pushlightuserdata(state, node);
             lua_setfield(state, ftable, "_identity");

test/structures/struct_fields.lua

@@ -0,0 +1,112 @@
+config.target = 'core'
+
+local COORD_FIELD_NAMES = {'x', 'y', 'z', 'isValid', 'clear'}
+local COORD_FIELD_EXPECTED_DATA = {
+    x = {type_name='int16_t'},
+    y = {type_name='int16_t'},
+    z = {type_name='int16_t'},
+    isValid = {type_name='function'},
+    clear = {type_name='function'},
+}
+
+local READONLY_MSG = 'Attempt to change a read%-only table.'
+
+local function listFieldNames(t)
+    local names = {}
+    for name in pairs(t._fields) do
+        table.insert(names, name)
+    end
+    return names
+end
+
+function test.access()
+    local fields = df.coord._fields
+    expect.true_(fields)
+    expect.eq(fields, df.coord._fields)
+
+    for name, expected in pairs(COORD_FIELD_EXPECTED_DATA) do
+        expect.true_(fields[name], name)
+        expect.eq(fields[name].name, name, name)
+        expect.eq(fields[name].type_name, expected.type_name, name)
+        expect.eq(type(fields[name].offset), 'number', name)
+        expect.eq(type(fields[name].mode), 'number', name)
+        expect.eq(type(fields[name].count), 'number', name)
+    end
+end
+
+function test.globals_original_name()
+    for name, info in pairs(df.global._fields) do
+        expect.eq(type(info.original_name), 'string', name)
+    end
+end
+
+function test.order()
+    expect.table_eq(listFieldNames(df.coord), COORD_FIELD_NAMES)
+end
+
+function test.nonexistent()
+    expect.nil_(df.coord._fields.nonexistent)
+
+    expect.error_match('string expected', function()
+        expect.nil_(df.coord._fields[2])
+    end)
+    expect.error_match('string expected', function()
+        expect.nil_(df.coord._fields[nil])
+    end)
+end
+
+function test.count()
+    expect.eq(df.unit._fields.relationship_ids.count, 10)
+end
+
+function test.index_enum()
+    expect.eq(df.unit._fields.relationship_ids.index_enum, df.unit_relationship_type)
+end
+
+function test.ref_target()
+    expect.eq(df.unit._fields.hist_figure_id.ref_target, df.historical_figure)
+end
+
+function test.readonly()
+    expect.error_match(READONLY_MSG, function()
+        df.coord._fields.x = 'foo'
+    end)
+    expect.error_match(READONLY_MSG, function()
+        df.coord._fields.nonexistent = 'foo'
+    end)
+    expect.nil_(df.coord._fields.nonexistent)
+
+    -- should have no effect
+    df.coord._fields.x.name = 'foo'
+    expect.eq(df.coord._fields.x.name, 'x')
+end
+
+function test.circular_refs_init()
+    expect.eq(df.job._fields.list_link.type, df.job_list_link)
+    expect.eq(df.job_list_link._fields.item.type, df.job)
+end
+
+function test.subclass_match()
+    for f, parent in pairs(df.viewscreen._fields) do
+        local child = df.viewscreen_titlest._fields[f]
+        expect.table_eq(parent, child, f)
+    end
+end
+
+function test.subclass_order()
+    -- ensure that parent class fields come before subclass fields
+    local hierarchy = {df.item, df.item_actual, df.item_crafted, df.item_constructed, df.item_bedst}
+    local field_names = {}
+    for _, t in pairs(hierarchy) do
+        field_names[t] = listFieldNames(t)
+    end
+    for ic = 1, #hierarchy do
+        for ip = 1, ic - 1 do
+            local parent_fields = listFieldNames(hierarchy[ip])
+            local child_fields = listFieldNames(hierarchy[ic])
+            child_fields = table.pack(table.unpack(child_fields, 1, #parent_fields))
+            child_fields.n = nil
+            expect.table_eq(child_fields, parent_fields, ('compare %s to %s'):format(hierarchy[ip], hierarchy[ic]))
+        end
+    end
+end