Merge branch 'master' of git://github.com/peterix/dfhack

depends/lua/CMakeLists.txt

@@ -1,8 +1,7 @@
 PROJECT ( lua CXX )
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
-# TODO: make this RelWithDebInfo only
-ADD_DEFINITIONS(-DLUA_USE_APICHECK)
+SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-DLUA_USE_APICHECK")
 
 IF(WIN32)
     ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE )

depends/lua/src/lauxlib.c

@@ -863,8 +863,8 @@ LUALIB_API int luaL_getsubtable (lua_State *L, int idx, const char *fname) {
   lua_getfield(L, idx, fname);
   if (lua_istable(L, -1)) return 1;  /* table already there */
   else {
-    idx = lua_absindex(L, idx);
     lua_pop(L, 1);  /* remove previous result */
+    idx = lua_absindex(L, idx);
     lua_newtable(L);
     lua_pushvalue(L, -1);  /* copy to be left at top */
     lua_setfield(L, idx, fname);  /* assign new table to field */

library/Core.cpp

@@ -879,31 +879,30 @@ int Core::Update()
             new_wdata = wdata;
         new_mapdata = df::global::world->map.block_index;
     }
+
     // if the world changes
     if (new_wdata != last_world_data_ptr)
     {
         // we check for map change too
         bool mapchange = new_mapdata != last_local_map_ptr;
+        last_world_data_ptr = new_wdata;
+        last_local_map_ptr = new_mapdata;
+
+        getWorld()->ClearPersistentCache();
+
         // and if the world is going away, we report the map change first
         if(!new_wdata && mapchange)
-        {
-            last_local_map_ptr = new_mapdata;
             plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
-        }
         // and if the world is appearing, we report map change after that
         plug_mgr->OnStateChange(out, new_wdata ? SC_WORLD_LOADED : SC_WORLD_UNLOADED);
         if(new_wdata && mapchange)
-        {
-            last_local_map_ptr = new_mapdata;
             plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
     }
-        // update tracking variable
-        last_world_data_ptr = new_wdata;
-    }
     // otherwise just check for map change...
     else if (new_mapdata != last_local_map_ptr)
     {
         last_local_map_ptr = new_mapdata;
+        getWorld()->ClearPersistentCache();
         plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
     }
 

library/LuaTools.cpp

@@ -36,6 +36,8 @@ distribution.
 #include "DataDefs.h"
 #include "DataIdentity.h"
 
+#include "modules/World.h"
+
 #include "LuaWrapper.h"
 #include "LuaTools.h"
 
@@ -189,7 +191,7 @@ bool DFHack::Lua::Require(color_ostream &out, lua_State *state,
 
 static bool load_with_env(color_ostream &out, lua_State *state, const std::string &code, int eidx)
 {
-    if (luaL_loadstring(state, code.c_str()) != LUA_OK)
+    if (luaL_loadbuffer(state, code.data(), code.size(), "=(interactive)") != LUA_OK)
     {
         report_error(out, state);
         return false;
@@ -223,11 +225,16 @@ bool DFHack::Lua::InterpreterLoop(color_ostream &out, lua_State *state,
     DFHack::CommandHistory hist;
     hist.load(hfile);
 
-    out.print("Type quit to exit interactive lua interpreter.\n"
-              "Shortcuts:\n"
+    out.print("Type quit to exit interactive lua interpreter.\n");
+
+    static bool print_banner = true;
+    if (print_banner) {
+        out.print("Shortcuts:\n"
                   " '= foo' => '_1,_2,... = foo'\n"
                   " '! foo' => 'print(foo)'\n"
-              "Both assign the first result to '_'\n");
+                  "Both save the first result as '_'.\n");
+        print_banner = false;
+    }
 
     Console &con = static_cast<Console&>(out);
 
@@ -303,7 +310,7 @@ bool DFHack::Lua::InterpreterLoop(color_ostream &out, lua_State *state,
         {
             if (!load_with_env(out, state, curline, base))
                 continue;
-            if (!SafeCall(out, state, 0, LUA_MULTRET))
+            if (!SafeCall(out, state, 0, 0))
                 continue;
         }
     }
@@ -330,15 +337,254 @@ static int lua_dfhack_interpreter(lua_State *state)
     return 1;
 }
 
+static int lua_dfhack_with_suspend(lua_State *L)
+{
+    int ctx;
+    int rv = lua_getctx(L, &ctx);
+
+    // Non-resume entry point:
+    if (rv == LUA_OK)
+    {
+        int nargs = lua_gettop(L);
+
+        luaL_checktype(L, 1, LUA_TFUNCTION);
+
+        Core::getInstance().Suspend();
+
+        lua_pushcfunction(L, traceback);
+        lua_insert(L, 1);
+
+        rv = lua_pcallk(L, nargs-1, LUA_MULTRET, 1, 0, lua_dfhack_with_suspend);
+    }
+
+    // Return, resume, or error entry point:
+    lua_remove(L, 1);
+
+    Core::getInstance().Resume();
+
+    if (rv != LUA_OK && rv != LUA_YIELD)
+        lua_error(L);
+
+    return lua_gettop(L);
+}
+
 static const luaL_Reg dfhack_funcs[] = {
     { "print", lua_dfhack_print },
     { "println", lua_dfhack_println },
     { "printerr", lua_dfhack_printerr },
     { "traceback", traceback },
     { "interpreter", lua_dfhack_interpreter },
+    { "with_suspend", lua_dfhack_with_suspend },
     { NULL, NULL }
 };
 
+/*
+ * Per-world persistent configuration storage.
+ */
+
+static PersistentDataItem persistent_by_struct(lua_State *state, int idx)
+{
+    lua_getfield(state, idx, "entry_id");
+    int id = lua_tointeger(state, -1);
+    lua_pop(state, 1);
+
+    PersistentDataItem ref = Core::getInstance().getWorld()->GetPersistentData(id);
+
+    if (ref.isValid())
+    {
+        lua_getfield(state, idx, "key");
+        const char *str = lua_tostring(state, -1);
+        if (!str || str != ref.key())
+            luaL_argerror(state, idx, "inconsistent id and key");
+        lua_pop(state, 1);
+    }
+
+    return ref;
+}
+
+static int read_persistent(lua_State *state, PersistentDataItem ref, bool create)
+{
+    if (!ref.isValid())
+    {
+        lua_pushnil(state);
+        lua_pushstring(state, "entry not found");
+        return 2;
+    }
+
+    if (create)
+        lua_createtable(state, 0, 4);
+
+    lua_pushvalue(state, lua_upvalueindex(1));
+    lua_setmetatable(state, -2);
+
+    lua_pushinteger(state, ref.entry_id());
+    lua_setfield(state, -2, "entry_id");
+    lua_pushstring(state, ref.key().c_str());
+    lua_setfield(state, -2, "key");
+    lua_pushstring(state, ref.val().c_str());
+    lua_setfield(state, -2, "value");
+
+    lua_createtable(state, PersistentDataItem::NumInts, 0);
+    for (int i = 0; i < PersistentDataItem::NumInts; i++)
+    {
+        lua_pushinteger(state, ref.ival(i));
+        lua_rawseti(state, -2, i+1);
+    }
+    lua_setfield(state, -2, "ints");
+
+    return 1;
+}
+
+static PersistentDataItem get_persistent(lua_State *state)
+{
+    luaL_checkany(state, 1);
+
+    if (lua_istable(state, 1))
+    {
+        if (!lua_getmetatable(state, 1) ||
+            !lua_rawequal(state, -1, lua_upvalueindex(1)))
+            luaL_argerror(state, 1, "invalid table type");
+
+        lua_settop(state, 1);
+
+        return persistent_by_struct(state, 1);
+    }
+    else
+    {
+        const char *str = luaL_checkstring(state, 1);
+
+        return Core::getInstance().getWorld()->GetPersistentData(str);
+    }
+}
+
+static int dfhack_persistent_get(lua_State *state)
+{
+    auto ref = get_persistent(state);
+
+    return read_persistent(state, ref, !lua_istable(state, 1));
+}
+
+static int dfhack_persistent_delete(lua_State *state)
+{
+    auto ref = get_persistent(state);
+
+    bool ok = Core::getInstance().getWorld()->DeletePersistentData(ref);
+
+    lua_pushboolean(state, ok);
+    return 1;
+}
+
+static int dfhack_persistent_get_all(lua_State *state)
+{
+    const char *str = luaL_checkstring(state, 1);
+    bool prefix = (lua_gettop(state)>=2 ? lua_toboolean(state,2) : false);
+
+    std::vector<PersistentDataItem> data;
+    Core::getInstance().getWorld()->GetPersistentData(&data, str, prefix);
+
+    if (data.empty())
+    {
+        lua_pushnil(state);
+    }
+    else
+    {
+        lua_createtable(state, data.size(), 0);
+        for (size_t i = 0; i < data.size(); ++i)
+        {
+            read_persistent(state, data[i], true);
+            lua_rawseti(state, -2, i+1);
+        }
+    }
+
+    return 1;
+}
+
+static int dfhack_persistent_save(lua_State *state)
+{
+    lua_settop(state, 2);
+    luaL_checktype(state, 1, LUA_TTABLE);
+    bool add = lua_toboolean(state, 2);
+
+    lua_getfield(state, 1, "key");
+    const char *str = lua_tostring(state, -1);
+    if (!str)
+        luaL_argerror(state, 1, "no key field");
+
+    lua_settop(state, 1);
+
+    PersistentDataItem ref;
+    bool added = false;
+
+    if (add)
+    {
+        ref = Core::getInstance().getWorld()->AddPersistentData(str);
+        added = true;
+    }
+    else if (lua_getmetatable(state, 1))
+    {
+        if (!lua_rawequal(state, -1, lua_upvalueindex(1)))
+            return luaL_argerror(state, 1, "invalid table type");
+        lua_pop(state, 1);
+
+        ref = persistent_by_struct(state, 1);
+    }
+    else
+    {
+        ref = Core::getInstance().getWorld()->GetPersistentData(str);
+    }
+
+    if (!ref.isValid())
+    {
+        ref = Core::getInstance().getWorld()->AddPersistentData(str);
+        if (!ref.isValid())
+            luaL_error(state, "cannot create persistent entry");
+        added = true;
+    }
+
+    lua_getfield(state, 1, "value");
+    if (const char *str = lua_tostring(state, -1))
+        ref.val() = str;
+    lua_pop(state, 1);
+
+    lua_getfield(state, 1, "ints");
+    if (lua_istable(state, -1))
+    {
+        for (int i = 0; i < PersistentDataItem::NumInts; i++)
+        {
+            lua_rawgeti(state, -1, i+1);
+            if (lua_isnumber(state, -1))
+                ref.ival(i) = lua_tointeger(state, -1);
+            lua_pop(state, 1);
+        }
+    }
+    lua_pop(state, 1);
+
+    read_persistent(state, ref, false);
+    lua_pushboolean(state, added);
+    return 2;
+}
+
+static const luaL_Reg dfhack_persistent_funcs[] = {
+    { "get", dfhack_persistent_get },
+    { "delete", dfhack_persistent_delete },
+    { "get_all", dfhack_persistent_get_all },
+    { "save", dfhack_persistent_save },
+    { NULL, NULL }
+};
+
+static void OpenPersistent(lua_State *state)
+{
+    luaL_getsubtable(state, lua_gettop(state), "persistent");
+
+    lua_dup(state);
+    luaL_setfuncs(state, dfhack_persistent_funcs, 1);
+
+    lua_dup(state);
+    lua_setfield(state, -2, "__index");
+
+    lua_pop(state, 1);
+}
+
 lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
 {
     if (!state)
@@ -354,6 +600,9 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
     // Create and initialize the dfhack global
     lua_newtable(state);
     luaL_setfuncs(state, dfhack_funcs, 0);
+
+    OpenPersistent(state);
+
     lua_setglobal(state, "dfhack");
 
     // load dfhack.lua

library/LuaTypes.cpp

@@ -1032,7 +1032,7 @@ static int meta_call_function(lua_State *state)
 static void AddMethodWrapper(lua_State *state, int meta_idx, int field_idx,
                              const char *name, function_identity_base *fun)
 {
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushvalue(state, meta_idx);
     lua_pushfstring(state, "%s()", name);
     lua_pushlightuserdata(state, fun);
@@ -1331,7 +1331,7 @@ void LuaWrapper::push_adhoc_pointer(lua_State *state, void *ptr, type_identity *
         return;
     }
 
-    LookupInTable(state, target, DFHACK_PTR_IDTABLE_NAME);
+    LookupInTable(state, target, &DFHACK_PTR_IDTABLE_TOKEN);
 
     type_identity *id = (type_identity*)lua_touserdata(state, -1);
     lua_pop(state, 1);
@@ -1347,7 +1347,7 @@ void LuaWrapper::push_adhoc_pointer(lua_State *state, void *ptr, type_identity *
         void *newobj = lua_newuserdata(state, sizeof(pointer_identity));
         id = new (newobj) pointer_identity(target);
 
-        SaveInTable(state, target, DFHACK_PTR_IDTABLE_NAME);
+        SaveInTable(state, target, &DFHACK_PTR_IDTABLE_TOKEN);
         lua_pop(state, 1);
     }
 

library/LuaWrapper.cpp

@@ -93,9 +93,9 @@ static void freeze_table(lua_State *state, bool leave_metatable = false, const c
         lua_pop(state, 1);
 }
 
-static void LookupInTable(lua_State *state, const char *tname)
+static void LookupInTable(lua_State *state, LuaToken *tname)
 {
-    lua_getfield(state, LUA_REGISTRYINDEX, tname);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, tname);
     lua_swap(state);
     lua_rawget(state, -2);
     lua_remove(state, -2);
@@ -115,12 +115,12 @@ bool LuaWrapper::LookupTypeInfo(lua_State *state, bool in_method)
     }
     else
     {
-        LookupInTable(state, DFHACK_TYPETABLE_NAME);
+        LookupInTable(state, &DFHACK_TYPETABLE_TOKEN);
     }
 
     // stack: [info]
 
-    if (!lua_islightuserdata(state, -1))
+    if (lua_isnil(state, -1))
     {
         lua_pop(state, 1);
         return false;
@@ -129,22 +129,20 @@ bool LuaWrapper::LookupTypeInfo(lua_State *state, bool in_method)
         return true;
 }
 
-void LuaWrapper::LookupInTable(lua_State *state, void *id, const char *tname)
+void LuaWrapper::LookupInTable(lua_State *state, void *id, LuaToken *tname)
 {
-    lua_getfield(state, LUA_REGISTRYINDEX, tname);
-    lua_pushlightuserdata(state, id);
-    lua_rawget(state, -2);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, tname);
+    lua_rawgetp(state, -1, id);
     lua_remove(state, -2);
 }
 
-void LuaWrapper::SaveInTable(lua_State *state, void *node, const char *tname)
+void LuaWrapper::SaveInTable(lua_State *state, void *node, LuaToken *tname)
 {
     // stack: [info]
-    lua_getfield(state, LUA_REGISTRYINDEX, tname);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, tname);
 
-    lua_pushlightuserdata(state, node);
-    lua_pushvalue(state, -3);
-    lua_rawset(state, -3);
+    lua_pushvalue(state, -2);
+    lua_rawsetp(state, -2, node);
 
     lua_pushvalue(state, -2);
     lua_pushlightuserdata(state, node);
@@ -156,7 +154,7 @@ void LuaWrapper::SaveInTable(lua_State *state, void *node, const char *tname)
 
 void LuaWrapper::SaveTypeInfo(lua_State *state, void *node)
 {
-    SaveInTable(state, node, DFHACK_TYPETABLE_NAME);
+    SaveInTable(state, node, &DFHACK_TYPETABLE_TOKEN);
 }
 
 static void BuildTypeMetatable(lua_State *state, type_identity *type);
@@ -311,7 +309,7 @@ bool LuaWrapper::is_type_compatible(lua_State *state, type_identity *type1, int
 static bool is_type_compatible(lua_State *state, type_identity *type1, int meta1,
                                int meta2, bool exact_equal)
 {
-    lua_getfield(state, meta2, "_identity");
+    lua_rawgetp(state, meta2, &DFHACK_IDENTITY_FIELD_TOKEN);
     auto type2 = (type_identity*)lua_touserdata(state, -1);
     lua_pop(state, 1);
 
@@ -323,7 +321,7 @@ static bool is_type_compatible(lua_State *state, int meta1, int meta2, bool exac
     if (lua_rawequal(state, meta1, meta2))
         return true;
 
-    lua_getfield(state, meta1, "_identity");
+    lua_rawgetp(state, meta1, &DFHACK_IDENTITY_FIELD_TOKEN);
     auto type1 = (type_identity*)lua_touserdata(state, -1);
     lua_pop(state, 1);
 
@@ -396,7 +394,7 @@ static bool is_valid_metatable(lua_State *state, int objidx, int metaidx)
     else
     {
         lua_pushvalue(state, objidx);
-        LookupInTable(state, DFHACK_TYPEID_TABLE_NAME);
+        LookupInTable(state, &DFHACK_TYPEID_TABLE_TOKEN);
     }
 
     bool ok = !lua_isnil(state, -1);
@@ -421,7 +419,7 @@ type_identity *LuaWrapper::get_object_identity(lua_State *state, int objidx,
         luaL_error(state, "Invalid object metatable in %s", ctx);
 
     // Extract identity from metatable
-    lua_getfield(state, -1, "_identity");
+    lua_rawgetp(state, -1, &DFHACK_IDENTITY_FIELD_TOKEN);
 
     type_identity *id = (type_identity*)lua_touserdata(state, -1);
     if (!id)
@@ -431,9 +429,9 @@ type_identity *LuaWrapper::get_object_identity(lua_State *state, int objidx,
     return id;
 }
 
-static void check_type_compatible(lua_State *state, int obj1, int obj2,
+static bool check_type_compatible(lua_State *state, int obj1, int obj2,
                                   type_identity **type1, type_identity **type2,
-                                  const char *ctx, bool allow_type, bool exact)
+                                  const char *ctx, bool allow_type, bool exact, bool error = true)
 {
     int base = lua_gettop(state);
 
@@ -442,6 +440,12 @@ static void check_type_compatible(lua_State *state, int obj1, int obj2,
 
     if (!is_type_compatible(state, *type1, base+1, *type2, base+2, exact))
     {
+        if (!error)
+        {
+            lua_pop(state, 2);
+            return false;
+        }
+
         lua_getfield(state, base+1, "__metatable");
         const char *cname1 = lua_tostring(state, -1);
         lua_getfield(state, base+2, "__metatable");
@@ -451,6 +455,7 @@ static void check_type_compatible(lua_State *state, int obj1, int obj2,
     }
 
     lua_pop(state, 2);
+    return true;
 }
 
 /**
@@ -741,6 +746,30 @@ static int meta_assign(lua_State *state)
     return 0;
 }
 
+/**
+ * Method: check if the two objects are assignment-compatible.
+ */
+static int meta_is_instance(lua_State *state)
+{
+    int argc = lua_gettop(state);
+
+    if (argc != 2)
+        luaL_error(state, "Usage: type:is_instance(obj) or df.is_instance(type,obj)");
+
+    // If garbage as second argument, return nil
+    if (!lua_istable(state, 2) && (!lua_isuserdata(state,2) || lua_islightuserdata(state, 2)))
+    {
+        lua_pushnil(state);
+        return 1;
+    }
+
+    type_identity *id1, *id2;
+    bool ok = check_type_compatible(state, 1, 2, &id1, &id2, "df.is_instance()", true, false, false);
+
+    lua_pushboolean(state, ok);
+    return 1;
+}
+
 /**
  * Method: deallocation for DF object references.
  */
@@ -865,9 +894,9 @@ void LuaWrapper::MakeMetatable(lua_State *state, type_identity *type, const char
     lua_setfield(state, base+1, "__metatable");
 
     lua_pushlightuserdata(state, type);
-    lua_setfield(state, base+1, "_identity");
+    lua_rawsetp(state, base+1, &DFHACK_IDENTITY_FIELD_TOKEN);
 
-    LookupInTable(state, type, DFHACK_TYPEID_TABLE_NAME);
+    LookupInTable(state, type, &DFHACK_TYPEID_TABLE_TOKEN);
     if (lua_isnil(state, -1))
     {
         // Copy the string from __metatable if no real type
@@ -900,7 +929,7 @@ void LuaWrapper::SetPtrMethods(lua_State *state, int meta_idx, int read_idx)
     lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_COMPARE_NAME);
     lua_setfield(state, meta_idx, "__eq");
 
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushvalue(state, meta_idx);
     lua_pushcclosure(state, meta_ptr_tostring, 2);
     lua_setfield(state, meta_idx, "__tostring");
@@ -952,7 +981,7 @@ void LuaWrapper::SetPairsMethod(lua_State *state, int meta_idx, const char *name
 void LuaWrapper::PushStructMethod(lua_State *state, int meta_idx, int ftable_idx,
                                   lua_CFunction function)
 {
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushvalue(state, meta_idx);
     lua_pushvalue(state, ftable_idx);
     lua_pushcclosure(state, function, 3);
@@ -975,7 +1004,7 @@ void LuaWrapper::PushContainerMethod(lua_State *state, int meta_idx, int ftable_
                                      lua_CFunction function,
                                      type_identity *container, type_identity *item, int count)
 {
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushvalue(state, meta_idx);
     lua_pushvalue(state, ftable_idx);
 
@@ -1008,10 +1037,10 @@ void LuaWrapper::AttachEnumKeys(lua_State *state, int meta_idx, int ftable_idx,
 {
     EnableMetaField(state, ftable_idx, "_enum");
 
-    LookupInTable(state, ienum, DFHACK_TYPEID_TABLE_NAME);
+    LookupInTable(state, ienum, &DFHACK_TYPEID_TABLE_TOKEN);
     lua_setfield(state, meta_idx, "_enum");
 
-    LookupInTable(state, ienum, DFHACK_ENUM_TABLE_NAME);
+    LookupInTable(state, ienum, &DFHACK_ENUM_TABLE_TOKEN);
 
     if (!lua_isnil(state, -1))
     {
@@ -1024,7 +1053,7 @@ void LuaWrapper::AttachEnumKeys(lua_State *state, int meta_idx, int ftable_idx,
     else
     {
         lua_pop(state, 1);
-        lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_EMPTY_TABLE_NAME);
+        lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_EMPTY_TABLE_TOKEN);
     }
 
     lua_dup(state);
@@ -1063,12 +1092,7 @@ static void FillEnumKeys(lua_State *state, int ftable, enum_identity *eid)
 
     // Create a new table attached to ftable as __index
     lua_newtable(state);
-
-    lua_dup(state);
-    lua_setmetatable(state, ftable);
-
     int base = lua_gettop(state);
-
     lua_newtable(state);
 
     // For enums, set mapping between keys and values
@@ -1087,12 +1111,12 @@ static void FillEnumKeys(lua_State *state, int ftable, enum_identity *eid)
         lua_setfield(state, base+1, "_last_item");
     }
 
-    SaveInTable(state, eid, DFHACK_ENUM_TABLE_NAME);
+    SaveInTable(state, eid, &DFHACK_ENUM_TABLE_TOKEN);
 
     // Add an attribute table if any
     if (eid->getAttrs())
     {
-        lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+        lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
         lua_pushlightuserdata(state, eid);
         lua_pushvalue(state, base+1);
         lua_pushcclosure(state, meta_enum_attr_index, 3);
@@ -1102,28 +1126,36 @@ static void FillEnumKeys(lua_State *state, int ftable, enum_identity *eid)
     }
 
     lua_setfield(state, base, "__index");
-    lua_pop(state, 1);
+    lua_setmetatable(state, ftable);
 }
 
 static void FillBitfieldKeys(lua_State *state, int ftable, bitfield_identity *eid)
 {
+    // Create a new table attached to ftable as __index
+    lua_newtable(state);
+    int base = lua_gettop(state);
+    lua_newtable(state);
+
     auto bits = eid->getBits();
 
     for (int i = 0; i < eid->getNumBits(); i++)
     {
         if (bits[i].name)
-            AssociateId(state, ftable, i, bits[i].name);
+            AssociateId(state, base+1, i, bits[i].name);
         if (bits[i].size > 1)
             i += bits[i].size-1;
     }
 
     lua_pushinteger(state, 0);
-    lua_setfield(state, ftable, "_first_item");
+    lua_setfield(state, base+1, "_first_item");
 
     lua_pushinteger(state, eid->getNumBits()-1);
-    lua_setfield(state, ftable, "_last_item");
+    lua_setfield(state, base+1, "_last_item");
+
+    SaveInTable(state, eid, &DFHACK_ENUM_TABLE_TOKEN);
 
-    SaveInTable(state, eid, DFHACK_ENUM_TABLE_NAME);
+    lua_setfield(state, base, "__index");
+    lua_setmetatable(state, ftable);
 }
 
 static void RenderType(lua_State *state, compound_identity *node)
@@ -1137,7 +1169,7 @@ static void RenderType(lua_State *state, compound_identity *node)
     if (!lua_checkstack(state, 20))
         return;
 
-    SaveInTable(state, node, DFHACK_TYPEID_TABLE_NAME);
+    SaveInTable(state, node, &DFHACK_TYPEID_TABLE_TOKEN);
 
     // metatable
     lua_newtable(state);
@@ -1152,7 +1184,7 @@ static void RenderType(lua_State *state, compound_identity *node)
     lua_setfield(state, base+2, "__tostring");
 
     lua_pushlightuserdata(state, node);
-    lua_setfield(state, base+2, "_identity");
+    lua_rawsetp(state, base+2, &DFHACK_IDENTITY_FIELD_TOKEN);
 
     // inner table
     lua_newtable(state);
@@ -1218,6 +1250,9 @@ static void RenderType(lua_State *state, compound_identity *node)
     lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_NEW_NAME);
     lua_setfield(state, ftable, "new");
 
+    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_IS_INSTANCE_NAME);
+    lua_setfield(state, ftable, "is_instance");
+
     lua_pop(state, 2);
 }
 
@@ -1237,16 +1272,16 @@ static int DoAttach(lua_State *state)
     int base = lua_gettop(state);
 
     lua_newtable(state);
-    lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_PTR_IDTABLE_NAME);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_PTR_IDTABLE_TOKEN);
 
     lua_newtable(state);
-    lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_TYPEID_TABLE_NAME);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPEID_TABLE_TOKEN);
 
     lua_newtable(state);
-    lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_ENUM_TABLE_NAME);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_ENUM_TABLE_TOKEN);
 
     lua_newtable(state);
-    lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_EMPTY_TABLE_NAME);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_EMPTY_TABLE_TOKEN);
 
     lua_pushcfunction(state, change_error);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_CHANGEERROR_NAME);
@@ -1257,23 +1292,27 @@ static int DoAttach(lua_State *state)
     lua_pushcfunction(state, meta_type_tostring);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_TYPE_TOSTRING_NAME);
 
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushcclosure(state, meta_sizeof, 1);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_SIZEOF_NAME);
 
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushcclosure(state, meta_displace, 1);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_DISPLACE_NAME);
 
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushcclosure(state, meta_new, 1);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_NEW_NAME);
 
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushcclosure(state, meta_assign, 1);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_ASSIGN_NAME);
 
-    lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_TYPETABLE_NAME);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
+    lua_pushcclosure(state, meta_is_instance, 1);
+    lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_IS_INSTANCE_NAME);
+
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
     lua_pushcclosure(state, meta_delete, 1);
     lua_setfield(state, LUA_REGISTRYINDEX, DFHACK_DELETE_NAME);
 
@@ -1294,6 +1333,8 @@ static int DoAttach(lua_State *state)
         lua_setfield(state, -2, "_displace");
         lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_ASSIGN_NAME);
         lua_setfield(state, -2, "assign");
+        lua_getfield(state, LUA_REGISTRYINDEX, DFHACK_IS_INSTANCE_NAME);
+        lua_setfield(state, -2, "is_instance");
 
         lua_pushlightuserdata(state, NULL);
         lua_setfield(state, -2, "NULL");
@@ -1312,11 +1353,29 @@ static int DoAttach(lua_State *state)
  */
 void LuaWrapper::AttachDFGlobals(lua_State *state)
 {
-    if (luaL_newmetatable(state, DFHACK_TYPETABLE_NAME))
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
+
+    if (lua_isnil(state, -1))
     {
+        lua_pop(state, 1);
+        lua_newtable(state);
+        lua_dup(state);
+        lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
+
         luaL_requiref(state, "df", DoAttach, 1);
         lua_pop(state, 1);
     }
 
     lua_pop(state, 1);
 }
+
+namespace DFHack { namespace LuaWrapper {
+    struct LuaToken { int reserved; };
+
+    LuaToken DFHACK_IDENTITY_FIELD_TOKEN;
+    LuaToken DFHACK_TYPETABLE_TOKEN;
+    LuaToken DFHACK_TYPEID_TABLE_TOKEN;
+    LuaToken DFHACK_ENUM_TABLE_TOKEN;
+    LuaToken DFHACK_PTR_IDTABLE_TOKEN;
+    LuaToken DFHACK_EMPTY_TABLE_TOKEN;
+}}

library/include/LuaWrapper.h

@@ -39,26 +39,32 @@ distribution.
  */
 
 namespace DFHack { namespace LuaWrapper {
+    struct LuaToken;
 
-/*
- * Registry name: hash of type metatables <-> type identities.
+    /**
+     * Metatable pkey: type identity of the object
      */
-#define DFHACK_TYPETABLE_NAME "DFHack::DFTypes"
+    extern LuaToken DFHACK_IDENTITY_FIELD_TOKEN;
 
-/*
- * Registry name: hash of type identity -> node in df.etc...
+    /**
+     * Registry pkey: hash of type metatables <-> type identities.
      */
-#define DFHACK_TYPEID_TABLE_NAME "DFHack::DFTypeIds"
+    extern LuaToken DFHACK_TYPETABLE_TOKEN;
 
-/*
- * Registry name: hash of enum/bitfield identity -> index lookup table
+    /**
+     * Registry pkey: hash of type identity -> node in df.etc...
      */
-#define DFHACK_ENUM_TABLE_NAME "DFHack::DFEnums"
+    extern LuaToken DFHACK_TYPEID_TABLE_TOKEN;
 
-/*
- * Registry name: hash of pointer target identity <-> adhoc pointer identity userdata.
+    /**
+     * Registry pkey: hash of enum/bitfield identity -> index lookup table
+     */
+    extern LuaToken DFHACK_ENUM_TABLE_TOKEN;
+
+    /**
+     * Registry pkey: hash of pointer target identity <-> adhoc pointer identity userdata.
      */
-#define DFHACK_PTR_IDTABLE_NAME "DFHack::PtrDFTypes"
+    extern LuaToken DFHACK_PTR_IDTABLE_TOKEN;
 
 // Function registry names
 #define DFHACK_CHANGEERROR_NAME "DFHack::ChangeError"
@@ -68,8 +74,10 @@ namespace DFHack { namespace LuaWrapper {
 #define DFHACK_DISPLACE_NAME "DFHack::Displace"
 #define DFHACK_NEW_NAME "DFHack::New"
 #define DFHACK_ASSIGN_NAME "DFHack::Assign"
+#define DFHACK_IS_INSTANCE_NAME "DFHack::IsInstance"
 #define DFHACK_DELETE_NAME "DFHack::Delete"
-#define DFHACK_EMPTY_TABLE_NAME "DFHack::EmptyTable"
+
+    extern LuaToken DFHACK_EMPTY_TABLE_TOKEN;
 
 /*
  * Upvalue: contents of DFHACK_TYPETABLE_NAME
@@ -157,8 +165,8 @@ namespace DFHack { namespace LuaWrapper {
                                        const char *ctx, bool allow_type = false,
                                        bool keep_metatable = false);
 
-    void LookupInTable(lua_State *state, void *id, const char *tname);
-    void SaveInTable(lua_State *state, void *node, const char *tname);
+    void LookupInTable(lua_State *state, void *id, LuaToken *tname);
+    void SaveInTable(lua_State *state, void *node, LuaToken *tname);
     void SaveTypeInfo(lua_State *state, void *node);
 
     void AssociateId(lua_State *state, int table, int val, const char *name);

library/include/df/custom/coord.methods.inc

@@ -33,7 +33,7 @@ coord operator-(const coord &other) const
 
 coord operator/(int number) const
 {
-    return coord(x/number, y/number, z);
+    return coord((x < 0 ? x - number : x)/number, (y < 0 ? y - number : y)/number, z);
 }
 coord operator*(int number) const
 {
@@ -41,7 +41,7 @@ coord operator*(int number) const
 }
 coord operator%(int number) const
 {
-    return coord(x%number, y%number, z);
+    return coord((x+number)%number, (y+number)%number, z);
 }
 
 coord operator-(int number) const

library/include/df/custom/coord2d.methods.inc

@@ -29,7 +29,7 @@ coord2d operator-(const coord2d &other) const
 
 coord2d operator/(int number) const
 {
-    return coord2d(x/number, y/number);
+    return coord2d((x < 0 ? x - number : x)/number, (y < 0 ? y - number : y)/number);
 }
 coord2d operator*(int number) const
 {
@@ -37,5 +37,5 @@ coord2d operator*(int number) const
 }
 coord2d operator%(int number) const
 {
-    return coord2d(x%number, y%number);
+    return coord2d((x+number)%number, (y+number)%number);
 }

library/include/modules/World.h

@@ -93,6 +93,7 @@ namespace DFHack
         static const int NumInts = 7;
 
         bool isValid() { return id != 0; }
+        int entry_id() { return -id; }
 
         const std::string &key() { return key_value; }
 
@@ -137,12 +138,18 @@ namespace DFHack
         // This ensures that the values are stored in save games.
         PersistentDataItem AddPersistentData(const std::string &key);
         PersistentDataItem GetPersistentData(const std::string &key);
-        void GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key);
-        void DeletePersistentData(const PersistentDataItem &item);
+        PersistentDataItem GetPersistentData(int entry_id);
+        void GetPersistentData(std::vector<PersistentDataItem> *vec,
+                               const std::string &key, bool prefix = false);
+        bool DeletePersistentData(const PersistentDataItem &item);
+
+        void ClearPersistentCache();
 
     private:
         struct Private;
         Private *d;
+
+        bool BuildPersistentCache();
     };
 }
 #endif

library/lua/dfhack.lua

@@ -1,5 +1,41 @@
 -- Common startup file for all dfhack plugins with lua support
+-- The global dfhack table is already created by C++ init code.
 
--- The global dfhack table is already created by C++ init
--- code. Feed it back to the require() mechanism.
+function mkmodule(module,env)
+    local pkg = package.loaded[module]
+    if pkg == nil then
+        pkg = {}
+    else
+        if type(pkg) ~= 'table' then
+            error("Not a table in package.loaded["..module.."]")
+        end
+    end
+    setmetatable(pkg, { __index = (env or _G) })
+    return pkg
+end
+
+function reload(module)
+    if type(package.loaded[module]) ~= 'table' then
+        error("Module not loaded: "..module)
+    end
+    local path,err = package.searchpath(module,package.path)
+    if not path then
+        error(err)
+    end
+    dofile(path)
+end
+
+function printall(table)
+    if table == nil then return end
+    for k,v in pairs(table) do
+        print(k,"   = "..tostring(v))
+    end
+end
+
+function dfhack.persistent:__tostring()
+    return "<persistent "..self.entry_id..":"..self.key.."=\""
+           ..self.value.."\":"..table.concat(self.ints,",")..">"
+end
+
+-- Feed the table back to the require() mechanism.
 return dfhack

library/modules/World.cpp

@@ -58,6 +58,7 @@ struct World::Private
     Private()
     {
         Inited = PauseInited = StartedWeather = StartedMode = false;
+        next_persistent_id = 0;
     }
     bool Inited;
 
@@ -72,9 +73,14 @@ struct World::Private
     void * controlmode_offset;
     void * controlmodecopy_offset;
 
+    int next_persistent_id;
+    std::multimap<std::string, int> persistent_index;
+
     Process * owner;
 };
 
+typedef std::pair<std::string, int> T_persistent_item;
+
 World::World()
 {
     Core & c = Core::getInstance();
@@ -217,65 +223,144 @@ static PersistentDataItem dataFromHFig(df::historical_figure *hfig)
     return PersistentDataItem(hfig->id, hfig->name.first_name, &hfig->name.nickname, hfig->name.words);
 }
 
-PersistentDataItem World::AddPersistentData(const std::string &key)
+void World::ClearPersistentCache()
+{
+    d->next_persistent_id = 0;
+    d->persistent_index.clear();
+}
+
+bool World::BuildPersistentCache()
 {
+    if (d->next_persistent_id)
+        return true;
+    if (!Core::getInstance().isWorldLoaded())
+        return false;
+
     std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
 
-    int new_id = -100;
-    if (hfvec.size() > 0 && hfvec[0]->id <= new_id)
-        new_id = hfvec[0]->id-1;
+    // Determine the next entry id as min(-100, lowest_id-1)
+    d->next_persistent_id = -100;
+
+    if (hfvec.size() > 0 && hfvec[0]->id <= -100)
+        d->next_persistent_id = hfvec[0]->id-1;
+
+    // Add the entries to the lookup table
+    d->persistent_index.clear();
+
+    for (size_t i = 0; i < hfvec.size() && hfvec[i]->id <= -100; i++)
+    {
+        if (!hfvec[i]->name.has_name || hfvec[i]->name.first_name.empty())
+            continue;
+
+        d->persistent_index.insert(T_persistent_item(hfvec[i]->name.first_name, -hfvec[i]->id));
+    }
+}
+
+PersistentDataItem World::AddPersistentData(const std::string &key)
+{
+    if (!BuildPersistentCache() || key.empty())
+        return PersistentDataItem();
+
+    std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
 
     df::historical_figure *hfig = new df::historical_figure();
-    hfig->id = new_id;
+    hfig->id = d->next_persistent_id--;
     hfig->name.has_name = true;
     hfig->name.first_name = key;
     memset(hfig->name.words, 0xFF, sizeof(hfig->name.words));
 
     hfvec.insert(hfvec.begin(), hfig);
+
+    d->persistent_index.insert(T_persistent_item(key, -hfig->id));
+
     return dataFromHFig(hfig);
 }
 
 PersistentDataItem World::GetPersistentData(const std::string &key)
 {
-    std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
-    for (size_t i = 0; i < hfvec.size(); i++)
-    {
-        df::historical_figure *hfig = hfvec[i];
+    if (!BuildPersistentCache())
+        return PersistentDataItem();
 
-        if (hfig->id >= 0)
-            break;
+    auto it = d->persistent_index.find(key);
+    if (it != d->persistent_index.end())
+        return GetPersistentData(it->second);
 
-        if (hfig->name.has_name && hfig->name.first_name == key)
-            return dataFromHFig(hfig);
+    return PersistentDataItem();
 }
 
+PersistentDataItem World::GetPersistentData(int entry_id)
+{
+    if (entry_id < 100)
+        return PersistentDataItem();
+
+    auto hfig = df::historical_figure::find(-entry_id);
+    if (hfig && hfig->name.has_name)
+        return dataFromHFig(hfig);
+
     return PersistentDataItem();
 }
 
-void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key)
+void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key, bool prefix)
 {
-    std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
-    for (size_t i = 0; i < hfvec.size(); i++)
+    if (!BuildPersistentCache())
+        return;
+
+    auto eqrange = d->persistent_index.equal_range(key);
+
+    if (prefix)
+    {
+        if (key.empty())
+        {
+            eqrange.first = d->persistent_index.begin();
+            eqrange.second = d->persistent_index.end();
+        }
+        else
         {
-        df::historical_figure *hfig = hfvec[i];
+            std::string bound = key;
+            if (bound[bound.size()-1] != '/')
+                bound += "/";
+            eqrange.first = d->persistent_index.lower_bound(bound);
 
-        if (hfig->id >= 0)
-            break;
+            bound[bound.size()-1]++;
+            eqrange.second = d->persistent_index.lower_bound(bound);
+        }
+    }
 
-        if (hfig->name.has_name && hfig->name.first_name == key)
+    for (auto it = eqrange.first; it != eqrange.second; ++it)
+    {
+        auto hfig = df::historical_figure::find(-it->second);
+        if (hfig && hfig->name.has_name)
             vec->push_back(dataFromHFig(hfig));
     }
 }
 
-void World::DeletePersistentData(const PersistentDataItem &item)
+bool World::DeletePersistentData(const PersistentDataItem &item)
 {
     if (item.id > -100)
-        return;
+        return false;
+    if (!BuildPersistentCache())
+        return false;
 
     std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
+
+    auto eqrange = d->persistent_index.equal_range(item.key_value);
+
+    for (auto it = eqrange.first; it != eqrange.second; ++it)
+    {
+        if (it->second != -item.id)
+            continue;
+
+        d->persistent_index.erase(it);
+
         int idx = binsearch_index(hfvec, item.id);
+
         if (idx >= 0) {
             delete hfvec[idx];
             hfvec.erase(hfvec.begin()+idx);
         }
+
+        return true;
+    }
+
+    return false;
 }

library/xml

@@ -1 +1 @@
-Subproject commit 767e2f679471db8a77e923bc5572d859f677e08a
+Subproject commit b630809b25a3e3b541e40f0cb145b6f5bd3b7316

plugins/Dfusion/luafiles/friendship/init.lua

@@ -14,9 +14,9 @@ function analyzeF(off)
 		return pos
 	end
 end
-function minEx(...)
-	local imin=arg[1]
-	for _,v in ipairs(arg) do
+function minEx(list)
+	local imin=list[1]
+	for _,v in ipairs(list) do
 		if imin> v and v~=0 then
 			imin=v
 		end

plugins/Dfusion/luafiles/friendship/patch.lua

@@ -11,8 +11,7 @@ function friendship_in.patch()
 		pos3=offsets.find(pos+7,0xa1,DWORD_,crace) -- mov eax,[ptr]
 		pos4=offsets.find(pos+7,0x66,0x8b,ANYBYTE,DWORD_,crace) -- mov ANYREG,[ptr]
 		--pos5=offsets.find(pos+7,0x66,0x8b,0x15,DWORD_,crace) -- mov dx,[ptr]
-
-		pos=minEx(pos1,pos2,pos3,pos4)
+		pos=minEx{pos1,pos2,pos3,pos4}
 		if pos ~=0 then 
 			hits[i]=pos
 			i=i+1
@@ -28,7 +27,7 @@ function friendship_in.patch()
 			--TODO read offset from memory.xml
 			pos1=offsets.find(myp,0x39,ANYBYTE,0x8c,00,00,00) -- compare [reg+08c] (creature race) with any reg 
 			pos2=offsets.find(myp,0x3b,ANYBYTE,0x8c,00,00,00) -- compare  any reg with [reg+08c] (creature race)
-			pos=minEx(pos1,pos2)
+			pos=minEx{pos1,pos2}
 			if pos ~=0 then
 				
 				if(pos-p>250) then

plugins/Dfusion/luafiles/init.lua

@@ -1,3 +1,7 @@
+Console.print = dfhack.print
+Console.println = dfhack.println
+Console.printerr = dfhack.printerr
+
 function err(msg) --make local maybe...
 	print(msg)
 	print(debug.traceback())
@@ -14,6 +18,10 @@ function dofile_silent(filename) --safer dofile, with traceback, no file not fou
 	f,perr=loadfile(filename)
 	if f~=nil then
 		return xpcall(f,err)
+	else
+		if(string.sub(perr,1,11)~="cannot open") then --ugly hack
+			print(perr)	
+		end
 	end
 end
 function loadall(t1) --loads all non interactive plugin parts, so that later they could be used