Improve performance of the persistent data api, and wrap it for lua.

Use an stl table for string keys to avoid linear cost of lookup.

This uncovered a bug in the new luaL_getsubtable function.

depends/lua/src/lauxlib.c

@@ -863,8 +863,8 @@ LUALIB_API int luaL_getsubtable (lua_State *L, int idx, const char *fname) {
   lua_getfield(L, idx, fname);
   if (lua_istable(L, -1)) return 1;  /* table already there */
   else {
-    idx = lua_absindex(L, idx);
     lua_pop(L, 1);  /* remove previous result */
+    idx = lua_absindex(L, idx);
     lua_newtable(L);
     lua_pushvalue(L, -1);  /* copy to be left at top */
     lua_setfield(L, idx, fname);  /* assign new table to field */

library/Core.cpp

@@ -879,31 +879,30 @@ int Core::Update()
             new_wdata = wdata;
         new_mapdata = df::global::world->map.block_index;
     }
+
     // if the world changes
     if (new_wdata != last_world_data_ptr)
     {
         // we check for map change too
         bool mapchange = new_mapdata != last_local_map_ptr;
+        last_world_data_ptr = new_wdata;
+        last_local_map_ptr = new_mapdata;
+
+        getWorld()->ClearPersistentCache();
+
         // and if the world is going away, we report the map change first
         if(!new_wdata && mapchange)
-        {
-            last_local_map_ptr = new_mapdata;
             plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
-        }
         // and if the world is appearing, we report map change after that
         plug_mgr->OnStateChange(out, new_wdata ? SC_WORLD_LOADED : SC_WORLD_UNLOADED);
         if(new_wdata && mapchange)
-        {
-            last_local_map_ptr = new_mapdata;
             plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
     }
-        // update tracking variable
-        last_world_data_ptr = new_wdata;
-    }
     // otherwise just check for map change...
     else if (new_mapdata != last_local_map_ptr)
     {
         last_local_map_ptr = new_mapdata;
+        getWorld()->ClearPersistentCache();
         plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
     }
 

library/LuaTools.cpp

@@ -36,6 +36,8 @@ distribution.
 #include "DataDefs.h"
 #include "DataIdentity.h"
 
+#include "modules/World.h"
+
 #include "LuaWrapper.h"
 #include "LuaTools.h"
 
@@ -339,6 +341,213 @@ static const luaL_Reg dfhack_funcs[] = {
     { NULL, NULL }
 };
 
+/*
+ * Per-world persistent configuration storage.
+ */
+
+static PersistentDataItem persistent_by_struct(lua_State *state, int idx)
+{
+    lua_getfield(state, idx, "entry_id");
+    int id = lua_tointeger(state, -1);
+    lua_pop(state, 1);
+
+    PersistentDataItem ref = Core::getInstance().getWorld()->GetPersistentData(id);
+
+    if (ref.isValid())
+    {
+        lua_getfield(state, idx, "key");
+        const char *str = lua_tostring(state, -1);
+        if (!str || str != ref.key())
+            luaL_argerror(state, idx, "inconsistent id and key");
+        lua_pop(state, 1);
+    }
+
+    return ref;
+}
+
+static int read_persistent(lua_State *state, PersistentDataItem ref, bool create)
+{
+    if (!ref.isValid())
+    {
+        lua_pushnil(state);
+        lua_pushstring(state, "entry not found");
+        return 2;
+    }
+
+    if (create)
+        lua_createtable(state, 0, 4);
+
+    lua_pushvalue(state, lua_upvalueindex(1));
+    lua_setmetatable(state, -2);
+
+    lua_pushinteger(state, ref.entry_id());
+    lua_setfield(state, -2, "entry_id");
+    lua_pushstring(state, ref.key().c_str());
+    lua_setfield(state, -2, "key");
+    lua_pushstring(state, ref.val().c_str());
+    lua_setfield(state, -2, "value");
+
+    lua_createtable(state, PersistentDataItem::NumInts, 0);
+    for (int i = 0; i < PersistentDataItem::NumInts; i++)
+    {
+        lua_pushinteger(state, ref.ival(i));
+        lua_rawseti(state, -2, i+1);
+    }
+    lua_setfield(state, -2, "ints");
+
+    return 1;
+}
+
+static PersistentDataItem get_persistent(lua_State *state)
+{
+    luaL_checkany(state, 1);
+
+    if (lua_istable(state, 1))
+    {
+        if (!lua_getmetatable(state, 1) ||
+            !lua_rawequal(state, -1, lua_upvalueindex(1)))
+            luaL_argerror(state, 1, "invalid table type");
+
+        lua_settop(state, 1);
+
+        return persistent_by_struct(state, 1);
+    }
+    else
+    {
+        const char *str = luaL_checkstring(state, 1);
+
+        return Core::getInstance().getWorld()->GetPersistentData(str);
+    }
+}
+
+static int dfhack_persistent_get(lua_State *state)
+{
+    auto ref = get_persistent(state);
+
+    return read_persistent(state, ref, !lua_istable(state, 1));
+}
+
+static int dfhack_persistent_delete(lua_State *state)
+{
+    auto ref = get_persistent(state);
+
+    bool ok = Core::getInstance().getWorld()->DeletePersistentData(ref);
+
+    lua_pushboolean(state, ok);
+    return 1;
+}
+
+static int dfhack_persistent_get_all(lua_State *state)
+{
+    const char *str = luaL_checkstring(state, 1);
+    bool prefix = (lua_gettop(state)>=2 ? lua_toboolean(state,2) : false);
+
+    std::vector<PersistentDataItem> data;
+    Core::getInstance().getWorld()->GetPersistentData(&data, str, prefix);
+
+    if (data.empty())
+    {
+        lua_pushnil(state);
+    }
+    else
+    {
+        lua_createtable(state, data.size(), 0);
+        for (size_t i = 0; i < data.size(); ++i)
+        {
+            read_persistent(state, data[i], true);
+            lua_rawseti(state, -2, i+1);
+        }
+    }
+
+    return 1;
+}
+
+static int dfhack_persistent_save(lua_State *state)
+{
+    lua_settop(state, 2);
+    luaL_checktype(state, 1, LUA_TTABLE);
+    bool add = lua_toboolean(state, 2);
+
+    lua_getfield(state, 1, "key");
+    const char *str = lua_tostring(state, -1);
+    if (!str)
+        luaL_argerror(state, 1, "no key field");
+
+    lua_settop(state, 1);
+
+    PersistentDataItem ref;
+    bool added = false;
+
+    if (add)
+    {
+        ref = Core::getInstance().getWorld()->AddPersistentData(str);
+        added = true;
+    }
+    else if (lua_getmetatable(state, 1))
+    {
+        if (!lua_rawequal(state, -1, lua_upvalueindex(1)))
+            return luaL_argerror(state, 1, "invalid table type");
+        lua_pop(state, 1);
+
+        ref = persistent_by_struct(state, 1);
+    }
+    else
+    {
+        ref = Core::getInstance().getWorld()->GetPersistentData(str);
+    }
+
+    if (!ref.isValid())
+    {
+        ref = Core::getInstance().getWorld()->AddPersistentData(str);
+        if (!ref.isValid())
+            luaL_error(state, "cannot create persistent entry");
+        added = true;
+    }
+
+    lua_getfield(state, 1, "value");
+    if (const char *str = lua_tostring(state, -1))
+        ref.val() = str;
+    lua_pop(state, 1);
+
+    lua_getfield(state, 1, "ints");
+    if (lua_istable(state, -1))
+    {
+        for (int i = 0; i < PersistentDataItem::NumInts; i++)
+        {
+            lua_rawgeti(state, -1, i+1);
+            if (lua_isnumber(state, -1))
+                ref.ival(i) = lua_tointeger(state, -1);
+            lua_pop(state, 1);
+        }
+    }
+    lua_pop(state, 1);
+
+    read_persistent(state, ref, false);
+    lua_pushboolean(state, added);
+    return 2;
+}
+
+static const luaL_Reg dfhack_persistent_funcs[] = {
+    { "get", dfhack_persistent_get },
+    { "delete", dfhack_persistent_delete },
+    { "get_all", dfhack_persistent_get_all },
+    { "save", dfhack_persistent_save },
+    { NULL, NULL }
+};
+
+static void OpenPersistent(lua_State *state)
+{
+    luaL_getsubtable(state, lua_gettop(state), "persistent");
+
+    lua_dup(state);
+    luaL_setfuncs(state, dfhack_persistent_funcs, 1);
+
+    lua_dup(state);
+    lua_setfield(state, -2, "__index");
+
+    lua_pop(state, 1);
+}
+
 lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
 {
     if (!state)
@@ -354,6 +563,9 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
     // Create and initialize the dfhack global
     lua_newtable(state);
     luaL_setfuncs(state, dfhack_funcs, 0);
+
+    OpenPersistent(state);
+
     lua_setglobal(state, "dfhack");
 
     // load dfhack.lua

library/include/modules/World.h

@@ -93,6 +93,7 @@ namespace DFHack
         static const int NumInts = 7;
 
         bool isValid() { return id != 0; }
+        int entry_id() { return -id; }
 
         const std::string &key() { return key_value; }
 
@@ -137,12 +138,18 @@ namespace DFHack
         // This ensures that the values are stored in save games.
         PersistentDataItem AddPersistentData(const std::string &key);
         PersistentDataItem GetPersistentData(const std::string &key);
-        void GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key);
+        PersistentDataItem GetPersistentData(int entry_id);
-        void DeletePersistentData(const PersistentDataItem &item);
+        void GetPersistentData(std::vector<PersistentDataItem> *vec,
+                               const std::string &key, bool prefix = false);
+        bool DeletePersistentData(const PersistentDataItem &item);
+
+        void ClearPersistentCache();
 
     private:
         struct Private;
         Private *d;
+
+        bool BuildPersistentCache();
     };
 }
 #endif

library/lua/dfhack.lua

@@ -25,5 +25,16 @@ function reload(module)
     dofile(path)
 end
 
+function printall(table)
+    for k,v in pairs(table) do
+        print(k,"   = "..tostring(v))
+    end
+end
+
+function dfhack.persistent:__tostring()
+    return "<persistent "..self.entry_id..":"..self.key.."=\""
+           ..self.value.."\":"..table.concat(self.ints,",")..">"
+end
+
 -- Feed the table back to the require() mechanism.
 return dfhack

library/modules/World.cpp

@@ -58,6 +58,7 @@ struct World::Private
     Private()
     {
         Inited = PauseInited = StartedWeather = StartedMode = false;
+        next_persistent_id = 0;
     }
     bool Inited;
 
@@ -72,9 +73,14 @@ struct World::Private
     void * controlmode_offset;
     void * controlmodecopy_offset;
 
+    int next_persistent_id;
+    std::multimap<std::string, int> persistent_index;
+
     Process * owner;
 };
 
+typedef std::pair<std::string, int> T_persistent_item;
+
 World::World()
 {
     Core & c = Core::getInstance();
@@ -217,65 +223,144 @@ static PersistentDataItem dataFromHFig(df::historical_figure *hfig)
     return PersistentDataItem(hfig->id, hfig->name.first_name, &hfig->name.nickname, hfig->name.words);
 }
 
-PersistentDataItem World::AddPersistentData(const std::string &key)
+void World::ClearPersistentCache()
+{
+    d->next_persistent_id = 0;
+    d->persistent_index.clear();
+}
+
+bool World::BuildPersistentCache()
 {
+    if (d->next_persistent_id)
+        return true;
+    if (!Core::getInstance().isWorldLoaded())
+        return false;
+
     std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
 
-    int new_id = -100;
+    // Determine the next entry id as min(-100, lowest_id-1)
-    if (hfvec.size() > 0 && hfvec[0]->id <= new_id)
+    d->next_persistent_id = -100;
-        new_id = hfvec[0]->id-1;
+
+    if (hfvec.size() > 0 && hfvec[0]->id <= -100)
+        d->next_persistent_id = hfvec[0]->id-1;
+
+    // Add the entries to the lookup table
+    d->persistent_index.clear();
+
+    for (size_t i = 0; i < hfvec.size() && hfvec[i]->id <= -100; i++)
+    {
+        if (!hfvec[i]->name.has_name || hfvec[i]->name.first_name.empty())
+            continue;
+
+        d->persistent_index.insert(T_persistent_item(hfvec[i]->name.first_name, -hfvec[i]->id));
+    }
+}
+
+PersistentDataItem World::AddPersistentData(const std::string &key)
+{
+    if (!BuildPersistentCache() || key.empty())
+        return PersistentDataItem();
+
+    std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
 
     df::historical_figure *hfig = new df::historical_figure();
-    hfig->id = new_id;
+    hfig->id = d->next_persistent_id--;
     hfig->name.has_name = true;
     hfig->name.first_name = key;
     memset(hfig->name.words, 0xFF, sizeof(hfig->name.words));
 
     hfvec.insert(hfvec.begin(), hfig);
+
+    d->persistent_index.insert(T_persistent_item(key, -hfig->id));
+
     return dataFromHFig(hfig);
 }
 
 PersistentDataItem World::GetPersistentData(const std::string &key)
 {
-    std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
+    if (!BuildPersistentCache())
-    for (size_t i = 0; i < hfvec.size(); i++)
+        return PersistentDataItem();
-    {
-        df::historical_figure *hfig = hfvec[i];
 
-        if (hfig->id >= 0)
+    auto it = d->persistent_index.find(key);
-            break;
+    if (it != d->persistent_index.end())
+        return GetPersistentData(it->second);
 
-        if (hfig->name.has_name && hfig->name.first_name == key)
+    return PersistentDataItem();
-            return dataFromHFig(hfig);
 }
 
+PersistentDataItem World::GetPersistentData(int entry_id)
+{
+    if (entry_id < 100)
+        return PersistentDataItem();
+
+    auto hfig = df::historical_figure::find(-entry_id);
+    if (hfig && hfig->name.has_name)
+        return dataFromHFig(hfig);
+
     return PersistentDataItem();
 }
 
-void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key)
+void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key, bool prefix)
 {
-    std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
+    if (!BuildPersistentCache())
-    for (size_t i = 0; i < hfvec.size(); i++)
+        return;
+
+    auto eqrange = d->persistent_index.equal_range(key);
+
+    if (prefix)
+    {
+        if (key.empty())
+        {
+            eqrange.first = d->persistent_index.begin();
+            eqrange.second = d->persistent_index.end();
+        }
+        else
         {
-        df::historical_figure *hfig = hfvec[i];
+            std::string bound = key;
+            if (bound[bound.size()-1] != '/')
+                bound += "/";
+            eqrange.first = d->persistent_index.lower_bound(bound);
 
-        if (hfig->id >= 0)
+            bound[bound.size()-1]++;
-            break;
+            eqrange.second = d->persistent_index.lower_bound(bound);
+        }
+    }
 
-        if (hfig->name.has_name && hfig->name.first_name == key)
+    for (auto it = eqrange.first; it != eqrange.second; ++it)
+    {
+        auto hfig = df::historical_figure::find(-it->second);
+        if (hfig && hfig->name.has_name)
             vec->push_back(dataFromHFig(hfig));
     }
 }
 
-void World::DeletePersistentData(const PersistentDataItem &item)
+bool World::DeletePersistentData(const PersistentDataItem &item)
 {
     if (item.id > -100)
-        return;
+        return false;
+    if (!BuildPersistentCache())
+        return false;
 
     std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
+
+    auto eqrange = d->persistent_index.equal_range(item.key_value);
+
+    for (auto it = eqrange.first; it != eqrange.second; ++it)
+    {
+        if (it->second != -item.id)
+            continue;
+
+        d->persistent_index.erase(it);
+
         int idx = binsearch_index(hfvec, item.id);
+
         if (idx >= 0) {
             delete hfvec[idx];
             hfvec.erase(hfvec.begin()+idx);
         }
+
+        return true;
+    }
+
+    return false;
 }