From afe4eba957dadf219ff4b4d2a3aa89b773f44fdd Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 1 Apr 2012 16:43:40 +0400 Subject: [PATCH] Improve performance of the persistent data api, and wrap it for lua. Use an stl table for string keys to avoid linear cost of lookup. This uncovered a bug in the new luaL_getsubtable function. --- depends/lua/src/lauxlib.c | 2 +- library/Core.cpp | 15 ++- library/LuaTools.cpp | 212 ++++++++++++++++++++++++++++++++ library/include/modules/World.h | 13 +- library/lua/dfhack.lua | 11 ++ library/modules/World.cpp | 141 ++++++++++++++++----- 6 files changed, 354 insertions(+), 40 deletions(-) diff --git a/depends/lua/src/lauxlib.c b/depends/lua/src/lauxlib.c index 0aa80fd94..c1b715f34 100644 --- a/depends/lua/src/lauxlib.c +++ b/depends/lua/src/lauxlib.c @@ -863,8 +863,8 @@ LUALIB_API int luaL_getsubtable (lua_State *L, int idx, const char *fname) { lua_getfield(L, idx, fname); if (lua_istable(L, -1)) return 1; /* table already there */ else { - idx = lua_absindex(L, idx); lua_pop(L, 1); /* remove previous result */ + idx = lua_absindex(L, idx); lua_newtable(L); lua_pushvalue(L, -1); /* copy to be left at top */ lua_setfield(L, idx, fname); /* assign new table to field */ diff --git a/library/Core.cpp b/library/Core.cpp index 0a7cbefd4..ffc174721 100644 --- a/library/Core.cpp +++ b/library/Core.cpp @@ -879,31 +879,30 @@ int Core::Update() new_wdata = wdata; new_mapdata = df::global::world->map.block_index; } + // if the world changes if (new_wdata != last_world_data_ptr) { // we check for map change too bool mapchange = new_mapdata != last_local_map_ptr; + last_world_data_ptr = new_wdata; + last_local_map_ptr = new_mapdata; + + getWorld()->ClearPersistentCache(); + // and if the world is going away, we report the map change first if(!new_wdata && mapchange) - { - last_local_map_ptr = new_mapdata; plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); - } // and if the world is appearing, we report map change after that plug_mgr->OnStateChange(out, new_wdata ? SC_WORLD_LOADED : SC_WORLD_UNLOADED); if(new_wdata && mapchange) - { - last_local_map_ptr = new_mapdata; plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); - } - // update tracking variable - last_world_data_ptr = new_wdata; } // otherwise just check for map change... else if (new_mapdata != last_local_map_ptr) { last_local_map_ptr = new_mapdata; + getWorld()->ClearPersistentCache(); plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); } diff --git a/library/LuaTools.cpp b/library/LuaTools.cpp index 68cd8aa33..8b3fa0377 100644 --- a/library/LuaTools.cpp +++ b/library/LuaTools.cpp @@ -36,6 +36,8 @@ distribution. #include "DataDefs.h" #include "DataIdentity.h" +#include "modules/World.h" + #include "LuaWrapper.h" #include "LuaTools.h" @@ -339,6 +341,213 @@ static const luaL_Reg dfhack_funcs[] = { { NULL, NULL } }; +/* + * Per-world persistent configuration storage. + */ + +static PersistentDataItem persistent_by_struct(lua_State *state, int idx) +{ + lua_getfield(state, idx, "entry_id"); + int id = lua_tointeger(state, -1); + lua_pop(state, 1); + + PersistentDataItem ref = Core::getInstance().getWorld()->GetPersistentData(id); + + if (ref.isValid()) + { + lua_getfield(state, idx, "key"); + const char *str = lua_tostring(state, -1); + if (!str || str != ref.key()) + luaL_argerror(state, idx, "inconsistent id and key"); + lua_pop(state, 1); + } + + return ref; +} + +static int read_persistent(lua_State *state, PersistentDataItem ref, bool create) +{ + if (!ref.isValid()) + { + lua_pushnil(state); + lua_pushstring(state, "entry not found"); + return 2; + } + + if (create) + lua_createtable(state, 0, 4); + + lua_pushvalue(state, lua_upvalueindex(1)); + lua_setmetatable(state, -2); + + lua_pushinteger(state, ref.entry_id()); + lua_setfield(state, -2, "entry_id"); + lua_pushstring(state, ref.key().c_str()); + lua_setfield(state, -2, "key"); + lua_pushstring(state, ref.val().c_str()); + lua_setfield(state, -2, "value"); + + lua_createtable(state, PersistentDataItem::NumInts, 0); + for (int i = 0; i < PersistentDataItem::NumInts; i++) + { + lua_pushinteger(state, ref.ival(i)); + lua_rawseti(state, -2, i+1); + } + lua_setfield(state, -2, "ints"); + + return 1; +} + +static PersistentDataItem get_persistent(lua_State *state) +{ + luaL_checkany(state, 1); + + if (lua_istable(state, 1)) + { + if (!lua_getmetatable(state, 1) || + !lua_rawequal(state, -1, lua_upvalueindex(1))) + luaL_argerror(state, 1, "invalid table type"); + + lua_settop(state, 1); + + return persistent_by_struct(state, 1); + } + else + { + const char *str = luaL_checkstring(state, 1); + + return Core::getInstance().getWorld()->GetPersistentData(str); + } +} + +static int dfhack_persistent_get(lua_State *state) +{ + auto ref = get_persistent(state); + + return read_persistent(state, ref, !lua_istable(state, 1)); +} + +static int dfhack_persistent_delete(lua_State *state) +{ + auto ref = get_persistent(state); + + bool ok = Core::getInstance().getWorld()->DeletePersistentData(ref); + + lua_pushboolean(state, ok); + return 1; +} + +static int dfhack_persistent_get_all(lua_State *state) +{ + const char *str = luaL_checkstring(state, 1); + bool prefix = (lua_gettop(state)>=2 ? lua_toboolean(state,2) : false); + + std::vector data; + Core::getInstance().getWorld()->GetPersistentData(&data, str, prefix); + + if (data.empty()) + { + lua_pushnil(state); + } + else + { + lua_createtable(state, data.size(), 0); + for (size_t i = 0; i < data.size(); ++i) + { + read_persistent(state, data[i], true); + lua_rawseti(state, -2, i+1); + } + } + + return 1; +} + +static int dfhack_persistent_save(lua_State *state) +{ + lua_settop(state, 2); + luaL_checktype(state, 1, LUA_TTABLE); + bool add = lua_toboolean(state, 2); + + lua_getfield(state, 1, "key"); + const char *str = lua_tostring(state, -1); + if (!str) + luaL_argerror(state, 1, "no key field"); + + lua_settop(state, 1); + + PersistentDataItem ref; + bool added = false; + + if (add) + { + ref = Core::getInstance().getWorld()->AddPersistentData(str); + added = true; + } + else if (lua_getmetatable(state, 1)) + { + if (!lua_rawequal(state, -1, lua_upvalueindex(1))) + return luaL_argerror(state, 1, "invalid table type"); + lua_pop(state, 1); + + ref = persistent_by_struct(state, 1); + } + else + { + ref = Core::getInstance().getWorld()->GetPersistentData(str); + } + + if (!ref.isValid()) + { + ref = Core::getInstance().getWorld()->AddPersistentData(str); + if (!ref.isValid()) + luaL_error(state, "cannot create persistent entry"); + added = true; + } + + lua_getfield(state, 1, "value"); + if (const char *str = lua_tostring(state, -1)) + ref.val() = str; + lua_pop(state, 1); + + lua_getfield(state, 1, "ints"); + if (lua_istable(state, -1)) + { + for (int i = 0; i < PersistentDataItem::NumInts; i++) + { + lua_rawgeti(state, -1, i+1); + if (lua_isnumber(state, -1)) + ref.ival(i) = lua_tointeger(state, -1); + lua_pop(state, 1); + } + } + lua_pop(state, 1); + + read_persistent(state, ref, false); + lua_pushboolean(state, added); + return 2; +} + +static const luaL_Reg dfhack_persistent_funcs[] = { + { "get", dfhack_persistent_get }, + { "delete", dfhack_persistent_delete }, + { "get_all", dfhack_persistent_get_all }, + { "save", dfhack_persistent_save }, + { NULL, NULL } +}; + +static void OpenPersistent(lua_State *state) +{ + luaL_getsubtable(state, lua_gettop(state), "persistent"); + + lua_dup(state); + luaL_setfuncs(state, dfhack_persistent_funcs, 1); + + lua_dup(state); + lua_setfield(state, -2, "__index"); + + lua_pop(state, 1); +} + lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state) { if (!state) @@ -354,6 +563,9 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state) // Create and initialize the dfhack global lua_newtable(state); luaL_setfuncs(state, dfhack_funcs, 0); + + OpenPersistent(state); + lua_setglobal(state, "dfhack"); // load dfhack.lua diff --git a/library/include/modules/World.h b/library/include/modules/World.h index 50feb1495..9ed6a3ed9 100644 --- a/library/include/modules/World.h +++ b/library/include/modules/World.h @@ -93,6 +93,7 @@ namespace DFHack static const int NumInts = 7; bool isValid() { return id != 0; } + int entry_id() { return -id; } const std::string &key() { return key_value; } @@ -137,12 +138,18 @@ namespace DFHack // This ensures that the values are stored in save games. PersistentDataItem AddPersistentData(const std::string &key); PersistentDataItem GetPersistentData(const std::string &key); - void GetPersistentData(std::vector *vec, const std::string &key); - void DeletePersistentData(const PersistentDataItem &item); + PersistentDataItem GetPersistentData(int entry_id); + void GetPersistentData(std::vector *vec, + const std::string &key, bool prefix = false); + bool DeletePersistentData(const PersistentDataItem &item); - private: + void ClearPersistentCache(); + + private: struct Private; Private *d; + + bool BuildPersistentCache(); }; } #endif diff --git a/library/lua/dfhack.lua b/library/lua/dfhack.lua index ffb415646..285da5f1b 100644 --- a/library/lua/dfhack.lua +++ b/library/lua/dfhack.lua @@ -25,5 +25,16 @@ function reload(module) dofile(path) end +function printall(table) + for k,v in pairs(table) do + print(k," = "..tostring(v)) + end +end + +function dfhack.persistent:__tostring() + return "" +end + -- Feed the table back to the require() mechanism. return dfhack diff --git a/library/modules/World.cpp b/library/modules/World.cpp index cf3140e69..d570abaec 100644 --- a/library/modules/World.cpp +++ b/library/modules/World.cpp @@ -58,6 +58,7 @@ struct World::Private Private() { Inited = PauseInited = StartedWeather = StartedMode = false; + next_persistent_id = 0; } bool Inited; @@ -72,9 +73,14 @@ struct World::Private void * controlmode_offset; void * controlmodecopy_offset; + int next_persistent_id; + std::multimap persistent_index; + Process * owner; }; +typedef std::pair T_persistent_item; + World::World() { Core & c = Core::getInstance(); @@ -217,65 +223,144 @@ static PersistentDataItem dataFromHFig(df::historical_figure *hfig) return PersistentDataItem(hfig->id, hfig->name.first_name, &hfig->name.nickname, hfig->name.words); } -PersistentDataItem World::AddPersistentData(const std::string &key) +void World::ClearPersistentCache() +{ + d->next_persistent_id = 0; + d->persistent_index.clear(); +} + +bool World::BuildPersistentCache() { + if (d->next_persistent_id) + return true; + if (!Core::getInstance().isWorldLoaded()) + return false; + std::vector &hfvec = df::historical_figure::get_vector(); - int new_id = -100; - if (hfvec.size() > 0 && hfvec[0]->id <= new_id) - new_id = hfvec[0]->id-1; + // Determine the next entry id as min(-100, lowest_id-1) + d->next_persistent_id = -100; + + if (hfvec.size() > 0 && hfvec[0]->id <= -100) + d->next_persistent_id = hfvec[0]->id-1; + + // Add the entries to the lookup table + d->persistent_index.clear(); + + for (size_t i = 0; i < hfvec.size() && hfvec[i]->id <= -100; i++) + { + if (!hfvec[i]->name.has_name || hfvec[i]->name.first_name.empty()) + continue; + + d->persistent_index.insert(T_persistent_item(hfvec[i]->name.first_name, -hfvec[i]->id)); + } +} + +PersistentDataItem World::AddPersistentData(const std::string &key) +{ + if (!BuildPersistentCache() || key.empty()) + return PersistentDataItem(); + + std::vector &hfvec = df::historical_figure::get_vector(); df::historical_figure *hfig = new df::historical_figure(); - hfig->id = new_id; + hfig->id = d->next_persistent_id--; hfig->name.has_name = true; hfig->name.first_name = key; memset(hfig->name.words, 0xFF, sizeof(hfig->name.words)); hfvec.insert(hfvec.begin(), hfig); + + d->persistent_index.insert(T_persistent_item(key, -hfig->id)); + return dataFromHFig(hfig); } PersistentDataItem World::GetPersistentData(const std::string &key) { - std::vector &hfvec = df::historical_figure::get_vector(); - for (size_t i = 0; i < hfvec.size(); i++) - { - df::historical_figure *hfig = hfvec[i]; + if (!BuildPersistentCache()) + return PersistentDataItem(); - if (hfig->id >= 0) - break; + auto it = d->persistent_index.find(key); + if (it != d->persistent_index.end()) + return GetPersistentData(it->second); - if (hfig->name.has_name && hfig->name.first_name == key) - return dataFromHFig(hfig); - } + return PersistentDataItem(); +} + +PersistentDataItem World::GetPersistentData(int entry_id) +{ + if (entry_id < 100) + return PersistentDataItem(); + + auto hfig = df::historical_figure::find(-entry_id); + if (hfig && hfig->name.has_name) + return dataFromHFig(hfig); return PersistentDataItem(); } -void World::GetPersistentData(std::vector *vec, const std::string &key) +void World::GetPersistentData(std::vector *vec, const std::string &key, bool prefix) { - std::vector &hfvec = df::historical_figure::get_vector(); - for (size_t i = 0; i < hfvec.size(); i++) - { - df::historical_figure *hfig = hfvec[i]; + if (!BuildPersistentCache()) + return; - if (hfig->id >= 0) - break; + auto eqrange = d->persistent_index.equal_range(key); + + if (prefix) + { + if (key.empty()) + { + eqrange.first = d->persistent_index.begin(); + eqrange.second = d->persistent_index.end(); + } + else + { + std::string bound = key; + if (bound[bound.size()-1] != '/') + bound += "/"; + eqrange.first = d->persistent_index.lower_bound(bound); + + bound[bound.size()-1]++; + eqrange.second = d->persistent_index.lower_bound(bound); + } + } - if (hfig->name.has_name && hfig->name.first_name == key) + for (auto it = eqrange.first; it != eqrange.second; ++it) + { + auto hfig = df::historical_figure::find(-it->second); + if (hfig && hfig->name.has_name) vec->push_back(dataFromHFig(hfig)); } } -void World::DeletePersistentData(const PersistentDataItem &item) +bool World::DeletePersistentData(const PersistentDataItem &item) { if (item.id > -100) - return; + return false; + if (!BuildPersistentCache()) + return false; std::vector &hfvec = df::historical_figure::get_vector(); - int idx = binsearch_index(hfvec, item.id); - if (idx >= 0) { - delete hfvec[idx]; - hfvec.erase(hfvec.begin()+idx); + + auto eqrange = d->persistent_index.equal_range(item.key_value); + + for (auto it = eqrange.first; it != eqrange.second; ++it) + { + if (it->second != -item.id) + continue; + + d->persistent_index.erase(it); + + int idx = binsearch_index(hfvec, item.id); + + if (idx >= 0) { + delete hfvec[idx]; + hfvec.erase(hfvec.begin()+idx); + } + + return true; } + + return false; }