Improve performance of the persistent data api, and wrap it for lua.

Use an stl table for string keys to avoid linear cost of lookup.

This uncovered a bug in the new luaL_getsubtable function.
develop
Alexander Gavrilov 2012-04-01 16:43:40 +04:00
parent a9a6fbd8b5
commit afe4eba957
6 changed files with 354 additions and 40 deletions

@ -863,8 +863,8 @@ LUALIB_API int luaL_getsubtable (lua_State *L, int idx, const char *fname) {
lua_getfield(L, idx, fname); lua_getfield(L, idx, fname);
if (lua_istable(L, -1)) return 1; /* table already there */ if (lua_istable(L, -1)) return 1; /* table already there */
else { else {
idx = lua_absindex(L, idx);
lua_pop(L, 1); /* remove previous result */ lua_pop(L, 1); /* remove previous result */
idx = lua_absindex(L, idx);
lua_newtable(L); lua_newtable(L);
lua_pushvalue(L, -1); /* copy to be left at top */ lua_pushvalue(L, -1); /* copy to be left at top */
lua_setfield(L, idx, fname); /* assign new table to field */ lua_setfield(L, idx, fname); /* assign new table to field */

@ -879,31 +879,30 @@ int Core::Update()
new_wdata = wdata; new_wdata = wdata;
new_mapdata = df::global::world->map.block_index; new_mapdata = df::global::world->map.block_index;
} }
// if the world changes // if the world changes
if (new_wdata != last_world_data_ptr) if (new_wdata != last_world_data_ptr)
{ {
// we check for map change too // we check for map change too
bool mapchange = new_mapdata != last_local_map_ptr; bool mapchange = new_mapdata != last_local_map_ptr;
last_world_data_ptr = new_wdata;
last_local_map_ptr = new_mapdata;
getWorld()->ClearPersistentCache();
// and if the world is going away, we report the map change first // and if the world is going away, we report the map change first
if(!new_wdata && mapchange) if(!new_wdata && mapchange)
{
last_local_map_ptr = new_mapdata;
plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
}
// and if the world is appearing, we report map change after that // and if the world is appearing, we report map change after that
plug_mgr->OnStateChange(out, new_wdata ? SC_WORLD_LOADED : SC_WORLD_UNLOADED); plug_mgr->OnStateChange(out, new_wdata ? SC_WORLD_LOADED : SC_WORLD_UNLOADED);
if(new_wdata && mapchange) if(new_wdata && mapchange)
{
last_local_map_ptr = new_mapdata;
plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
}
// update tracking variable
last_world_data_ptr = new_wdata;
} }
// otherwise just check for map change... // otherwise just check for map change...
else if (new_mapdata != last_local_map_ptr) else if (new_mapdata != last_local_map_ptr)
{ {
last_local_map_ptr = new_mapdata; last_local_map_ptr = new_mapdata;
getWorld()->ClearPersistentCache();
plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); plug_mgr->OnStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
} }

@ -36,6 +36,8 @@ distribution.
#include "DataDefs.h" #include "DataDefs.h"
#include "DataIdentity.h" #include "DataIdentity.h"
#include "modules/World.h"
#include "LuaWrapper.h" #include "LuaWrapper.h"
#include "LuaTools.h" #include "LuaTools.h"
@ -339,6 +341,213 @@ static const luaL_Reg dfhack_funcs[] = {
{ NULL, NULL } { NULL, NULL }
}; };
/*
* Per-world persistent configuration storage.
*/
static PersistentDataItem persistent_by_struct(lua_State *state, int idx)
{
lua_getfield(state, idx, "entry_id");
int id = lua_tointeger(state, -1);
lua_pop(state, 1);
PersistentDataItem ref = Core::getInstance().getWorld()->GetPersistentData(id);
if (ref.isValid())
{
lua_getfield(state, idx, "key");
const char *str = lua_tostring(state, -1);
if (!str || str != ref.key())
luaL_argerror(state, idx, "inconsistent id and key");
lua_pop(state, 1);
}
return ref;
}
static int read_persistent(lua_State *state, PersistentDataItem ref, bool create)
{
if (!ref.isValid())
{
lua_pushnil(state);
lua_pushstring(state, "entry not found");
return 2;
}
if (create)
lua_createtable(state, 0, 4);
lua_pushvalue(state, lua_upvalueindex(1));
lua_setmetatable(state, -2);
lua_pushinteger(state, ref.entry_id());
lua_setfield(state, -2, "entry_id");
lua_pushstring(state, ref.key().c_str());
lua_setfield(state, -2, "key");
lua_pushstring(state, ref.val().c_str());
lua_setfield(state, -2, "value");
lua_createtable(state, PersistentDataItem::NumInts, 0);
for (int i = 0; i < PersistentDataItem::NumInts; i++)
{
lua_pushinteger(state, ref.ival(i));
lua_rawseti(state, -2, i+1);
}
lua_setfield(state, -2, "ints");
return 1;
}
static PersistentDataItem get_persistent(lua_State *state)
{
luaL_checkany(state, 1);
if (lua_istable(state, 1))
{
if (!lua_getmetatable(state, 1) ||
!lua_rawequal(state, -1, lua_upvalueindex(1)))
luaL_argerror(state, 1, "invalid table type");
lua_settop(state, 1);
return persistent_by_struct(state, 1);
}
else
{
const char *str = luaL_checkstring(state, 1);
return Core::getInstance().getWorld()->GetPersistentData(str);
}
}
static int dfhack_persistent_get(lua_State *state)
{
auto ref = get_persistent(state);
return read_persistent(state, ref, !lua_istable(state, 1));
}
static int dfhack_persistent_delete(lua_State *state)
{
auto ref = get_persistent(state);
bool ok = Core::getInstance().getWorld()->DeletePersistentData(ref);
lua_pushboolean(state, ok);
return 1;
}
static int dfhack_persistent_get_all(lua_State *state)
{
const char *str = luaL_checkstring(state, 1);
bool prefix = (lua_gettop(state)>=2 ? lua_toboolean(state,2) : false);
std::vector<PersistentDataItem> data;
Core::getInstance().getWorld()->GetPersistentData(&data, str, prefix);
if (data.empty())
{
lua_pushnil(state);
}
else
{
lua_createtable(state, data.size(), 0);
for (size_t i = 0; i < data.size(); ++i)
{
read_persistent(state, data[i], true);
lua_rawseti(state, -2, i+1);
}
}
return 1;
}
static int dfhack_persistent_save(lua_State *state)
{
lua_settop(state, 2);
luaL_checktype(state, 1, LUA_TTABLE);
bool add = lua_toboolean(state, 2);
lua_getfield(state, 1, "key");
const char *str = lua_tostring(state, -1);
if (!str)
luaL_argerror(state, 1, "no key field");
lua_settop(state, 1);
PersistentDataItem ref;
bool added = false;
if (add)
{
ref = Core::getInstance().getWorld()->AddPersistentData(str);
added = true;
}
else if (lua_getmetatable(state, 1))
{
if (!lua_rawequal(state, -1, lua_upvalueindex(1)))
return luaL_argerror(state, 1, "invalid table type");
lua_pop(state, 1);
ref = persistent_by_struct(state, 1);
}
else
{
ref = Core::getInstance().getWorld()->GetPersistentData(str);
}
if (!ref.isValid())
{
ref = Core::getInstance().getWorld()->AddPersistentData(str);
if (!ref.isValid())
luaL_error(state, "cannot create persistent entry");
added = true;
}
lua_getfield(state, 1, "value");
if (const char *str = lua_tostring(state, -1))
ref.val() = str;
lua_pop(state, 1);
lua_getfield(state, 1, "ints");
if (lua_istable(state, -1))
{
for (int i = 0; i < PersistentDataItem::NumInts; i++)
{
lua_rawgeti(state, -1, i+1);
if (lua_isnumber(state, -1))
ref.ival(i) = lua_tointeger(state, -1);
lua_pop(state, 1);
}
}
lua_pop(state, 1);
read_persistent(state, ref, false);
lua_pushboolean(state, added);
return 2;
}
static const luaL_Reg dfhack_persistent_funcs[] = {
{ "get", dfhack_persistent_get },
{ "delete", dfhack_persistent_delete },
{ "get_all", dfhack_persistent_get_all },
{ "save", dfhack_persistent_save },
{ NULL, NULL }
};
static void OpenPersistent(lua_State *state)
{
luaL_getsubtable(state, lua_gettop(state), "persistent");
lua_dup(state);
luaL_setfuncs(state, dfhack_persistent_funcs, 1);
lua_dup(state);
lua_setfield(state, -2, "__index");
lua_pop(state, 1);
}
lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state) lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
{ {
if (!state) if (!state)
@ -354,6 +563,9 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
// Create and initialize the dfhack global // Create and initialize the dfhack global
lua_newtable(state); lua_newtable(state);
luaL_setfuncs(state, dfhack_funcs, 0); luaL_setfuncs(state, dfhack_funcs, 0);
OpenPersistent(state);
lua_setglobal(state, "dfhack"); lua_setglobal(state, "dfhack");
// load dfhack.lua // load dfhack.lua

@ -93,6 +93,7 @@ namespace DFHack
static const int NumInts = 7; static const int NumInts = 7;
bool isValid() { return id != 0; } bool isValid() { return id != 0; }
int entry_id() { return -id; }
const std::string &key() { return key_value; } const std::string &key() { return key_value; }
@ -137,12 +138,18 @@ namespace DFHack
// This ensures that the values are stored in save games. // This ensures that the values are stored in save games.
PersistentDataItem AddPersistentData(const std::string &key); PersistentDataItem AddPersistentData(const std::string &key);
PersistentDataItem GetPersistentData(const std::string &key); PersistentDataItem GetPersistentData(const std::string &key);
void GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key); PersistentDataItem GetPersistentData(int entry_id);
void DeletePersistentData(const PersistentDataItem &item); void GetPersistentData(std::vector<PersistentDataItem> *vec,
const std::string &key, bool prefix = false);
bool DeletePersistentData(const PersistentDataItem &item);
private: void ClearPersistentCache();
private:
struct Private; struct Private;
Private *d; Private *d;
bool BuildPersistentCache();
}; };
} }
#endif #endif

@ -25,5 +25,16 @@ function reload(module)
dofile(path) dofile(path)
end end
function printall(table)
for k,v in pairs(table) do
print(k," = "..tostring(v))
end
end
function dfhack.persistent:__tostring()
return "<persistent "..self.entry_id..":"..self.key.."=\""
..self.value.."\":"..table.concat(self.ints,",")..">"
end
-- Feed the table back to the require() mechanism. -- Feed the table back to the require() mechanism.
return dfhack return dfhack

@ -58,6 +58,7 @@ struct World::Private
Private() Private()
{ {
Inited = PauseInited = StartedWeather = StartedMode = false; Inited = PauseInited = StartedWeather = StartedMode = false;
next_persistent_id = 0;
} }
bool Inited; bool Inited;
@ -72,9 +73,14 @@ struct World::Private
void * controlmode_offset; void * controlmode_offset;
void * controlmodecopy_offset; void * controlmodecopy_offset;
int next_persistent_id;
std::multimap<std::string, int> persistent_index;
Process * owner; Process * owner;
}; };
typedef std::pair<std::string, int> T_persistent_item;
World::World() World::World()
{ {
Core & c = Core::getInstance(); Core & c = Core::getInstance();
@ -217,65 +223,144 @@ static PersistentDataItem dataFromHFig(df::historical_figure *hfig)
return PersistentDataItem(hfig->id, hfig->name.first_name, &hfig->name.nickname, hfig->name.words); return PersistentDataItem(hfig->id, hfig->name.first_name, &hfig->name.nickname, hfig->name.words);
} }
PersistentDataItem World::AddPersistentData(const std::string &key) void World::ClearPersistentCache()
{
d->next_persistent_id = 0;
d->persistent_index.clear();
}
bool World::BuildPersistentCache()
{ {
if (d->next_persistent_id)
return true;
if (!Core::getInstance().isWorldLoaded())
return false;
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector(); std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
int new_id = -100; // Determine the next entry id as min(-100, lowest_id-1)
if (hfvec.size() > 0 && hfvec[0]->id <= new_id) d->next_persistent_id = -100;
new_id = hfvec[0]->id-1;
if (hfvec.size() > 0 && hfvec[0]->id <= -100)
d->next_persistent_id = hfvec[0]->id-1;
// Add the entries to the lookup table
d->persistent_index.clear();
for (size_t i = 0; i < hfvec.size() && hfvec[i]->id <= -100; i++)
{
if (!hfvec[i]->name.has_name || hfvec[i]->name.first_name.empty())
continue;
d->persistent_index.insert(T_persistent_item(hfvec[i]->name.first_name, -hfvec[i]->id));
}
}
PersistentDataItem World::AddPersistentData(const std::string &key)
{
if (!BuildPersistentCache() || key.empty())
return PersistentDataItem();
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
df::historical_figure *hfig = new df::historical_figure(); df::historical_figure *hfig = new df::historical_figure();
hfig->id = new_id; hfig->id = d->next_persistent_id--;
hfig->name.has_name = true; hfig->name.has_name = true;
hfig->name.first_name = key; hfig->name.first_name = key;
memset(hfig->name.words, 0xFF, sizeof(hfig->name.words)); memset(hfig->name.words, 0xFF, sizeof(hfig->name.words));
hfvec.insert(hfvec.begin(), hfig); hfvec.insert(hfvec.begin(), hfig);
d->persistent_index.insert(T_persistent_item(key, -hfig->id));
return dataFromHFig(hfig); return dataFromHFig(hfig);
} }
PersistentDataItem World::GetPersistentData(const std::string &key) PersistentDataItem World::GetPersistentData(const std::string &key)
{ {
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector(); if (!BuildPersistentCache())
for (size_t i = 0; i < hfvec.size(); i++) return PersistentDataItem();
{
df::historical_figure *hfig = hfvec[i];
if (hfig->id >= 0) auto it = d->persistent_index.find(key);
break; if (it != d->persistent_index.end())
return GetPersistentData(it->second);
if (hfig->name.has_name && hfig->name.first_name == key) return PersistentDataItem();
return dataFromHFig(hfig); }
}
PersistentDataItem World::GetPersistentData(int entry_id)
{
if (entry_id < 100)
return PersistentDataItem();
auto hfig = df::historical_figure::find(-entry_id);
if (hfig && hfig->name.has_name)
return dataFromHFig(hfig);
return PersistentDataItem(); return PersistentDataItem();
} }
void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key) void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key, bool prefix)
{ {
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector(); if (!BuildPersistentCache())
for (size_t i = 0; i < hfvec.size(); i++) return;
{
df::historical_figure *hfig = hfvec[i];
if (hfig->id >= 0) auto eqrange = d->persistent_index.equal_range(key);
break;
if (prefix)
{
if (key.empty())
{
eqrange.first = d->persistent_index.begin();
eqrange.second = d->persistent_index.end();
}
else
{
std::string bound = key;
if (bound[bound.size()-1] != '/')
bound += "/";
eqrange.first = d->persistent_index.lower_bound(bound);
bound[bound.size()-1]++;
eqrange.second = d->persistent_index.lower_bound(bound);
}
}
if (hfig->name.has_name && hfig->name.first_name == key) for (auto it = eqrange.first; it != eqrange.second; ++it)
{
auto hfig = df::historical_figure::find(-it->second);
if (hfig && hfig->name.has_name)
vec->push_back(dataFromHFig(hfig)); vec->push_back(dataFromHFig(hfig));
} }
} }
void World::DeletePersistentData(const PersistentDataItem &item) bool World::DeletePersistentData(const PersistentDataItem &item)
{ {
if (item.id > -100) if (item.id > -100)
return; return false;
if (!BuildPersistentCache())
return false;
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector(); std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
int idx = binsearch_index(hfvec, item.id);
if (idx >= 0) { auto eqrange = d->persistent_index.equal_range(item.key_value);
delete hfvec[idx];
hfvec.erase(hfvec.begin()+idx); for (auto it = eqrange.first; it != eqrange.second; ++it)
{
if (it->second != -item.id)
continue;
d->persistent_index.erase(it);
int idx = binsearch_index(hfvec, item.id);
if (idx >= 0) {
delete hfvec[idx];
hfvec.erase(hfvec.begin()+idx);
}
return true;
} }
return false;
} }