Add Persistence module.

Alter World to use Persistence instead of storing data in historical figures.

Fake historical figures will be converted to the new format when a world is loaded.

Added plugin_save and plugin_load functions to the plugin API.

Made the weird int7/int28 code that was in the old persistence API slightly safer.
develop
Ben Lubar 2018-08-26 18:26:09 -05:00
parent e56cb2a25a
commit 4e690df96a
No known key found for this signature in database
GPG Key ID: 92939677AB59EDA4
10 changed files with 793 additions and 259 deletions

@ -126,6 +126,7 @@ include/modules/Maps.h
include/modules/Materials.h include/modules/Materials.h
include/modules/Notes.h include/modules/Notes.h
include/modules/Once.h include/modules/Once.h
include/modules/Persistence.h
include/modules/Random.h include/modules/Random.h
include/modules/Renderer.h include/modules/Renderer.h
include/modules/Screen.h include/modules/Screen.h
@ -152,6 +153,7 @@ modules/Maps.cpp
modules/Materials.cpp modules/Materials.cpp
modules/Notes.cpp modules/Notes.cpp
modules/Once.cpp modules/Once.cpp
modules/Persistence.cpp
modules/Random.cpp modules/Random.cpp
modules/Renderer.cpp modules/Renderer.cpp
modules/Screen.cpp modules/Screen.cpp

@ -53,6 +53,7 @@ using namespace std;
#include "modules/World.h" #include "modules/World.h"
#include "modules/Graphic.h" #include "modules/Graphic.h"
#include "modules/Windows.h" #include "modules/Windows.h"
#include "modules/Persistence.h"
#include "RemoteServer.h" #include "RemoteServer.h"
#include "RemoteTools.h" #include "RemoteTools.h"
#include "LuaTools.h" #include "LuaTools.h"
@ -72,6 +73,7 @@ using namespace DFHack;
#include "df/viewscreen_loadgamest.h" #include "df/viewscreen_loadgamest.h"
#include "df/viewscreen_new_regionst.h" #include "df/viewscreen_new_regionst.h"
#include "df/viewscreen_savegamest.h" #include "df/viewscreen_savegamest.h"
#include "df/viewscreen_optionst.h"
#include <df/graphic.h> #include <df/graphic.h>
#include <stdio.h> #include <stdio.h>
@ -2004,8 +2006,6 @@ void Core::doUpdate(color_ostream &out, bool first_update)
last_world_data_ptr = new_wdata; last_world_data_ptr = new_wdata;
last_local_map_ptr = new_mapdata; last_local_map_ptr = new_mapdata;
World::ClearPersistentCache();
// and if the world is going away, we report the map change first // and if the world is going away, we report the map change first
if(had_map) if(had_map)
onStateChange(out, SC_MAP_UNLOADED); onStateChange(out, SC_MAP_UNLOADED);
@ -2022,7 +2022,6 @@ void Core::doUpdate(color_ostream &out, bool first_update)
if (isMapLoaded() != had_map) if (isMapLoaded() != had_map)
{ {
World::ClearPersistentCache();
onStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED); onStateChange(out, new_mapdata ? SC_MAP_LOADED : SC_MAP_UNLOADED);
} }
} }
@ -2042,6 +2041,11 @@ void Core::doUpdate(color_ostream &out, bool first_update)
// Execute per-frame handlers // Execute per-frame handlers
onUpdate(out); onUpdate(out);
if (df::global::ui->main.autosave_request || strict_virtual_cast<df::viewscreen_optionst>(screen))
{
doSave(out);
}
out << std::flush; out << std::flush;
} }
@ -2285,6 +2289,27 @@ void Core::onStateChange(color_ostream &out, state_change_event event)
Lua::Core::onStateChange(out, event); Lua::Core::onStateChange(out, event);
handleLoadAndUnloadScripts(out, event); handleLoadAndUnloadScripts(out, event);
if (event == SC_WORLD_UNLOADED)
{
Persistence::Internal::clear();
}
if (event == SC_WORLD_LOADED)
{
doLoad(out);
}
}
void Core::doSave(color_ostream &out)
{
plug_mgr->doSave(out);
Persistence::Internal::save();
}
void Core::doLoad(color_ostream &out)
{
Persistence::Internal::load();
plug_mgr->doLoad(out);
} }
int Core::Shutdown ( void ) int Core::Shutdown ( void )

@ -197,6 +197,9 @@ Plugin::Plugin(Core * core, const std::string & path,
plugin_rpcconnect = 0; plugin_rpcconnect = 0;
plugin_enable = 0; plugin_enable = 0;
plugin_is_enabled = 0; plugin_is_enabled = 0;
plugin_save = 0;
plugin_load = 0;
plugin_eval_ruby = 0;
state = PS_UNLOADED; state = PS_UNLOADED;
access = new RefLock(); access = new RefLock();
} }
@ -348,6 +351,8 @@ bool Plugin::load(color_ostream &con)
plugin_rpcconnect = (RPCService* (*)(color_ostream &)) LookupPlugin(plug, "plugin_rpcconnect"); plugin_rpcconnect = (RPCService* (*)(color_ostream &)) LookupPlugin(plug, "plugin_rpcconnect");
plugin_enable = (command_result (*)(color_ostream &,bool)) LookupPlugin(plug, "plugin_enable"); plugin_enable = (command_result (*)(color_ostream &,bool)) LookupPlugin(plug, "plugin_enable");
plugin_is_enabled = (bool*) LookupPlugin(plug, "plugin_is_enabled"); plugin_is_enabled = (bool*) LookupPlugin(plug, "plugin_is_enabled");
plugin_save = (command_result (*)(color_ostream &)) LookupPlugin(plug, "plugin_save");
plugin_load = (command_result (*)(color_ostream &)) LookupPlugin(plug, "plugin_load");
plugin_eval_ruby = (command_result (*)(color_ostream &, const char*)) LookupPlugin(plug, "plugin_eval_ruby"); plugin_eval_ruby = (command_result (*)(color_ostream &, const char*)) LookupPlugin(plug, "plugin_eval_ruby");
index_lua(plug); index_lua(plug);
plugin_lib = plug; plugin_lib = plug;
@ -359,6 +364,8 @@ bool Plugin::load(color_ostream &con)
parent->registerCommands(this); parent->registerCommands(this);
if ((plugin_onupdate || plugin_enable) && !plugin_is_enabled) if ((plugin_onupdate || plugin_enable) && !plugin_is_enabled)
con.printerr("Plugin %s has no enabled var!\n", name.c_str()); con.printerr("Plugin %s has no enabled var!\n", name.c_str());
if (Core::getInstance().isWorldLoaded() && plugin_load && plugin_load(con) != CR_OK)
con.printerr("Plugin %s has failed to load saved data.\n", name.c_str());
fprintf(stderr, "loaded plugin %s; DFHack build %s\n", name.c_str(), plug_git_desc); fprintf(stderr, "loaded plugin %s; DFHack build %s\n", name.c_str(), plug_git_desc);
fflush(stderr); fflush(stderr);
return true; return true;
@ -403,6 +410,8 @@ bool Plugin::unload(color_ostream &con)
// enter suspend // enter suspend
CoreSuspender suspend; CoreSuspender suspend;
access->lock(); access->lock();
if (Core::getInstance().isWorldLoaded() && plugin_save && plugin_save(con) != CR_OK)
con.printerr("Plugin %s has failed to save data.\n", name.c_str());
// notify plugin about shutdown, if it has a shutdown function // notify plugin about shutdown, if it has a shutdown function
command_result cr = CR_OK; command_result cr = CR_OK;
if(plugin_shutdown) if(plugin_shutdown)
@ -410,6 +419,8 @@ bool Plugin::unload(color_ostream &con)
// cleanup... // cleanup...
plugin_is_enabled = 0; plugin_is_enabled = 0;
plugin_onupdate = 0; plugin_onupdate = 0;
plugin_save = 0;
plugin_load = 0;
reset_lua(); reset_lua();
parent->unregisterCommands(this); parent->unregisterCommands(this);
commands.clear(); commands.clear();
@ -570,6 +581,32 @@ command_result Plugin::on_state_change(color_ostream &out, state_change_event ev
return cr; return cr;
} }
command_result Plugin::save_data(color_ostream &out)
{
command_result cr = CR_NOT_IMPLEMENTED;
access->lock_add();
if(state == PS_LOADED && plugin_save)
{
cr = plugin_save(out);
Lua::Core::Reset(out, "plugin_save");
}
access->lock_sub();
return cr;
}
command_result Plugin::load_data(color_ostream &out)
{
command_result cr = CR_NOT_IMPLEMENTED;
access->lock_add();
if(state == PS_LOADED && plugin_load)
{
cr = plugin_load(out);
Lua::Core::Reset(out, "plugin_load");
}
access->lock_sub();
return cr;
}
RPCService *Plugin::rpc_connect(color_ostream &out) RPCService *Plugin::rpc_connect(color_ostream &out)
{ {
RPCService *rv = NULL; RPCService *rv = NULL;
@ -1014,6 +1051,28 @@ void PluginManager::unregisterCommands( Plugin * p )
cmdlist_mutex->unlock(); cmdlist_mutex->unlock();
} }
void PluginManager::doSave(color_ostream &out)
{
for (auto it = begin(); it != end(); ++it)
{
command_result cr = it->second->save_data(out);
if (cr != CR_OK && cr != CR_NOT_IMPLEMENTED)
out.printerr("Plugin %s has failed to save data.\n", it->first.c_str());
}
}
void PluginManager::doLoad(color_ostream &out)
{
for (auto it = begin(); it != end(); ++it)
{
command_result cr = it->second->load_data(out);
if (cr != CR_OK && cr != CR_NOT_IMPLEMENTED)
out.printerr("Plugin %s has failed to load saved data.\n", it->first.c_str());
}
}
Plugin *PluginManager::operator[] (std::string name) Plugin *PluginManager::operator[] (std::string name)
{ {
MUTEX_GUARD(plugin_mutex); MUTEX_GUARD(plugin_mutex);

@ -222,6 +222,8 @@ namespace DFHack
void onUpdate(color_ostream &out); void onUpdate(color_ostream &out);
void onStateChange(color_ostream &out, state_change_event event); void onStateChange(color_ostream &out, state_change_event event);
void handleLoadAndUnloadScripts(color_ostream &out, state_change_event event); void handleLoadAndUnloadScripts(color_ostream &out, state_change_event event);
void doSave(color_ostream &out);
void doLoad(color_ostream &out);
Core(Core const&); // Don't Implement Core(Core const&); // Don't Implement
void operator=(Core const&); // Don't implement void operator=(Core const&); // Don't implement

@ -148,6 +148,8 @@ namespace DFHack
~Plugin(); ~Plugin();
command_result on_update(color_ostream &out); command_result on_update(color_ostream &out);
command_result on_state_change(color_ostream &out, state_change_event event); command_result on_state_change(color_ostream &out, state_change_event event);
command_result save_data(color_ostream &out);
command_result load_data(color_ostream &out);
void detach_connection(RPCService *svc); void detach_connection(RPCService *svc);
public: public:
enum plugin_state enum plugin_state
@ -238,6 +240,8 @@ namespace DFHack
command_result (*plugin_enable)(color_ostream &, bool); command_result (*plugin_enable)(color_ostream &, bool);
RPCService* (*plugin_rpcconnect)(color_ostream &); RPCService* (*plugin_rpcconnect)(color_ostream &);
command_result (*plugin_eval_ruby)(color_ostream &, const char*); command_result (*plugin_eval_ruby)(color_ostream &, const char*);
command_result (*plugin_save)(color_ostream &);
command_result (*plugin_load)(color_ostream &);
}; };
class DFHACK_EXPORT PluginManager class DFHACK_EXPORT PluginManager
{ {
@ -251,6 +255,8 @@ namespace DFHack
void OnStateChange(color_ostream &out, state_change_event event); void OnStateChange(color_ostream &out, state_change_event event);
void registerCommands( Plugin * p ); void registerCommands( Plugin * p );
void unregisterCommands( Plugin * p ); void unregisterCommands( Plugin * p );
void doSave(color_ostream &out);
void doLoad(color_ostream &out);
// PUBLIC METHODS // PUBLIC METHODS
public: public:
// list names of all plugins present in hack/plugins // list names of all plugins present in hack/plugins

@ -0,0 +1,231 @@
/*
https://github.com/DFHack/dfhack
Copyright (c) 2009-2018 DFHack Team
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any
damages arising from the use of this software.
Permission is granted to anyone to use this software for any
purpose, including commercial applications, and to alter it and
redistribute it freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product documentation
would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and
must not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*/
#pragma once
#ifndef CL_MOD_PERSISTENCE
#define CL_MOD_PERSISTENCE
/**
* \defgroup grp_persistence Persistence: code related to saving and loading data
* @ingroup grp_modules
*/
#include <fstream>
#include <memory>
#include <string>
#include <vector>
#include "Export.h"
#include "Error.h"
namespace DFHack
{
class Core;
namespace Persistence
{
struct LegacyData;
class Internal;
}
class DFHACK_EXPORT PersistentDataItem {
size_t index;
std::shared_ptr<Persistence::LegacyData> data;
public:
static const int NumInts = 7;
bool isValid() const;
size_t get_index() const
{
CHECK_INVALID_ARGUMENT(isValid());
return index;
}
int entry_id() const { return isValid() ? int(index) + 100 : 0; }
int raw_id() const { return isValid() ? -int(index) - 100 : 0; }
const std::string &key() const;
std::string &val();
const std::string &val() const;
int &ival(int i);
int ival(int i) const;
// Pack binary data into string field.
// Since DF serialization chokes on NUL bytes,
// use bit magic to ensure none of the bytes is 0.
// Choose the lowest bit for padding so that
// sign-extend can be used normally.
size_t data_size() const
{
return val().size();
}
bool check_data(size_t off, size_t sz = 1) const
{
return (data_size() >= off + sz);
}
void ensure_data(size_t off, size_t sz = 0)
{
if (data_size() < off + sz)
{
val().resize(off + sz, '\x01');
}
}
template<size_t N>
uint8_t (&pdata(size_t off))[N]
{
ensure_data(off, N);
typedef uint8_t array[N];
return *(array *)(val().data() + off);
}
template<size_t N>
const uint8_t (&pdata(size_t off) const)[N]
{
CHECK_INVALID_ARGUMENT(check_data(off, N));
typedef const uint8_t array[N];
return *(array *)(val().data() + off);
}
template<size_t N>
const uint8_t (&cpdata(size_t off) const)[N]
{
return pdata<N>(off);
}
static const size_t int7_size = 1;
uint8_t get_uint7(size_t off) const
{
auto p = pdata<int7_size>(off);
return p[0] >> 1;
}
int8_t get_int7(size_t off) const
{
return int8_t(get_uint7(off));
}
void set_uint7(size_t off, uint8_t val)
{
auto p = pdata<int7_size>(off);
p[0] = uint8_t((val << 1) | 1);
}
void set_int7(size_t off, int8_t val)
{
set_uint7(off, uint8_t(val));
}
static const size_t int28_size = 4;
uint32_t get_uint28(size_t off) const
{
auto p = pdata<int28_size>(off);
return (p[0]>>1) | ((p[1]&~1U)<<6) | ((p[2]&~1U)<<13) | ((p[3]&~1U)<<20);
}
int32_t get_int28(size_t off) const
{
return int32_t(get_uint28(off));
}
void set_uint28(size_t off, uint32_t val)
{
auto p = pdata<int28_size>(off);
p[0] = uint8_t((val<<1) | 1);
p[1] = uint8_t((val>>6) | 1);
p[2] = uint8_t((val>>13) | 1);
p[3] = uint8_t((val>>20) | 1);
}
void set_int28(size_t off, int32_t val)
{
set_uint28(off, val);
}
PersistentDataItem() : index(0), data(nullptr) {}
PersistentDataItem(size_t index, const std::shared_ptr<Persistence::LegacyData> &data)
: index(index), data(data) {}
};
namespace Persistence
{
class Internal
{
static void clear();
static void save();
static void load();
friend class ::DFHack::Core;
};
// Returns a new PersistentDataItem with the specified key.
// If there is no world loaded or the key is empty, returns an invalid item.
DFHACK_EXPORT PersistentDataItem addItem(const std::string &key);
// Returns an existing PersistentDataItem with the specified key.
// If "added" is not null and there is no such item, a new item is returned and
// the bool value is set to true. If "added" is not null and an item is found or
// no new item can be created, the bool value is set to false.
DFHACK_EXPORT PersistentDataItem getByKey(const std::string &key, bool *added = nullptr);
// Returns an existing PersistentDataItem with the specified index.
// If there is no world loaded or the index is empty, returns an invalid item.
DFHACK_EXPORT PersistentDataItem getByIndex(size_t index);
// If the item is invalid, returns false. Otherwise, deletes the item and returns
// true. All references to the item are invalid as soon as this function returns.
DFHACK_EXPORT bool deleteItem(const PersistentDataItem &item);
// Fills the vector with references to each persistent item.
DFHACK_EXPORT void getAll(std::vector<PersistentDataItem> &vec);
// Fills the vector with references to each persistent item with a key that is
// greater than or equal to "min" and less than "max".
DFHACK_EXPORT void getAllByKeyRange(std::vector<PersistentDataItem> &vec, const std::string &min, const std::string &max);
// Fills the vector with references to each persistent item with a key that is
// equal to the given key.
DFHACK_EXPORT void getAllByKey(std::vector<PersistentDataItem> &vec, const std::string &key);
#if defined(__GNUC__) && __GNUC__ < 5
// file stream move constructors are missing in libstdc++ before version 5.
template<typename T>
class DFHACK_EXPORT gcc_4_fstream_shim : public T
{
const std::string file;
public:
explicit gcc_4_fstream_shim() : T(), file() {}
explicit gcc_4_fstream_shim(const std::string &file) : T(file), file() {}
gcc_4_fstream_shim(gcc_4_fstream_shim<T> && s) : T(), file(s.file)
{
if (!file.empty())
{
T::open(file.c_str());
}
}
};
#define FSTREAM(x) gcc_4_fstream_shim<x>
#else
#define FSTREAM(x) x
#endif
// Returns an input stream that data can be read from. If no world is currently loaded,
// or no data has been saved with the specified key, the stream is invalid and acts
// as if the file is empty.
DFHACK_EXPORT FSTREAM(std::ifstream) readSaveData(const std::string &key);
// Returns an output stream that data can be saved to. If no world is currently loaded,
// an invalid stream is returned, and writing to it has no effect.
DFHACK_EXPORT FSTREAM(std::ofstream) writeSaveData(const std::string &key);
#undef FSTREAM
}
}
#endif

@ -33,6 +33,7 @@ distribution.
#include "Export.h" #include "Export.h"
#include "Module.h" #include "Module.h"
#include "modules/Persistence.h"
#include <ostream> #include <ostream>
#include "DataDefs.h" #include "DataDefs.h"
@ -60,81 +61,6 @@ namespace DFHack
}; };
class DFContextShared; class DFContextShared;
class DFHACK_EXPORT PersistentDataItem {
int id;
std::string key_value;
std::string *str_value;
int *int_values;
public:
static const int NumInts = 7;
bool isValid() const { return id != 0; }
int entry_id() const { return -id; }
int raw_id() const { return id; }
const std::string &key() const { return key_value; }
std::string &val() { return *str_value; }
const std::string &val() const { return *str_value; }
int &ival(int i) { return int_values[i]; }
int ival(int i) const { return int_values[i]; }
// Pack binary data into string field.
// Since DF serialization chokes on NUL bytes,
// use bit magic to ensure none of the bytes is 0.
// Choose the lowest bit for padding so that
// sign-extend can be used normally.
size_t data_size() const { return str_value->size(); }
bool check_data(size_t off, size_t sz = 1) {
return (str_value->size() >= off+sz);
}
void ensure_data(size_t off, size_t sz = 0) {
if (str_value->size() < off+sz) str_value->resize(off+sz, '\x01');
}
uint8_t *pdata(size_t off) { return (uint8_t*)&(*str_value)[off]; }
static const size_t int7_size = 1;
uint8_t get_uint7(size_t off) {
uint8_t *p = pdata(off);
return p[0]>>1;
}
int8_t get_int7(size_t off) {
uint8_t *p = pdata(off);
return int8_t(p[0])>>1;
}
void set_uint7(size_t off, uint8_t val) {
uint8_t *p = pdata(off);
p[0] = uint8_t((val<<1) | 1);
}
void set_int7(size_t off, int8_t val) { set_uint7(off, val); }
static const size_t int28_size = 4;
uint32_t get_uint28(size_t off) {
uint8_t *p = pdata(off);
return (p[0]>>1) | ((p[1]&~1U)<<6) | ((p[2]&~1U)<<13) | ((p[3]&~1U)<<20);
}
int32_t get_int28(size_t off) {
uint8_t *p = pdata(off);
return (p[0]>>1) | ((p[1]&~1U)<<6) | ((p[2]&~1U)<<13) | ((int8_t(p[3])&~1)<<20);
}
void set_uint28(size_t off, uint32_t val) {
uint8_t *p = pdata(off);
p[0] = uint8_t((val<<1) | 1);
p[1] = uint8_t((val>>6) | 1);
p[2] = uint8_t((val>>13) | 1);
p[3] = uint8_t((val>>20) | 1);
}
void set_int28(size_t off, int32_t val) { set_uint28(off, val); }
PersistentDataItem() : id(0), str_value(0), int_values(0) {}
PersistentDataItem(int id, const std::string &key, std::string *sv, int *iv)
: id(id), key_value(key), str_value(sv), int_values(iv) {}
};
/** /**
* The World module * The World module
* \ingroup grp_modules * \ingroup grp_modules
@ -179,8 +105,6 @@ namespace DFHack
// Deletes the item; returns true if success. // Deletes the item; returns true if success.
DFHACK_EXPORT bool DeletePersistentData(const PersistentDataItem &item); DFHACK_EXPORT bool DeletePersistentData(const PersistentDataItem &item);
DFHACK_EXPORT void ClearPersistentCache();
DFHACK_EXPORT df::tile_bitmask *getPersistentTilemask(const PersistentDataItem &item, df::map_block *block, bool create = false); DFHACK_EXPORT df::tile_bitmask *getPersistentTilemask(const PersistentDataItem &item, df::map_block *block, bool create = false);
DFHACK_EXPORT bool deletePersistentTilemask(const PersistentDataItem &item, df::map_block *block); DFHACK_EXPORT bool deletePersistentTilemask(const PersistentDataItem &item, df::map_block *block);
} }

@ -0,0 +1,420 @@
/*
https://github.com/peterix/dfhack
Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any
damages arising from the use of this software.
Permission is granted to anyone to use this software for any
purpose, including commercial applications, and to alter it and
redistribute it freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product documentation
would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and
must not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*/
#include "Internal.h"
#include <array>
#include <map>
#include <json/json.h>
#include "Core.h"
#include "DataDefs.h"
#include "modules/Persistence.h"
#include "modules/World.h"
#include "df/historical_figure.h"
using namespace DFHack;
static std::vector<std::shared_ptr<Persistence::LegacyData>> legacy_data;
static std::multimap<std::string, size_t> index_cache;
struct Persistence::LegacyData
{
const std::string key;
std::string str_value;
std::array<int, PersistentDataItem::NumInts> int_values;
explicit LegacyData(const std::string &key) : key(key)
{
for (int i = 0; i < PersistentDataItem::NumInts; i++)
{
int_values.at(i) = -1;
}
}
explicit LegacyData(Json::Value &json) : key(json["k"].asString())
{
str_value = json["s"].asString();
for (int i = 0; i < PersistentDataItem::NumInts; i++)
{
int_values.at(i) = json["i"][i].asInt();
}
}
explicit LegacyData(const df::language_name &name) : key(name.first_name)
{
str_value = name.nickname;
for (int i = 0; i < PersistentDataItem::NumInts; i++)
{
int_values.at(i) = name.words[i];
}
}
Json::Value toJSON()
{
Json::Value json(Json::objectValue);
json["k"] = key;
json["s"] = str_value;
Json::Value ints(Json::arrayValue);
for (int i = 0; i < PersistentDataItem::NumInts; i++)
{
ints[i] = int_values.at(i);
}
json["i"] = std::move(ints);
return std::move(json);
}
};
const std::string &PersistentDataItem::key() const
{
CHECK_INVALID_ARGUMENT(isValid());
return data->key;
}
std::string &PersistentDataItem::val()
{
CHECK_INVALID_ARGUMENT(isValid());
return data->str_value;
}
const std::string &PersistentDataItem::val() const
{
CHECK_INVALID_ARGUMENT(isValid());
return data->str_value;
}
int &PersistentDataItem::ival(int i)
{
CHECK_INVALID_ARGUMENT(isValid());
CHECK_INVALID_ARGUMENT(i >= 0 && i < NumInts);
return data->int_values.at(i);
}
int PersistentDataItem::ival(int i) const
{
CHECK_INVALID_ARGUMENT(isValid());
CHECK_INVALID_ARGUMENT(i >= 0 && i < NumInts);
return data->int_values.at(i);
}
bool PersistentDataItem::isValid() const
{
if (data == nullptr)
return false;
CoreSuspender suspend;
if (legacy_data.size() <= index)
return false;
return legacy_data.at(index) == data;
}
void Persistence::Internal::clear()
{
CoreSuspender suspend;
legacy_data.clear();
index_cache.clear();
}
void Persistence::Internal::save()
{
CoreSuspender suspend;
Json::Value json(Json::arrayValue);
for (size_t i = 0; i < legacy_data.size(); i++)
{
if (legacy_data.at(i) != nullptr)
{
while (json.size() < i)
{
json[json.size()] = Json::Value();
}
json[int(i)] = legacy_data.at(i)->toJSON();
}
}
auto file = writeSaveData("legacy-data");
file << json;
}
static void convertHFigs()
{
auto &figs = df::historical_figure::get_vector();
auto src = figs.begin();
while (src != figs.end() && (*src)->id > -100)
{
++src;
}
if (src == figs.end())
{
return;
}
auto dst = src;
while (src != figs.end())
{
auto fig = *src;
if (fig->id > -100)
{
*dst = *src;
++dst;
}
else
{
if (fig->name.has_name && !fig->name.first_name.empty())
{
size_t index = size_t(-fig->id - 100);
if (legacy_data.size() <= index)
{
legacy_data.resize(index + 1);
}
legacy_data.at(index) = std::shared_ptr<Persistence::LegacyData>(new Persistence::LegacyData(fig->name));
}
delete fig;
}
++src;
}
figs.erase(dst, figs.end());
}
void Persistence::Internal::load()
{
CoreSuspender suspend;
clear();
auto file = readSaveData("legacy-data");
Json::Value json;
try
{
file >> json;
}
catch (std::exception &)
{
// empty file?
}
if (json.isArray())
{
legacy_data.resize(json.size());
for (size_t i = 0; i < legacy_data.size(); i++)
{
if (json[int(i)].isObject())
{
legacy_data.at(i) = std::shared_ptr<LegacyData>(new LegacyData(json[int(i)]));
}
}
}
convertHFigs();
for (size_t i = 0; i < legacy_data.size(); i++)
{
if (legacy_data.at(i) == nullptr)
{
continue;
}
index_cache.insert(std::make_pair(legacy_data.at(i)->key, i));
}
}
PersistentDataItem Persistence::addItem(const std::string &key)
{
if (key.empty() || !Core::getInstance().isWorldLoaded())
return PersistentDataItem();
CoreSuspender suspend;
size_t index = 0;
while (index < legacy_data.size() && legacy_data.at(index) != nullptr)
{
index++;
}
auto ptr = std::shared_ptr<LegacyData>(new LegacyData(key));
if (index == legacy_data.size())
{
legacy_data.push_back(ptr);
}
else
{
legacy_data.at(index) = ptr;
}
index_cache.insert(std::make_pair(key, index));
return PersistentDataItem(index, ptr);
}
PersistentDataItem Persistence::getByKey(const std::string &key, bool *added)
{
CoreSuspender suspend;
auto it = index_cache.find(key);
if (added)
{
*added = it == index_cache.end();
}
if (it != index_cache.end())
{
return PersistentDataItem(it->second, legacy_data.at(it->second));
}
if (!added)
{
return PersistentDataItem();
}
return addItem(key);
}
PersistentDataItem Persistence::getByIndex(size_t index)
{
CoreSuspender suspend;
if (index < legacy_data.size() && legacy_data.at(index) != nullptr)
{
return PersistentDataItem(index, legacy_data.at(index));
}
return PersistentDataItem();
}
bool Persistence::deleteItem(const PersistentDataItem &item)
{
CoreSuspender suspend;
if (!item.isValid())
{
return false;
}
size_t index = item.get_index();
auto range = index_cache.equal_range(item.key());
for (auto it = range.first; it != range.second; ++it)
{
if (it->second == index)
{
index_cache.erase(it);
break;
}
}
legacy_data.at(index) = nullptr;
return true;
}
void Persistence::getAll(std::vector<PersistentDataItem> &vec)
{
vec.clear();
CoreSuspender suspend;
for (size_t i = 0; i < legacy_data.size(); i++)
{
if (legacy_data.at(i) != nullptr)
{
vec.push_back(PersistentDataItem(i, legacy_data.at(i)));
}
}
}
void Persistence::getAllByKeyRange(std::vector<PersistentDataItem> &vec, const std::string &min, const std::string &max)
{
vec.clear();
CoreSuspender suspend;
auto begin = index_cache.lower_bound(min);
auto end = index_cache.lower_bound(max);
for (auto it = begin; it != end; ++it)
{
vec.push_back(PersistentDataItem(it->second, legacy_data.at(it->second)));
}
}
void Persistence::getAllByKey(std::vector<PersistentDataItem> &vec, const std::string &key)
{
vec.clear();
CoreSuspender suspend;
auto range = index_cache.equal_range(key);
for (auto it = range.first; it != range.second; ++it)
{
vec.push_back(PersistentDataItem(it->second, legacy_data.at(it->second)));
}
}
static std::string filterSaveFileName(std::string s)
{
for (auto &ch : s)
{
if (!isalnum(ch) && ch != '-' && ch != '_')
{
ch = '_';
}
}
return s;
}
static std::string getSaveFilePath(const std::string &world, const std::string &name)
{
return "data/save/" + world + "/dfhack-" + filterSaveFileName(name) + ".dat";
}
#if defined(__GNUC__) && __GNUC__ < 5
// file stream move constructors are missing in libstdc++ before version 5.
#define FSTREAM(x) Persistence::gcc_4_fstream_shim<x>
#else
#define FSTREAM(x) x
#endif
FSTREAM(std::ifstream) Persistence::readSaveData(const std::string &name)
{
if (!Core::getInstance().isWorldLoaded())
{
// No world loaded - return unopened stream.
return FSTREAM(std::ifstream)();
}
return FSTREAM(std::ifstream)(getSaveFilePath(World::ReadWorldFolder(), name));
}
FSTREAM(std::ofstream) Persistence::writeSaveData(const std::string &name)
{
if (!Core::getInstance().isWorldLoaded())
{
// No world loaded - return unopened stream.
return FSTREAM(std::ofstream)();
}
return FSTREAM(std::ofstream)(getSaveFilePath("current", name));
}
#undef FSTREAM

@ -24,6 +24,7 @@ distribution.
#include "Internal.h" #include "Internal.h"
#include <array>
#include <string> #include <string>
#include <vector> #include <vector>
#include <map> #include <map>
@ -56,10 +57,6 @@ using namespace df::enums;
using df::global::world; using df::global::world;
static int next_persistent_id = 0;
static std::multimap<std::string, int> persistent_index;
typedef std::pair<std::string, int> T_persistent_item;
bool World::ReadPauseState() bool World::ReadPauseState()
{ {
return DF_GLOBAL_VALUE(pause_state, false); return DF_GLOBAL_VALUE(pause_state, false);
@ -183,117 +180,14 @@ bool World::isLegends(df::game_type t)
return (t == game_type::VIEW_LEGENDS); return (t == game_type::VIEW_LEGENDS);
} }
static PersistentDataItem dataFromHFig(df::historical_figure *hfig)
{
return PersistentDataItem(hfig->id, hfig->name.first_name, &hfig->name.nickname, hfig->name.words);
}
// Hide fake histfigs from legends xml export
static bool in_export_xml = false;
struct hide_fake_histfigs_hook : df::viewscreen_legendsst {
typedef df::viewscreen_legendsst interpose_base;
DEFINE_VMETHOD_INTERPOSE(void, feed, (set<df::interface_key> *input))
{
if (input->count(interface_key::LEGENDS_EXPORT_XML))
{
auto &figs = df::historical_figure::get_vector();
auto it = figs.begin();
while (it != figs.end() && (*it)->id <= -100)
++it;
// Move our histfigs to a temporary vector
std::vector<df::historical_figure*> fakes(figs.begin(), it);
figs.erase(figs.begin(), it);
in_export_xml = true;
INTERPOSE_NEXT(feed)(input);
in_export_xml = false;
figs.insert(figs.begin(), fakes.begin(), fakes.end());
}
else
INTERPOSE_NEXT(feed)(input);
}
};
IMPLEMENT_VMETHOD_INTERPOSE_PRIO(hide_fake_histfigs_hook, feed, -10000);
void World::ClearPersistentCache()
{
next_persistent_id = 0;
persistent_index.clear();
INTERPOSE_HOOK(hide_fake_histfigs_hook, feed).apply(Core::getInstance().isWorldLoaded());
}
static bool BuildPersistentCache()
{
if (in_export_xml)
return false;
if (next_persistent_id)
return true;
if (!Core::getInstance().isWorldLoaded())
return false;
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
// Determine the next entry id as min(-100, lowest_id-1)
next_persistent_id = -100;
if (hfvec.size() > 0 && hfvec[0]->id <= -100)
next_persistent_id = hfvec[0]->id-1;
// Add the entries to the lookup table
persistent_index.clear();
for (size_t i = 0; i < hfvec.size() && hfvec[i]->id <= -100; i++)
{
if (!hfvec[i]->name.has_name || hfvec[i]->name.first_name.empty())
continue;
persistent_index.insert(T_persistent_item(hfvec[i]->name.first_name, -hfvec[i]->id));
}
return true;
}
PersistentDataItem World::AddPersistentData(const std::string &key) PersistentDataItem World::AddPersistentData(const std::string &key)
{ {
if (!BuildPersistentCache() || key.empty()) return Persistence::addItem(key);
return PersistentDataItem();
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
df::historical_figure *hfig = new df::historical_figure();
hfig->id = next_persistent_id;
hfig->name.has_name = true;
hfig->name.first_name = key;
memset(hfig->name.words, 0xFF, sizeof(hfig->name.words));
if (!hfvec.empty())
hfig->id = std::min(hfig->id, hfvec[0]->id-1);
next_persistent_id = hfig->id-1;
hfvec.insert(hfvec.begin(), hfig);
persistent_index.insert(T_persistent_item(key, -hfig->id));
return dataFromHFig(hfig);
} }
PersistentDataItem World::GetPersistentData(const std::string &key) PersistentDataItem World::GetPersistentData(const std::string &key)
{ {
if (!BuildPersistentCache()) return Persistence::getByKey(key);
return PersistentDataItem();
auto it = persistent_index.find(key);
if (it != persistent_index.end())
return GetPersistentData(it->second);
return PersistentDataItem();
} }
PersistentDataItem World::GetPersistentData(int entry_id) PersistentDataItem World::GetPersistentData(int entry_id)
@ -301,96 +195,45 @@ PersistentDataItem World::GetPersistentData(int entry_id)
if (entry_id < 100) if (entry_id < 100)
return PersistentDataItem(); return PersistentDataItem();
auto hfig = df::historical_figure::find(-entry_id); return Persistence::getByIndex(size_t(entry_id - 100));
if (hfig && hfig->name.has_name)
return dataFromHFig(hfig);
return PersistentDataItem();
} }
PersistentDataItem World::GetPersistentData(const std::string &key, bool *added) PersistentDataItem World::GetPersistentData(const std::string &key, bool *added)
{ {
if (added) *added = false; bool temp = false;
if (!added)
PersistentDataItem rv = GetPersistentData(key); added = &temp;
if (!rv.isValid())
{
if (added) *added = true;
rv = AddPersistentData(key);
}
return rv; return Persistence::getByKey(key, added);
} }
void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key, bool prefix) void World::GetPersistentData(std::vector<PersistentDataItem> *vec, const std::string &key, bool prefix)
{ {
vec->clear(); if (prefix && key.empty())
if (!BuildPersistentCache())
return;
auto eqrange = persistent_index.equal_range(key);
if (prefix)
{ {
if (key.empty()) Persistence::getAll(*vec);
}
else if (prefix)
{
std::string min = key;
if (min.back() != '/')
{ {
eqrange.first = persistent_index.begin(); min.push_back('/');
eqrange.second = persistent_index.end();
} }
else std::string max = min;
{ ++max.back();
std::string bound = key;
if (bound[bound.size()-1] != '/')
bound += "/";
eqrange.first = persistent_index.lower_bound(bound);
bound[bound.size()-1]++; Persistence::getAllByKeyRange(*vec, min, max);
eqrange.second = persistent_index.lower_bound(bound);
}
} }
else
for (auto it = eqrange.first; it != eqrange.second; ++it)
{ {
auto hfig = df::historical_figure::find(-it->second); Persistence::getAllByKey(*vec, key);
if (hfig && hfig->name.has_name)
vec->push_back(dataFromHFig(hfig));
} }
} }
bool World::DeletePersistentData(const PersistentDataItem &item) bool World::DeletePersistentData(const PersistentDataItem &item)
{ {
int id = item.raw_id(); return Persistence::deleteItem(item);
if (id > -100)
return false;
if (!BuildPersistentCache())
return false;
std::vector<df::historical_figure*> &hfvec = df::historical_figure::get_vector();
auto eqrange = persistent_index.equal_range(item.key());
for (auto it2 = eqrange.first; it2 != eqrange.second; )
{
auto it = it2; ++it2;
if (it->second != -id)
continue;
persistent_index.erase(it);
int idx = binsearch_index(hfvec, id);
if (idx >= 0) {
delete hfvec[idx];
hfvec.erase(hfvec.begin()+idx);
}
return true;
}
return false;
} }
df::tile_bitmask *World::getPersistentTilemask(const PersistentDataItem &item, df::map_block *block, bool create) df::tile_bitmask *World::getPersistentTilemask(const PersistentDataItem &item, df::map_block *block, bool create)

@ -6,6 +6,9 @@
#include <Export.h> #include <Export.h>
#include <PluginManager.h> #include <PluginManager.h>
// If you need to save data per-world:
//#include "modules/Persistence.h"
// DF data structure definition headers // DF data structure definition headers
#include "DataDefs.h" #include "DataDefs.h"
//#include "df/world.h" //#include "df/world.h"
@ -86,6 +89,25 @@ DFhackCExport command_result plugin_onupdate ( color_ostream &out )
} }
*/ */
// If you need to save or load world-specific data, define these functions.
// plugin_save is called when the game might be about to save the world,
// and plugin_load is called whenever a new world is loaded. If the plugin
// is loaded or unloaded while a world is active, plugin_save or plugin_load
// will be called immediately.
/*
DFhackCExport command_result plugin_save( color_ostream &out )
{
// Call functions in the Persistence module here.
return CR_OK;
}
DFhackCExport command_result plugin_load( color_ostream &out )
{
// Call functions in the Persistence module here.
return CR_OK;
}
*/
// A command! It sits around and looks pretty. And it's nice and friendly. // A command! It sits around and looks pretty. And it's nice and friendly.
command_result skeleton (color_ostream &out, std::vector <std::string> & parameters) command_result skeleton (color_ostream &out, std::vector <std::string> & parameters)
{ {