Add Persistence module.
Alter World to use Persistence instead of storing data in historical figures. Fake historical figures will be converted to the new format when a world is loaded. Added plugin_save and plugin_load functions to the plugin API. Made the weird int7/int28 code that was in the old persistence API slightly safer.develop
Ben Lubar
parent
e56cb2a25a
commit
4e690df96a
@ -0,0 +1,231 @@
|
||||
/*
|
||||
https://github.com/DFHack/dfhack
|
||||
Copyright (c) 2009-2018 DFHack Team
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any
|
||||
damages arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any
|
||||
purpose, including commercial applications, and to alter it and
|
||||
redistribute it freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must
|
||||
not claim that you wrote the original software. If you use this
|
||||
software in a product, an acknowledgment in the product documentation
|
||||
would be appreciated but is not required.
|
||||
|
||||
2. Altered source versions must be plainly marked as such, and
|
||||
must not be misrepresented as being the original software.
|
||||
|
||||
3. This notice may not be removed or altered from any source
|
||||
distribution.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#ifndef CL_MOD_PERSISTENCE
|
||||
#define CL_MOD_PERSISTENCE
|
||||
|
||||
/**
|
||||
* \defgroup grp_persistence Persistence: code related to saving and loading data
|
||||
* @ingroup grp_modules
|
||||
*/
|
||||
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Export.h"
|
||||
#include "Error.h"
|
||||
|
||||
namespace DFHack
|
||||
{
|
||||
class Core;
|
||||
|
||||
namespace Persistence
|
||||
{
|
||||
struct LegacyData;
|
||||
class Internal;
|
||||
}
|
||||
|
||||
class DFHACK_EXPORT PersistentDataItem {
|
||||
size_t index;
|
||||
std::shared_ptr<Persistence::LegacyData> data;
|
||||
public:
|
||||
static const int NumInts = 7;
|
||||
|
||||
bool isValid() const;
|
||||
size_t get_index() const
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(isValid());
|
||||
return index;
|
||||
}
|
||||
int entry_id() const { return isValid() ? int(index) + 100 : 0; }
|
||||
|
||||
int raw_id() const { return isValid() ? -int(index) - 100 : 0; }
|
||||
|
||||
const std::string &key() const;
|
||||
|
||||
std::string &val();
|
||||
const std::string &val() const;
|
||||
int &ival(int i);
|
||||
int ival(int i) const;
|
||||
|
||||
// Pack binary data into string field.
|
||||
// Since DF serialization chokes on NUL bytes,
|
||||
// use bit magic to ensure none of the bytes is 0.
|
||||
// Choose the lowest bit for padding so that
|
||||
// sign-extend can be used normally.
|
||||
|
||||
size_t data_size() const
|
||||
{
|
||||
return val().size();
|
||||
}
|
||||
|
||||
bool check_data(size_t off, size_t sz = 1) const
|
||||
{
|
||||
return (data_size() >= off + sz);
|
||||
}
|
||||
void ensure_data(size_t off, size_t sz = 0)
|
||||
{
|
||||
if (data_size() < off + sz)
|
||||
{
|
||||
val().resize(off + sz, '\x01');
|
||||
}
|
||||
}
|
||||
template<size_t N>
|
||||
uint8_t (&pdata(size_t off))[N]
|
||||
{
|
||||
ensure_data(off, N);
|
||||
typedef uint8_t array[N];
|
||||
return *(array *)(val().data() + off);
|
||||
}
|
||||
template<size_t N>
|
||||
const uint8_t (&pdata(size_t off) const)[N]
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(check_data(off, N));
|
||||
typedef const uint8_t array[N];
|
||||
return *(array *)(val().data() + off);
|
||||
}
|
||||
template<size_t N>
|
||||
const uint8_t (&cpdata(size_t off) const)[N]
|
||||
{
|
||||
return pdata<N>(off);
|
||||
}
|
||||
|
||||
static const size_t int7_size = 1;
|
||||
uint8_t get_uint7(size_t off) const
|
||||
{
|
||||
auto p = pdata<int7_size>(off);
|
||||
return p[0] >> 1;
|
||||
}
|
||||
int8_t get_int7(size_t off) const
|
||||
{
|
||||
return int8_t(get_uint7(off));
|
||||
}
|
||||
void set_uint7(size_t off, uint8_t val)
|
||||
{
|
||||
auto p = pdata<int7_size>(off);
|
||||
p[0] = uint8_t((val << 1) | 1);
|
||||
}
|
||||
void set_int7(size_t off, int8_t val)
|
||||
{
|
||||
set_uint7(off, uint8_t(val));
|
||||
}
|
||||
|
||||
static const size_t int28_size = 4;
|
||||
uint32_t get_uint28(size_t off) const
|
||||
{
|
||||
auto p = pdata<int28_size>(off);
|
||||
return (p[0]>>1) | ((p[1]&~1U)<<6) | ((p[2]&~1U)<<13) | ((p[3]&~1U)<<20);
|
||||
}
|
||||
int32_t get_int28(size_t off) const
|
||||
{
|
||||
return int32_t(get_uint28(off));
|
||||
}
|
||||
void set_uint28(size_t off, uint32_t val)
|
||||
{
|
||||
auto p = pdata<int28_size>(off);
|
||||
p[0] = uint8_t((val<<1) | 1);
|
||||
p[1] = uint8_t((val>>6) | 1);
|
||||
p[2] = uint8_t((val>>13) | 1);
|
||||
p[3] = uint8_t((val>>20) | 1);
|
||||
}
|
||||
void set_int28(size_t off, int32_t val)
|
||||
{
|
||||
set_uint28(off, val);
|
||||
}
|
||||
|
||||
PersistentDataItem() : index(0), data(nullptr) {}
|
||||
PersistentDataItem(size_t index, const std::shared_ptr<Persistence::LegacyData> &data)
|
||||
: index(index), data(data) {}
|
||||
};
|
||||
namespace Persistence
|
||||
{
|
||||
class Internal
|
||||
{
|
||||
static void clear();
|
||||
static void save();
|
||||
static void load();
|
||||
friend class ::DFHack::Core;
|
||||
};
|
||||
|
||||
// Returns a new PersistentDataItem with the specified key.
|
||||
// If there is no world loaded or the key is empty, returns an invalid item.
|
||||
DFHACK_EXPORT PersistentDataItem addItem(const std::string &key);
|
||||
// Returns an existing PersistentDataItem with the specified key.
|
||||
// If "added" is not null and there is no such item, a new item is returned and
|
||||
// the bool value is set to true. If "added" is not null and an item is found or
|
||||
// no new item can be created, the bool value is set to false.
|
||||
DFHACK_EXPORT PersistentDataItem getByKey(const std::string &key, bool *added = nullptr);
|
||||
// Returns an existing PersistentDataItem with the specified index.
|
||||
// If there is no world loaded or the index is empty, returns an invalid item.
|
||||
DFHACK_EXPORT PersistentDataItem getByIndex(size_t index);
|
||||
// If the item is invalid, returns false. Otherwise, deletes the item and returns
|
||||
// true. All references to the item are invalid as soon as this function returns.
|
||||
DFHACK_EXPORT bool deleteItem(const PersistentDataItem &item);
|
||||
// Fills the vector with references to each persistent item.
|
||||
DFHACK_EXPORT void getAll(std::vector<PersistentDataItem> &vec);
|
||||
// Fills the vector with references to each persistent item with a key that is
|
||||
// greater than or equal to "min" and less than "max".
|
||||
DFHACK_EXPORT void getAllByKeyRange(std::vector<PersistentDataItem> &vec, const std::string &min, const std::string &max);
|
||||
// Fills the vector with references to each persistent item with a key that is
|
||||
// equal to the given key.
|
||||
DFHACK_EXPORT void getAllByKey(std::vector<PersistentDataItem> &vec, const std::string &key);
|
||||
|
||||
#if defined(__GNUC__) && __GNUC__ < 5
|
||||
// file stream move constructors are missing in libstdc++ before version 5.
|
||||
template<typename T>
|
||||
class DFHACK_EXPORT gcc_4_fstream_shim : public T
|
||||
{
|
||||
const std::string file;
|
||||
public:
|
||||
explicit gcc_4_fstream_shim() : T(), file() {}
|
||||
explicit gcc_4_fstream_shim(const std::string &file) : T(file), file() {}
|
||||
gcc_4_fstream_shim(gcc_4_fstream_shim<T> && s) : T(), file(s.file)
|
||||
{
|
||||
if (!file.empty())
|
||||
{
|
||||
T::open(file.c_str());
|
||||
}
|
||||
}
|
||||
};
|
||||
#define FSTREAM(x) gcc_4_fstream_shim<x>
|
||||
#else
|
||||
#define FSTREAM(x) x
|
||||
#endif
|
||||
|
||||
// Returns an input stream that data can be read from. If no world is currently loaded,
|
||||
// or no data has been saved with the specified key, the stream is invalid and acts
|
||||
// as if the file is empty.
|
||||
DFHACK_EXPORT FSTREAM(std::ifstream) readSaveData(const std::string &key);
|
||||
// Returns an output stream that data can be saved to. If no world is currently loaded,
|
||||
// an invalid stream is returned, and writing to it has no effect.
|
||||
DFHACK_EXPORT FSTREAM(std::ofstream) writeSaveData(const std::string &key);
|
||||
|
||||
#undef FSTREAM
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,420 @@
|
||||
/*
|
||||
https://github.com/peterix/dfhack
|
||||
Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any
|
||||
damages arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any
|
||||
purpose, including commercial applications, and to alter it and
|
||||
redistribute it freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must
|
||||
not claim that you wrote the original software. If you use this
|
||||
software in a product, an acknowledgment in the product documentation
|
||||
would be appreciated but is not required.
|
||||
|
||||
2. Altered source versions must be plainly marked as such, and
|
||||
must not be misrepresented as being the original software.
|
||||
|
||||
3. This notice may not be removed or altered from any source
|
||||
distribution.
|
||||
*/
|
||||
|
||||
#include "Internal.h"
|
||||
#include <array>
|
||||
#include <map>
|
||||
#include <json/json.h>
|
||||
|
||||
#include "Core.h"
|
||||
#include "DataDefs.h"
|
||||
#include "modules/Persistence.h"
|
||||
#include "modules/World.h"
|
||||
|
||||
#include "df/historical_figure.h"
|
||||
|
||||
using namespace DFHack;
|
||||
|
||||
static std::vector<std::shared_ptr<Persistence::LegacyData>> legacy_data;
|
||||
static std::multimap<std::string, size_t> index_cache;
|
||||
|
||||
struct Persistence::LegacyData
|
||||
{
|
||||
const std::string key;
|
||||
std::string str_value;
|
||||
std::array<int, PersistentDataItem::NumInts> int_values;
|
||||
|
||||
explicit LegacyData(const std::string &key) : key(key)
|
||||
{
|
||||
for (int i = 0; i < PersistentDataItem::NumInts; i++)
|
||||
{
|
||||
int_values.at(i) = -1;
|
||||
}
|
||||
}
|
||||
explicit LegacyData(Json::Value &json) : key(json["k"].asString())
|
||||
{
|
||||
str_value = json["s"].asString();
|
||||
for (int i = 0; i < PersistentDataItem::NumInts; i++)
|
||||
{
|
||||
int_values.at(i) = json["i"][i].asInt();
|
||||
}
|
||||
}
|
||||
explicit LegacyData(const df::language_name &name) : key(name.first_name)
|
||||
{
|
||||
str_value = name.nickname;
|
||||
for (int i = 0; i < PersistentDataItem::NumInts; i++)
|
||||
{
|
||||
int_values.at(i) = name.words[i];
|
||||
}
|
||||
}
|
||||
|
||||
Json::Value toJSON()
|
||||
{
|
||||
Json::Value json(Json::objectValue);
|
||||
json["k"] = key;
|
||||
json["s"] = str_value;
|
||||
Json::Value ints(Json::arrayValue);
|
||||
for (int i = 0; i < PersistentDataItem::NumInts; i++)
|
||||
{
|
||||
ints[i] = int_values.at(i);
|
||||
}
|
||||
json["i"] = std::move(ints);
|
||||
|
||||
return std::move(json);
|
||||
}
|
||||
};
|
||||
|
||||
const std::string &PersistentDataItem::key() const
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(isValid());
|
||||
return data->key;
|
||||
}
|
||||
|
||||
std::string &PersistentDataItem::val()
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(isValid());
|
||||
return data->str_value;
|
||||
}
|
||||
const std::string &PersistentDataItem::val() const
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(isValid());
|
||||
return data->str_value;
|
||||
}
|
||||
int &PersistentDataItem::ival(int i)
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(isValid());
|
||||
CHECK_INVALID_ARGUMENT(i >= 0 && i < NumInts);
|
||||
return data->int_values.at(i);
|
||||
}
|
||||
int PersistentDataItem::ival(int i) const
|
||||
{
|
||||
CHECK_INVALID_ARGUMENT(isValid());
|
||||
CHECK_INVALID_ARGUMENT(i >= 0 && i < NumInts);
|
||||
return data->int_values.at(i);
|
||||
}
|
||||
|
||||
bool PersistentDataItem::isValid() const
|
||||
{
|
||||
if (data == nullptr)
|
||||
return false;
|
||||
|
||||
CoreSuspender suspend;
|
||||
|
||||
if (legacy_data.size() <= index)
|
||||
return false;
|
||||
|
||||
return legacy_data.at(index) == data;
|
||||
}
|
||||
|
||||
void Persistence::Internal::clear()
|
||||
{
|
||||
CoreSuspender suspend;
|
||||
|
||||
legacy_data.clear();
|
||||
index_cache.clear();
|
||||
}
|
||||
|
||||
void Persistence::Internal::save()
|
||||
{
|
||||
CoreSuspender suspend;
|
||||
|
||||
Json::Value json(Json::arrayValue);
|
||||
for (size_t i = 0; i < legacy_data.size(); i++)
|
||||
{
|
||||
if (legacy_data.at(i) != nullptr)
|
||||
{
|
||||
while (json.size() < i)
|
||||
{
|
||||
json[json.size()] = Json::Value();
|
||||
}
|
||||
|
||||
json[int(i)] = legacy_data.at(i)->toJSON();
|
||||
}
|
||||
}
|
||||
|
||||
auto file = writeSaveData("legacy-data");
|
||||
file << json;
|
||||
}
|
||||
|
||||
static void convertHFigs()
|
||||
{
|
||||
auto &figs = df::historical_figure::get_vector();
|
||||
|
||||
auto src = figs.begin();
|
||||
while (src != figs.end() && (*src)->id > -100)
|
||||
{
|
||||
++src;
|
||||
}
|
||||
|
||||
if (src == figs.end())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
auto dst = src;
|
||||
while (src != figs.end())
|
||||
{
|
||||
auto fig = *src;
|
||||
if (fig->id > -100)
|
||||
{
|
||||
*dst = *src;
|
||||
++dst;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (fig->name.has_name && !fig->name.first_name.empty())
|
||||
{
|
||||
size_t index = size_t(-fig->id - 100);
|
||||
if (legacy_data.size() <= index)
|
||||
{
|
||||
legacy_data.resize(index + 1);
|
||||
}
|
||||
legacy_data.at(index) = std::shared_ptr<Persistence::LegacyData>(new Persistence::LegacyData(fig->name));
|
||||
}
|
||||
delete fig;
|
||||
}
|
||||
++src;
|
||||
}
|
||||
|
||||
figs.erase(dst, figs.end());
|
||||
}
|
||||
|
||||
void Persistence::Internal::load()
|
||||
{
|
||||
CoreSuspender suspend;
|
||||
|
||||
clear();
|
||||
|
||||
auto file = readSaveData("legacy-data");
|
||||
Json::Value json;
|
||||
try
|
||||
{
|
||||
file >> json;
|
||||
}
|
||||
catch (std::exception &)
|
||||
{
|
||||
// empty file?
|
||||
}
|
||||
|
||||
if (json.isArray())
|
||||
{
|
||||
legacy_data.resize(json.size());
|
||||
for (size_t i = 0; i < legacy_data.size(); i++)
|
||||
{
|
||||
if (json[int(i)].isObject())
|
||||
{
|
||||
legacy_data.at(i) = std::shared_ptr<LegacyData>(new LegacyData(json[int(i)]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
convertHFigs();
|
||||
|
||||
for (size_t i = 0; i < legacy_data.size(); i++)
|
||||
{
|
||||
if (legacy_data.at(i) == nullptr)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
index_cache.insert(std::make_pair(legacy_data.at(i)->key, i));
|
||||
}
|
||||
}
|
||||
|
||||
PersistentDataItem Persistence::addItem(const std::string &key)
|
||||
{
|
||||
if (key.empty() || !Core::getInstance().isWorldLoaded())
|
||||
return PersistentDataItem();
|
||||
|
||||
CoreSuspender suspend;
|
||||
|
||||
size_t index = 0;
|
||||
while (index < legacy_data.size() && legacy_data.at(index) != nullptr)
|
||||
{
|
||||
index++;
|
||||
}
|
||||
|
||||
auto ptr = std::shared_ptr<LegacyData>(new LegacyData(key));
|
||||
|
||||
if (index == legacy_data.size())
|
||||
{
|
||||
legacy_data.push_back(ptr);
|
||||
}
|
||||
else
|
||||
{
|
||||
legacy_data.at(index) = ptr;
|
||||
}
|
||||
|
||||
index_cache.insert(std::make_pair(key, index));
|
||||
|
||||
return PersistentDataItem(index, ptr);
|
||||
}
|
||||
|
||||
PersistentDataItem Persistence::getByKey(const std::string &key, bool *added)
|
||||
{
|
||||
CoreSuspender suspend;
|
||||
|
||||
auto it = index_cache.find(key);
|
||||
|
||||
if (added)
|
||||
{
|
||||
*added = it == index_cache.end();
|
||||
}
|
||||
|
||||
if (it != index_cache.end())
|
||||
{
|
||||
return PersistentDataItem(it->second, legacy_data.at(it->second));
|
||||
}
|
||||
|
||||
if (!added)
|
||||
{
|
||||
return PersistentDataItem();
|
||||
}
|
||||
|
||||
return addItem(key);
|
||||
}
|
||||
|
||||
PersistentDataItem Persistence::getByIndex(size_t index)
|
||||
{
|
||||
CoreSuspender suspend;
|
||||
|
||||
if (index < legacy_data.size() && legacy_data.at(index) != nullptr)
|
||||
{
|
||||
return PersistentDataItem(index, legacy_data.at(index));
|
||||
}
|
||||
|
||||
return PersistentDataItem();
|
||||
}
|
||||
|
||||
bool Persistence::deleteItem(const PersistentDataItem &item)
|
||||
{
|
||||
CoreSuspender suspend;
|
||||
|
||||
if (!item.isValid())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t index = item.get_index();
|
||||
auto range = index_cache.equal_range(item.key());
|
||||
for (auto it = range.first; it != range.second; ++it)
|
||||
{
|
||||
if (it->second == index)
|
||||
{
|
||||
index_cache.erase(it);
|
||||
break;
|
||||
}
|
||||
}
|
||||
legacy_data.at(index) = nullptr;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Persistence::getAll(std::vector<PersistentDataItem> &vec)
|
||||
{
|
||||
vec.clear();
|
||||
|
||||
CoreSuspender suspend;
|
||||
|
||||
for (size_t i = 0; i < legacy_data.size(); i++)
|
||||
{
|
||||
if (legacy_data.at(i) != nullptr)
|
||||
{
|
||||
vec.push_back(PersistentDataItem(i, legacy_data.at(i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Persistence::getAllByKeyRange(std::vector<PersistentDataItem> &vec, const std::string &min, const std::string &max)
|
||||
{
|
||||
vec.clear();
|
||||
|
||||
CoreSuspender suspend;
|
||||
|
||||
auto begin = index_cache.lower_bound(min);
|
||||
auto end = index_cache.lower_bound(max);
|
||||
for (auto it = begin; it != end; ++it)
|
||||
{
|
||||
vec.push_back(PersistentDataItem(it->second, legacy_data.at(it->second)));
|
||||
}
|
||||
}
|
||||
|
||||
void Persistence::getAllByKey(std::vector<PersistentDataItem> &vec, const std::string &key)
|
||||
{
|
||||
vec.clear();
|
||||
|
||||
CoreSuspender suspend;
|
||||
|
||||
auto range = index_cache.equal_range(key);
|
||||
for (auto it = range.first; it != range.second; ++it)
|
||||
{
|
||||
vec.push_back(PersistentDataItem(it->second, legacy_data.at(it->second)));
|
||||
}
|
||||
}
|
||||
|
||||
static std::string filterSaveFileName(std::string s)
|
||||
{
|
||||
for (auto &ch : s)
|
||||
{
|
||||
if (!isalnum(ch) && ch != '-' && ch != '_')
|
||||
{
|
||||
ch = '_';
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
static std::string getSaveFilePath(const std::string &world, const std::string &name)
|
||||
{
|
||||
return "data/save/" + world + "/dfhack-" + filterSaveFileName(name) + ".dat";
|
||||
}
|
||||
|
||||
#if defined(__GNUC__) && __GNUC__ < 5
|
||||
// file stream move constructors are missing in libstdc++ before version 5.
|
||||
#define FSTREAM(x) Persistence::gcc_4_fstream_shim<x>
|
||||
#else
|
||||
#define FSTREAM(x) x
|
||||
#endif
|
||||
FSTREAM(std::ifstream) Persistence::readSaveData(const std::string &name)
|
||||
{
|
||||
if (!Core::getInstance().isWorldLoaded())
|
||||
{
|
||||
// No world loaded - return unopened stream.
|
||||
return FSTREAM(std::ifstream)();
|
||||
}
|
||||
|
||||
return FSTREAM(std::ifstream)(getSaveFilePath(World::ReadWorldFolder(), name));
|
||||
}
|
||||
|
||||
FSTREAM(std::ofstream) Persistence::writeSaveData(const std::string &name)
|
||||
{
|
||||
if (!Core::getInstance().isWorldLoaded())
|
||||
{
|
||||
// No world loaded - return unopened stream.
|
||||
return FSTREAM(std::ofstream)();
|
||||
}
|
||||
|
||||
return FSTREAM(std::ofstream)(getSaveFilePath("current", name));
|
||||
}
|
||||
#undef FSTREAM
|
||||
Loading…
Reference in New Issue