new logic for deserialization

develop
Myk Taylor 2023-03-15 21:54:48 -07:00
parent 3f0a93a9b1
commit 5a317820f9
No known key found for this signature in database
7 changed files with 972 additions and 684 deletions

@ -193,6 +193,13 @@ show-unit-syndromes
===================
Replaced with a GUI version: `gui/unit-syndromes`.
.. _stocksettings:
stocksettings
=============
Along with ``copystock``, ``loadstock`` and ``savestock``, replaced with the new
`stockpiles` API.
.. _warn-stuck-trees:
warn-stuck-trees

@ -53,24 +53,23 @@ Examples
Options
-------
``-s``, ``--stockpile <id>``
Specify a specific stockpile ID instead of using the one currently selected
in the UI.
``-i``, ``--include <comma separated list of elements to include>``
When exporting, you can include this option to select only specific elements
of the stockpile to record. If not specified, everything is included. When
the file is later imported, only the included settings will be modified. The
options are explained below in the next section.
``-d``, ``--disable``
When importing, treat the settings in the file as elements to *remove** from
the current stockpile configuration. Elements that are enabled in the file
will be *disabled* on the stockpile. No other stockpile configuration will
be changed.
``-e``, ``--enable``
When importing, treat the settings in the file as elements to *add* to the
current stockpile configuration. Elements that are enabled in the file will
be enabled on the stockpile, but nothing currently enabled on the stockpile
will be disabled.
``-s``, ``--stockpile <id>``
Specify a specific stockpile ID instead of using the one currently selected
in the UI.
``-m``, ``--mode (set|enable|disable)``
When importing, choose the algorithm used to apply the settings. In ``set``
mode (the default), the stockpile is cleared and the settings in the file
are enabled. In ``enable`` mode, enabled settings in the file are *added*
to the stockpile, but no other settings are changed. In ``disable`` mode,
enabled settings in the file are *removed* from the current stockpile
configuration, and nothing else is changed.
``-f``, ``--filter <filter>``
When importing, only modify the settings that contain the given substring.
Configuration elements
----------------------
@ -78,10 +77,11 @@ Configuration elements
The different configuration elements you can include in an exported settings file
are:
:general: Max bins, barrels, and wheelbarrows; whether the stockpile takes from
links only; whether organic and/or inorganic materials are allowed.
:categories: The top-level categories of items that are enabled for the stockpile,
like Ammo, Finished goods, or Stone.
:containers: Max bins, max barrels, and num wheelbarrows.
:general: Whether the stockpile takes from links only and whether organic
and/or inorganic materials are allowed.
:categories: The top-level categories of items that are enabled for the
stockpile, like Ammo, Finished goods, or Stone.
:types: The elements below the categories, which include the sub-categories, the
specific item types, and any toggles the category might have (like Prepared
meals for the Food category).

@ -74,10 +74,25 @@ local function get_sp_id(opts)
return nil
end
local included_elements = {
containers=1,
general=2,
categories=4,
types=8,
}
local function export_stockpile(name, opts)
assert_safe_name(name)
name = STOCKPILES_DIR .. '/' .. name
stockpiles_export(name, get_sp_id(opts))
local includedElements = 0
for _,inc in ipairs(opts.includes) do
if included_elements[inc] then
includedElements = includedElements | included_elements[inc]
end
end
stockpiles_export(name, get_sp_id(opts), includedElements)
end
local function import_stockpile(name, opts)
@ -92,7 +107,28 @@ local function import_stockpile(name, opts)
else
name = STOCKPILES_LIBRARY_DIR .. '/' .. name
end
stockpiles_import(name, get_sp_id(opts))
stockpiles_import(name, get_sp_id(opts), opts.mode, opts.filter)
end
local valid_includes = {general=true, categories=true, types=true}
local function parse_include(arg)
local includes = argparse.stringList(arg, 'include')
for _,v in ipairs(includes) do
if not valid_includes[v] then
qerror(('invalid included element: "%s"'):format(v))
end
end
return includes
end
local valid_modes = {set=true, enable=true, disable=true}
local function parse_mode(arg)
if not valid_modes[arg] then
qerror(('invalid mode: "%s"'):format(arg))
end
return arg
end
local function process_args(opts, args)
@ -101,10 +137,20 @@ local function process_args(opts, args)
return
end
opts.includes = {}
opts.mode = 'set'
opts.filter = ''
return argparse.processArgsGetopt(args, {
{'f', 'filter', has_arg=true,
handler=function(arg) opts.filter = arg end},
{'h', 'help', handler=function() opts.help = true end},
{'i', 'include', has_arg=true,
handler=function(arg) opts.includes = parse_include(arg) end},
{'m', 'mode', has_arg=true,
handler=function(arg) opts.mode = parse_mode(arg) end},
{'s', 'stockpile', has_arg=true,
handler=function(arg) opts.id = argparse.nonnegativeInt(art, 'stockpile') end},
handler=function(arg) opts.id = argparse.nonnegativeInt(arg, 'stockpile') end},
})
end

File diff suppressed because it is too large Load Diff

@ -14,16 +14,52 @@ namespace df
struct building_stockpilest;
}
enum IncludedElements {
INCLUDED_ELEMENTS_NONE = 0x00,
INCLUDED_ELEMENTS_CONTAINERS = 0x01,
INCLUDED_ELEMENTS_GENERAL = 0x02,
INCLUDED_ELEMENTS_CATEGORIES = 0x04,
INCLUDED_ELEMENTS_TYPES = 0x08,
};
enum DeserializeMode {
DESERIALIZE_MODE_SET = 0,
DESERIALIZE_MODE_ENABLE = 1,
DESERIALIZE_MODE_DISABLE = 2,
};
// read the token from the serialized list during import
typedef std::function<std::string(const size_t&)> FuncReadImport;
// add the token to the serialized list during export
typedef std::function<void(const std::string&)> FuncWriteExport;
// are item's of item_type allowed?
typedef std::function<bool(df::enums::item_type::item_type)> FuncItemAllowed;
// is this material allowed?
typedef std::function<bool(const DFHack::MaterialInfo&)> FuncMaterialAllowed;
// convenient struct for parsing food stockpile items
struct food_pair {
// exporting
FuncWriteExport set_value;
std::vector<char>* stockpile_values;
// importing
FuncReadImport get_value;
size_t serialized_count;
bool valid;
food_pair(FuncWriteExport s, std::vector<char>* sp_v, FuncReadImport g, size_t count)
: set_value(s), stockpile_values(sp_v), get_value(g), serialized_count(count), valid(true) { }
food_pair(): valid(false) { }
};
/**
* Class for serializing the stockpile_settings structure into a Google protobuf
*/
class StockpileSerializer {
public:
/**
* @param out for debugging
* @param stockpile stockpile to read or write settings to
*/
StockpileSerializer(df::building_stockpilest* stockpile);
~StockpileSerializer();
@ -32,191 +68,64 @@ public:
* Since we depend on protobuf-lite, not the full lib, we copy this function from
* protobuf message.cc
*/
bool serialize_to_ostream(std::ostream* output);
bool serialize_to_ostream(std::ostream* output, uint32_t includedElements);
/**
* Will serialize stockpile settings to a file (overwrites existing files)
* @return success/failure
*/
bool serialize_to_file(const std::string& file);
bool serialize_to_file(const std::string& file, uint32_t includedElements);
/**
* Again, copied from message.cc
*/
bool parse_from_istream(std::istream* input);
bool parse_from_istream(std::istream* input, DeserializeMode mode, const std::string& filter);
/**
* Read stockpile settings from file
*/
bool unserialize_from_file(const std::string& file);
bool unserialize_from_file(const std::string& file, DeserializeMode mode, const std::string& filter);
private:
df::building_stockpilest* mPile;
dfstockpiles::StockpileSettings mBuffer;
std::map<int, std::string> mOtherMatsFurniture;
std::map<int, std::string> mOtherMatsFinishedGoods;
std::map<int, std::string> mOtherMatsBars;
std::map<int, std::string> mOtherMatsBlocks;
std::map<int, std::string> mOtherMatsWeaponsArmor;
/**
read memory structures and serialize to protobuf
*/
void write();
// parse serialized data into ui indices
void read();
/**
* Find an enum's value based off the string label.
* @param traits the enum's trait struct
* @param token the string value in key_table
* @return the enum's value, -1 if not found
*/
template <typename E>
static typename df::enum_traits<E>::base_type linear_index(df::enum_traits<E> traits, const std::string& token) {
auto j = traits.first_item_value;
auto limit = traits.last_item_value;
// sometimes enums start at -1, which is bad news for array indexing
if (j < 0) {
j += abs(traits.first_item_value);
limit += abs(traits.first_item_value);
}
for (; j <= limit; ++j) {
if (token.compare(traits.key_table[j]) == 0)
return j;
}
return -1;
}
// read the token from the serailized list during import
typedef std::function<std::string(const size_t&)> FuncReadImport;
// add the token to the serialized list during export
typedef std::function<void(const std::string&)> FuncWriteExport;
// are item's of item_type allowed?
typedef std::function<bool(df::enums::item_type::item_type)> FuncItemAllowed;
// is this material allowed?
typedef std::function<bool(const DFHack::MaterialInfo&)> FuncMaterialAllowed;
// convenient struct for parsing food stockpile items
struct food_pair {
// exporting
FuncWriteExport set_value;
std::vector<char>* stockpile_values;
// importing
FuncReadImport get_value;
size_t serialized_count;
bool valid;
food_pair(FuncWriteExport s, std::vector<char>* sp_v, FuncReadImport g, size_t count)
: set_value(s), stockpile_values(sp_v), get_value(g), serialized_count(count), valid(true) { }
food_pair(): valid(false) { }
};
/**
* There are many repeated (un)serialization cases throughout the stockpile_settings structure,
* so the most common cases have been generalized into generic functions using lambdas.
*
* The basic process to serialize a stockpile_settings structure is:
* 1. loop through the list
* 2. for every element that is TRUE:
* 3. map the specific stockpile_settings index into a general material, creature, etc index
* 4. verify that type is allowed in the list (e.g., no stone in gems stockpiles)
* 5. add it to the protobuf using FuncWriteExport
*
* The unserialization process is the same in reverse.
*/
void serialize_list_organic_mat(FuncWriteExport add_value, const std::vector<char>* list, df::enums::organic_mat_category::organic_mat_category cat);
/**
* @see serialize_list_organic_mat
*/
void unserialize_list_organic_mat(FuncReadImport get_value, size_t list_size, std::vector<char>* pile_list, df::enums::organic_mat_category::organic_mat_category cat);
/**
* @see serialize_list_organic_mat
*/
void serialize_list_item_type(FuncItemAllowed is_allowed, FuncWriteExport add_value, const std::vector<char>& list);
/**
* @see serialize_list_organic_mat
*/
void unserialize_list_item_type(FuncItemAllowed is_allowed, FuncReadImport read_value, int32_t list_size, std::vector<char>* pile_list);
/**
* @see serialize_list_organic_mat
*/
void serialize_list_material(FuncMaterialAllowed is_allowed, FuncWriteExport add_value, const std::vector<char>& list);
/**
* @see serialize_list_organic_mat
*/
void unserialize_list_material(FuncMaterialAllowed is_allowed, FuncReadImport read_value, int32_t list_size, std::vector<char>* pile_list);
/**
* @see serialize_list_organic_mat
*/
void serialize_list_quality(FuncWriteExport add_value, const bool(&quality_list)[7]);
/**
* Set all values in a bool[7] to false
*/
void quality_clear(bool(&pile_list)[7]);
// read memory structures and serialize to protobuf
void write(uint32_t includedElements);
/**
* @see serialize_list_organic_mat
*/
void unserialize_list_quality(FuncReadImport read_value, int32_t list_size, bool(&pile_list)[7]);
// parse serialized data into ui indices
void read(DeserializeMode mode, const std::string& filter);
/**
* @see serialize_list_organic_mat
*/
void serialize_list_other_mats(const std::map<int, std::string> other_mats, FuncWriteExport add_value, std::vector<char> list);
/**
* @see serialize_list_organic_mat
*/
void unserialize_list_other_mats(const std::map<int, std::string> other_mats, FuncReadImport read_value, int32_t list_size, std::vector<char>* pile_list);
/**
* @see serialize_list_organic_mat
*/
void serialize_list_itemdef(FuncWriteExport add_value, std::vector<char> list, std::vector<df::itemdef*> items, df::enums::item_type::item_type type);
/**
* @see serialize_list_organic_mat
*/
void unserialize_list_itemdef(FuncReadImport read_value, int32_t list_size, std::vector<char>* pile_list, df::enums::item_type::item_type type);
/**
* Given a list of other_materials and an index, return its corresponding token
* Given a list of other_materials and an index, return its corresponding token
* @return empty string if not found
* @see other_mats_token
*/
std::string other_mats_index(const std::map<int, std::string> other_mats, int idx);
/**
* Given a list of other_materials and a token, return its corresponding index
* Given a list of other_materials and a token, return its corresponding index
* @return -1 if not found
* @see other_mats_index
*/
int other_mats_token(const std::map<int, std::string> other_mats, const std::string& token);
void write_containers();
void read_containers(DeserializeMode mode);
void write_general();
void read_general();
void read_general(DeserializeMode mode);
void write_animals();
void read_animals();
void write_animals(dfstockpiles::StockpileSettings::AnimalsSet* animals);
void read_animals(DeserializeMode mode, const std::string& filter);
food_pair food_map(df::enums::organic_mat_category::organic_mat_category cat);
void write_food();
void read_food();
void write_food(dfstockpiles::StockpileSettings::FoodSet* food);
void read_food(DeserializeMode mode, const std::string& filter);
void furniture_setup_other_mats();
void write_furniture();
void write_furniture(dfstockpiles::StockpileSettings::FurnitureSet* furniture);
bool furniture_mat_is_allowed(const DFHack::MaterialInfo& mi);
void read_furniture();
void read_furniture(DeserializeMode mode, const std::string& filter);
bool refuse_creature_is_allowed(const df::creature_raw* raw);
@ -224,60 +133,57 @@ private:
bool refuse_type_is_allowed(df::enums::item_type::item_type type);
void write_refuse();
void write_refuse(dfstockpiles::StockpileSettings::RefuseSet* refuse);
void refuse_read_helper(std::function<std::string(const size_t&)> get_value, size_t list_size, std::vector<char>* pile_list);
void read_refuse();
void read_refuse(DeserializeMode mode, const std::string& filter);
bool stone_is_allowed(const DFHack::MaterialInfo& mi);
void write_stone();
void write_stone(dfstockpiles::StockpileSettings::StoneSet* stone);
void read_stone();
void read_stone(DeserializeMode mode, const std::string& filter);
bool ammo_mat_is_allowed(const DFHack::MaterialInfo& mi);
void write_ammo();
void read_ammo();
bool write_ammo(dfstockpiles::StockpileSettings::AmmoSet* ammo);
void read_ammo(DeserializeMode mode, const std::string& filter);
bool coins_mat_is_allowed(const DFHack::MaterialInfo& mi);
void write_coins();
void read_coins();
void bars_blocks_setup_other_mats();
void write_coins(dfstockpiles::StockpileSettings::CoinSet* coins);
void read_coins(DeserializeMode mode, const std::string& filter);
bool bars_mat_is_allowed(const DFHack::MaterialInfo& mi);
bool blocks_mat_is_allowed(const DFHack::MaterialInfo& mi);
void write_bars_blocks();
void read_bars_blocks();
void write_bars_blocks(dfstockpiles::StockpileSettings::BarsBlocksSet* bars_blocks);
void read_bars_blocks(DeserializeMode mode, const std::string& filter);
bool gem_mat_is_allowed(const DFHack::MaterialInfo& mi);
bool gem_cut_mat_is_allowed(const DFHack::MaterialInfo& mi);
bool gem_other_mat_is_allowed(DFHack::MaterialInfo& mi);
void write_gems();
void write_gems(dfstockpiles::StockpileSettings::GemsSet* gems);
void read_gems();
void read_gems(DeserializeMode mode, const std::string& filter);
bool finished_goods_type_is_allowed(df::enums::item_type::item_type type);
void finished_goods_setup_other_mats();
bool finished_goods_mat_is_allowed(const DFHack::MaterialInfo& mi);
void write_finished_goods();
void read_finished_goods();
void write_leather();
void read_leather();
void write_cloth();
void read_cloth();
void write_finished_goods(dfstockpiles::StockpileSettings::FinishedGoodsSet* finished_goods);
void read_finished_goods(DeserializeMode mode, const std::string& filter);
void write_leather(dfstockpiles::StockpileSettings::LeatherSet* leather);
void read_leather(DeserializeMode mode, const std::string& filter);
void write_cloth(dfstockpiles::StockpileSettings::ClothSet* cloth);
void read_cloth(DeserializeMode mode, const std::string& filter);
bool wood_mat_is_allowed(const df::plant_raw* plant);
void write_wood();
void read_wood();
void write_wood(dfstockpiles::StockpileSettings::WoodSet* wood);
void read_wood(DeserializeMode mode, const std::string& filter);
bool weapons_mat_is_allowed(const DFHack::MaterialInfo& mi);
void write_weapons();
void read_weapons();
void weapons_armor_setup_other_mats();
void write_weapons(dfstockpiles::StockpileSettings::WeaponsSet* weapons);
void read_weapons(DeserializeMode mode, const std::string& filter);
bool armor_mat_is_allowed(const DFHack::MaterialInfo& mi);
void write_armor();
void read_armor();
void write_armor(dfstockpiles::StockpileSettings::ArmorSet* armor);
void read_armor(DeserializeMode mode, const std::string& filter);
void write_corpses(dfstockpiles::StockpileSettings::CorpsesSet* corpses);
void read_corpses(DeserializeMode mode, const std::string& filter);
void write_sheet(dfstockpiles::StockpileSettings::SheetSet* sheet);
void read_sheet(DeserializeMode mode, const std::string& filter);
};

@ -7,12 +7,14 @@ option optimize_for = LITE_RUNTIME;
message StockpileSettings {
message AnimalsSet {
optional bool all = 4;
optional bool empty_cages = 1;
optional bool empty_traps = 2;
repeated string enabled = 3;
}
message FoodSet {
optional bool all = 21;
repeated string meat = 1;
repeated string fish = 2;
repeated string unprepared_fish = 20;
@ -36,6 +38,7 @@ message StockpileSettings {
}
message FurnitureSet {
optional bool all = 7;
repeated string type = 1;
repeated string other_mats = 2;
repeated string mats = 3;
@ -44,6 +47,7 @@ message StockpileSettings {
// UNUSED: optional bool sand_bags = 6;
}
message RefuseSet {
optional bool all = 12;
repeated string type = 1;
repeated string corpses = 2;
repeated string body_parts = 3;
@ -57,12 +61,14 @@ message StockpileSettings {
optional bool rotten_raw_hide = 11;
}
message StoneSet {
repeated string mats = 1;
optional bool all = 2;
repeated string mats = 1;
}
message OreSet {
repeated string mats = 1;
repeated string mats = 1;
}
message AmmoSet {
optional bool all = 6;
repeated string type = 1;
repeated string other_mats = 2;
repeated string mats = 3;
@ -70,21 +76,25 @@ message StockpileSettings {
repeated string quality_total = 5;
}
message CoinSet {
optional bool all = 2;
repeated string mats = 1;
}
message BarsBlocksSet {
optional bool all = 5;
repeated string bars_other_mats = 1;
repeated string blocks_other_mats = 2;
repeated string bars_mats = 3;
repeated string blocks_mats = 4;
}
message GemsSet {
optional bool all = 5;
repeated string rough_other_mats = 1;
repeated string cut_other_mats = 2;
repeated string rough_mats = 3;
repeated string cut_mats = 4;
}
message FinishedGoodsSet {
optional bool all = 6;
repeated string type = 1;
repeated string other_mats = 2;
repeated string mats = 3;
@ -92,9 +102,11 @@ message StockpileSettings {
repeated string quality_total = 5;
}
message LeatherSet {
optional bool all = 2;
repeated string mats = 1;
}
message ClothSet {
optional bool all = 9;
repeated string thread_silk = 1;
repeated string thread_plant = 2;
repeated string thread_yarn = 3;
@ -105,9 +117,11 @@ message StockpileSettings {
repeated string cloth_metal = 8;
}
message WoodSet {
optional bool all = 2;
repeated string mats = 1;
}
message WeaponsSet {
optional bool all = 9;
repeated string weapon_type = 1;
repeated string trapcomp_type = 2;
repeated string other_mats = 3;
@ -117,8 +131,8 @@ message StockpileSettings {
optional bool usable = 7;
optional bool unusable = 8;
}
message ArmorSet {
optional bool all = 13;
repeated string body = 1;
repeated string head = 2;
repeated string feet = 3;
@ -129,33 +143,45 @@ message StockpileSettings {
repeated string mats = 8;
repeated string quality_core = 9;
repeated string quality_total = 10;
optional bool usable =11;
optional bool usable = 11;
optional bool unusable = 12;
}
message CorpsesSet {
optional bool all = 1;
}
message SheetSet {
optional bool all = 1;
}
// general settings
optional int32 max_barrels = 20;
optional int32 max_bins = 21;
optional int32 max_wheelbarrows = 22;
optional bool use_links_only = 23;
optional bool allow_organic = 18;
optional bool allow_inorganic = 19;
// categories
optional AmmoSet ammo = 8;
optional AnimalsSet animals = 1;
optional ArmorSet armor = 17;
optional BarsBlocksSet barsblocks = 10;
optional ClothSet cloth = 14;
optional CoinSet coin = 9;
optional FinishedGoodsSet finished_goods = 12;
optional FoodSet food = 2;
optional FurnitureSet furniture = 3;
optional int32 unknown1 = 4 [deprecated=true];
optional RefuseSet refuse = 5;
optional StoneSet stone = 6;
optional OreSet ore = 7;
optional AmmoSet ammo = 8;
optional CoinSet coin = 9;
optional BarsBlocksSet barsblocks = 10;
optional GemsSet gems = 11;
optional FinishedGoodsSet finished_goods = 12;
optional LeatherSet leather = 13;
optional ClothSet cloth = 14;
optional WoodSet wood = 15;
optional CorpsesSet corpses_v50 = 25;
optional RefuseSet refuse = 5;
optional SheetSet sheet = 26;
optional StoneSet stone = 6;
optional WeaponsSet weapons = 16;
optional ArmorSet armor = 17;
optional bool allow_organic = 18;
optional bool allow_inorganic = 19;
optional bool corpses = 24;
// extras
optional int32 max_barrels = 20;
optional int32 max_bins = 21;
optional int32 max_wheelbarrows = 22;
optional bool use_links_only = 23;
optional WoodSet wood = 15;
// deprecated
optional bool corpses = 24; // not marked as deprecated since we still read it
optional OreSet ore = 7 [deprecated=true];
optional int32 unknown1 = 4 [deprecated=true];
}

@ -83,7 +83,7 @@ static df::building_stockpilest* get_stockpile(int id) {
return virtual_cast<df::building_stockpilest>(df::building::find(id));
}
static bool stockpiles_export(color_ostream& out, string fname, int id) {
static bool stockpiles_export(color_ostream& out, string fname, int id, uint32_t includedElements) {
df::building_stockpilest* sp = get_stockpile(id);
if (!sp) {
out.printerr("Specified building isn't a stockpile: %d.\n", id);
@ -95,7 +95,7 @@ static bool stockpiles_export(color_ostream& out, string fname, int id) {
try {
StockpileSerializer cereal(sp);
if (!cereal.serialize_to_file(fname)) {
if (!cereal.serialize_to_file(fname, includedElements)) {
out.printerr("could not save to '%s'\n", fname.c_str());
return false;
}
@ -108,7 +108,7 @@ static bool stockpiles_export(color_ostream& out, string fname, int id) {
return true;
}
static bool stockpiles_import(color_ostream& out, string fname, int id) {
static bool stockpiles_import(color_ostream& out, string fname, int id, string mode_str, string filter) {
df::building_stockpilest* sp = get_stockpile(id);
if (!sp) {
out.printerr("Specified building isn't a stockpile: %d.\n", id);
@ -123,9 +123,15 @@ static bool stockpiles_import(color_ostream& out, string fname, int id) {
return false;
}
DeserializeMode mode = DESERIALIZE_MODE_SET;
if (mode_str == "enable")
mode = DESERIALIZE_MODE_ENABLE;
else if (mode_str == "disable")
mode = DESERIALIZE_MODE_DISABLE;
try {
StockpileSerializer cereal(sp);
if (!cereal.unserialize_from_file(fname)) {
if (!cereal.unserialize_from_file(fname, mode, filter)) {
out.printerr("deserialization failed: '%s'\n", fname.c_str());
return false;
}